@bendyline/gezel-knowledge 1.0.7 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +28 -17
  2. package/dist/index.d.ts +606 -161
  3. package/dist/index.js +2389 -508
  4. package/package.json +6 -4
package/dist/index.js CHANGED
@@ -1,13 +1,28 @@
1
1
  // src/format/constants.ts
2
- var GEZK_FORMAT_VERSION = 1;
3
- var GEZK_INDEX_SCHEMA_VERSION = 1;
4
- var GEZK_APPLICATION_ID = 1195727435;
5
- var ROUTER_DB_PATH = "index/router.db";
6
- var MANIFEST_PATH = "manifest.json";
2
+ import {
3
+ BODY_CODEC_MIN_BYTES,
4
+ GEZK_APPLICATION_ID,
5
+ GEZK_FORMAT_GENERATIONS,
6
+ GEZK_FORMAT_VERSION,
7
+ GEZK_INDEX_SCHEMA_VERSION,
8
+ GEZK_MIME_TYPE,
9
+ GEZK_SUPPORTED_FORMAT_VERSIONS,
10
+ GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
11
+ LICENSE_NOTICE_PATH,
12
+ MANIFEST_PATH,
13
+ MAX_KNOWLEDGE_DOCUMENT_BYTES,
14
+ MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
15
+ MAX_KNOWLEDGE_TOPIC_DEPTH,
16
+ MIMETYPE_PATH,
17
+ README_PATH,
18
+ ROUTER_DB_PATH,
19
+ SOURCE_NOTICES_PATH,
20
+ ZIP_FIXED_MTIME,
21
+ isSupportedFormatVersion,
22
+ isSupportedIndexSchemaVersion
23
+ } from "@bendyline/gezk";
7
24
  var SHARD_TARGET_CHUNKS = 2e5;
8
25
  var SHARD_MAX_CHUNKS = 25e4;
9
- var BODY_CODEC_MIN_BYTES = 512;
10
- var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
11
26
  var CENTROID_CHUNKS_PER = 12500;
12
27
  var CENTROID_MAX_PER_SHARD = 32;
13
28
  var CENTROID_KMEANS_MAX_ITER = 25;
@@ -18,144 +33,16 @@ var RERANK_FINAL_K = 24;
18
33
  function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
19
34
  return Math.min(512, Math.max(128, 8 * finalK), chunkCount);
20
35
  }
21
- var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
22
36
 
23
37
  // src/format/ddl.ts
24
- var ROUTER_DDL = `
25
- CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
26
-
27
- CREATE TABLE topics(
28
- id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,
29
- description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL
30
- );
31
-
32
- CREATE TABLE documents(
33
- id TEXT PRIMARY KEY,
34
- title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
35
- language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
36
- shard_id INTEGER NOT NULL,
37
- chunk_count INTEGER NOT NULL,
38
- source_url TEXT, source_revision TEXT, source_updated_at TEXT,
39
- attribution_json TEXT,
40
- body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
41
- body_blob BLOB NOT NULL
42
- );
43
- CREATE INDEX documents_topic ON documents(topic_id, slug);
44
- CREATE INDEX documents_shard ON documents(shard_id);
45
-
46
- CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
47
- PRIMARY KEY (alias, document_id)) WITHOUT ROWID;
48
-
49
- CREATE TABLE shards(
50
- id INTEGER PRIMARY KEY,
51
- path TEXT NOT NULL,
52
- chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,
53
- topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,
54
- bytes INTEGER NOT NULL
55
- );
56
-
57
- CREATE TABLE route_centroids(
58
- id INTEGER PRIMARY KEY,
59
- shard_id INTEGER NOT NULL REFERENCES shards(id),
60
- embedding BLOB NOT NULL,
61
- weight INTEGER NOT NULL
62
- );
63
- CREATE INDEX route_centroids_shard ON route_centroids(shard_id);
64
-
65
- CREATE VIRTUAL TABLE fts_documents USING fts5(
66
- title, summary, aliases, document_id UNINDEXED,
67
- tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
68
- );
69
- `;
70
- var SHARD_DDL = `
71
- CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
72
-
73
- CREATE TABLE chunks(
74
- id INTEGER PRIMARY KEY,
75
- chunk_uid TEXT NOT NULL,
76
- document_id TEXT NOT NULL,
77
- ordinal INTEGER NOT NULL,
78
- title TEXT NOT NULL,
79
- heading_path TEXT NOT NULL,
80
- heading_text TEXT NOT NULL,
81
- line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,
82
- token_count INTEGER NOT NULL,
83
- content_hash TEXT NOT NULL,
84
- text TEXT NOT NULL
85
- );
86
- CREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);
87
- CREATE INDEX chunks_document ON chunks(document_id, ordinal);
88
-
89
- CREATE VIRTUAL TABLE fts_chunks USING fts5(
90
- title, heading_text, text,
91
- content = 'chunks', content_rowid = 'id',
92
- tokenize = 'unicode61 remove_diacritics 2'
93
- );
94
-
95
- CREATE TABLE chunk_vectors_int8(
96
- chunk_id INTEGER PRIMARY KEY,
97
- v BLOB NOT NULL
98
- );
99
- `;
100
- function vecChunksDdl(dim) {
101
- return `CREATE VIRTUAL TABLE vec_chunks USING vec0(
102
- chunk_id INTEGER PRIMARY KEY,
103
- embedding bit[${dim}],
104
- chunk_size=1024
105
- );`;
106
- }
38
+ import { ROUTER_DDL, SHARD_DDL } from "@bendyline/gezk";
107
39
 
108
40
  // src/format/ids.ts
109
- import { createHash } from "crypto";
110
- function chunkUid(documentId, ordinal, chunkText) {
111
- const textHash = createHash("sha256").update(chunkText, "utf8").digest();
112
- return createHash("sha256").update(documentId, "utf8").update(Buffer.from([0])).update(String(ordinal), "utf8").update(Buffer.from([0])).update(textHash).digest("hex").slice(0, 32);
113
- }
114
- function chunkContentHash(chunkText) {
115
- return createHash("sha256").update(chunkText, "utf8").digest("hex");
116
- }
117
- function documentSlug(title) {
118
- const slug = title.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
119
- return slug || "untitled";
120
- }
41
+ import { documentSlug } from "@bendyline/gezk";
42
+ import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
121
43
 
122
44
  // src/format/quantize.ts
123
- function quantizeInt8(unitVector) {
124
- const out = new Int8Array(unitVector.length);
125
- for (let i = 0; i < unitVector.length; i++) {
126
- const q = Math.round(unitVector[i] * 127);
127
- out[i] = q > 127 ? 127 : q < -127 ? -127 : q;
128
- }
129
- return out;
130
- }
131
- function quantizeBinary(unitVector) {
132
- const bytes = Math.ceil(unitVector.length / 8);
133
- const out = new Uint8Array(bytes);
134
- for (let i = 0; i < unitVector.length; i++) {
135
- if (unitVector[i] > 0) {
136
- out[i >> 3] = out[i >> 3] | 1 << (i & 7);
137
- }
138
- }
139
- return out;
140
- }
141
- function rerankScore(query, passageInt8) {
142
- let dot = 0;
143
- const n = Math.min(query.length, passageInt8.length);
144
- for (let i = 0; i < n; i++) dot += query[i] * (passageInt8[i] / 127);
145
- return dot;
146
- }
147
- function l2Normalize(vector) {
148
- let sum = 0;
149
- for (let i = 0; i < vector.length; i++) {
150
- const v = vector[i];
151
- sum += v * v;
152
- }
153
- const norm = Math.sqrt(sum);
154
- if (!Number.isFinite(norm) || norm === 0) throw new Error("degenerate vector (zero norm)");
155
- const out = new Float32Array(vector.length);
156
- for (let i = 0; i < vector.length; i++) out[i] = vector[i] / norm;
157
- return out;
158
- }
45
+ import { l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from "@bendyline/gezk";
159
46
 
160
47
  // src/chunking/markdown-chunker.ts
161
48
  var HEADING = /^(#{1,6})\s+(.*)$/;
@@ -284,12 +171,24 @@ function splitOversizeLine(line, opts) {
284
171
  }
285
172
 
286
173
  // src/compiler/compile.ts
287
- import { createHash as createHash3 } from "crypto";
288
- import { mkdirSync, rmSync } from "fs";
174
+ import { createHash } from "crypto";
175
+ import { mkdirSync, readFileSync, rmSync, statSync } from "fs";
289
176
  import { stat } from "fs/promises";
290
177
  import { join } from "path";
291
178
  import { brotliCompressSync, constants as zlibConstants } from "zlib";
292
- import { KnowledgeCatalogManifestSchema } from "@bendyline/gezel";
179
+ import {
180
+ GEZK_MANIFEST_KIND,
181
+ KnowledgeAssetPathSchema,
182
+ KnowledgeCatalogManifestSchema,
183
+ MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
184
+ MAX_KNOWLEDGE_ASSET_BYTES,
185
+ MAX_KNOWLEDGE_ASSET_COUNT,
186
+ assetExtension,
187
+ assetKindForExtension,
188
+ canonicalizeJson,
189
+ sniffAssetType,
190
+ svgInertnessProblem
191
+ } from "@bendyline/gezk";
293
192
 
294
193
  // src/archive/write.ts
295
194
  import { createWriteStream } from "fs";
@@ -301,9 +200,13 @@ async function writeGezkArchive(outputPath, entries) {
301
200
  const yazl = nodeRequire("yazl");
302
201
  await mkdir(dirname(outputPath), { recursive: true });
303
202
  const zip = new yazl.ZipFile();
304
- const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress: true };
203
+ const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress: false };
204
+ zip.addBuffer(Buffer.from(GEZK_MIME_TYPE, "utf8"), MIMETYPE_PATH, options);
305
205
  const sorted = [...entries].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
306
206
  for (const entry of sorted) {
207
+ if (entry.path === MIMETYPE_PATH) {
208
+ throw new Error(`the writer owns the ${MIMETYPE_PATH} entry`);
209
+ }
307
210
  if (entry.path.includes("\\") || entry.path.startsWith("/") || entry.path.includes("..")) {
308
211
  throw new Error(`unsafe archive path: ${entry.path}`);
309
212
  }
@@ -311,10 +214,10 @@ async function writeGezkArchive(outputPath, entries) {
311
214
  else if (entry.absPath) zip.addFile(entry.absPath, entry.path, options);
312
215
  else throw new Error(`archive entry ${entry.path} has neither content nor absPath`);
313
216
  }
314
- await new Promise((resolve3, reject) => {
217
+ await new Promise((resolve5, reject) => {
315
218
  const out = createWriteStream(outputPath);
316
219
  out.on("error", reject);
317
- out.on("close", () => resolve3());
220
+ out.on("close", () => resolve5());
318
221
  zip.outputStream.on("error", reject);
319
222
  zip.outputStream.pipe(out);
320
223
  zip.end();
@@ -322,27 +225,7 @@ async function writeGezkArchive(outputPath, entries) {
322
225
  }
323
226
 
324
227
  // src/format/file-hash.ts
325
- import { createHash as createHash2 } from "crypto";
326
- import { createReadStream } from "fs";
327
- async function hashFileStreaming(path, maxBytes = Number.MAX_SAFE_INTEGER) {
328
- const hasher = createHash2("sha256");
329
- let sizeBytes = 0;
330
- await new Promise((resolve3, reject) => {
331
- const stream = createReadStream(path, { highWaterMark: 4 * 1024 * 1024 });
332
- stream.on("data", (chunk) => {
333
- const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
334
- sizeBytes += bytes.byteLength;
335
- if (sizeBytes > maxBytes) {
336
- stream.destroy(new Error(`file exceeds ${maxBytes} byte limit`));
337
- return;
338
- }
339
- hasher.update(bytes);
340
- });
341
- stream.on("error", reject);
342
- stream.on("end", resolve3);
343
- });
344
- return { sha256: hasher.digest("hex"), sizeBytes };
345
- }
228
+ import { hashFileStreaming } from "@bendyline/gezk/node";
346
229
 
347
230
  // src/format/node-sqlite.ts
348
231
  import { createRequire as createRequire2 } from "module";
@@ -371,25 +254,13 @@ function documentSmokeQueryMisses(db, smoke) {
371
254
  return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
372
255
  }
373
256
 
374
- // src/compiler/vec-load.ts
257
+ // src/toolchain.ts
375
258
  import { createRequire as createRequire3 } from "module";
376
259
  var nodeRequire3 = createRequire3(import.meta.url);
377
- function loadVecExtension(db) {
378
- try {
379
- db.enableLoadExtension(true);
380
- const sqliteVec = nodeRequire3("sqlite-vec");
381
- db.loadExtension(sqliteVec.getLoadablePath());
382
- } catch (err) {
383
- throw new Error(
384
- `sqlite-vec extension failed to load \u2014 the compiler cannot build vector tables: ${err instanceof Error ? err.message : String(err)}`
385
- );
386
- } finally {
387
- try {
388
- db.enableLoadExtension(false);
389
- } catch {
390
- }
391
- }
392
- }
260
+ var KNOWLEDGE_TOOLCHAIN = {
261
+ name: "@bendyline/gezel-knowledge",
262
+ version: nodeRequire3("../package.json").version
263
+ };
393
264
 
394
265
  // src/compiler/compile.ts
395
266
  async function compileKnowledgeCatalog(opts) {
@@ -398,12 +269,13 @@ async function compileKnowledgeCatalog(opts) {
398
269
  "a catalog must ship a table of contents: provide at least one topic (a flat corpus gets a single root topic)"
399
270
  );
400
271
  }
401
- const topicIds = new Set(opts.topics.map((t) => t.id));
272
+ const topicById = assertTopicForest(opts.topics);
273
+ const preparedAssets = prepareAssets(opts.assets ?? [], opts.extraFiles ?? {});
402
274
  const profile = opts.embeddingProfile;
403
275
  const chunkerOpts = {
404
- unit: "tokens",
405
- target: opts.chunkingProfile.targetTokens,
406
- overlap: opts.chunkingProfile.overlapTokens,
276
+ unit: opts.chunkingProfile.unit,
277
+ target: opts.chunkingProfile.target,
278
+ overlap: opts.chunkingProfile.overlap,
407
279
  countTokens: opts.countTokens
408
280
  };
409
281
  const prepared = [];
@@ -411,13 +283,11 @@ async function compileKnowledgeCatalog(opts) {
411
283
  for await (const doc of opts.documents) {
412
284
  if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
413
285
  seenIds.add(doc.id);
414
- const rootTopic = doc.topicPath[0];
415
- if (!rootTopic || !topicIds.has(rootTopic)) {
416
- throw new Error(`document ${doc.id} names unknown topic '${rootTopic ?? ""}'`);
417
- }
418
286
  prepared.push({
419
287
  doc,
420
288
  topicPathKey: doc.topicPath.join("/"),
289
+ leafTopicId: assertTopicPath(topicById, doc.id, doc.topicPath),
290
+ metaJson: encodeDocumentMeta(doc),
421
291
  chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
422
292
  });
423
293
  }
@@ -463,7 +333,6 @@ async function compileKnowledgeCatalog(opts) {
463
333
  router.exec("BEGIN");
464
334
  if (embedded) {
465
335
  router.exec(SHARD_DDL.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
466
- router.exec(vecChunksDdl(profile.dimensions));
467
336
  }
468
337
  const commonMeta = {
469
338
  format_version: String(GEZK_FORMAT_VERSION),
@@ -485,8 +354,7 @@ async function compileKnowledgeCatalog(opts) {
485
354
  });
486
355
  const topicDocCount = /* @__PURE__ */ new Map();
487
356
  for (const p of prepared) {
488
- const t = p.doc.topicPath[0];
489
- topicDocCount.set(t, (topicDocCount.get(t) ?? 0) + 1);
357
+ topicDocCount.set(p.leafTopicId, (topicDocCount.get(p.leafTopicId) ?? 0) + 1);
490
358
  }
491
359
  {
492
360
  const stmt = router.prepare(
@@ -505,9 +373,10 @@ async function compileKnowledgeCatalog(opts) {
505
373
  }
506
374
  {
507
375
  const insertDoc = router.prepare(
508
- `INSERT INTO documents (id, title, slug, summary, language, topic_id, shard_id, chunk_count,
509
- source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob)
510
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
376
+ `INSERT INTO documents (id, title, slug, summary, language, topic_id, ordinal, shard_id,
377
+ chunk_count, source_url, source_revision, source_updated_at, attribution_json, meta_json,
378
+ body_codec, body_blob)
379
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
511
380
  );
512
381
  const insertAlias = router.prepare(
513
382
  "INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
@@ -535,13 +404,15 @@ async function compileKnowledgeCatalog(opts) {
535
404
  p.doc.slug || documentSlug(p.doc.title),
536
405
  p.doc.summary ?? null,
537
406
  p.doc.language,
538
- p.doc.topicPath[0],
407
+ p.leafTopicId,
408
+ p.doc.ordinal === void 0 ? null : BigInt(p.doc.ordinal),
539
409
  BigInt(shardOf.get(p.doc.id)),
540
410
  BigInt(p.chunks.length),
541
411
  p.doc.sourceUrl ?? null,
542
412
  p.doc.sourceRevision ?? null,
543
413
  p.doc.sourceUpdatedAt ?? null,
544
414
  p.doc.attribution ? JSON.stringify(p.doc.attribution) : null,
415
+ p.metaJson,
545
416
  useBrotli ? "br" : "none",
546
417
  blob
547
418
  );
@@ -560,7 +431,6 @@ async function compileKnowledgeCatalog(opts) {
560
431
  if (!db) {
561
432
  db = createCatalogDb(join(opts.workDir, shardPath(id)));
562
433
  db.exec(SHARD_DDL);
563
- db.exec(vecChunksDdl(profile.dimensions));
564
434
  db.exec("BEGIN");
565
435
  shardDbs.set(id, db);
566
436
  }
@@ -571,7 +441,7 @@ async function compileKnowledgeCatalog(opts) {
571
441
  let stmts = vecStmts.get(db);
572
442
  if (!stmts) {
573
443
  stmts = {
574
- vec: db.prepare("INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, vec_bit(?))"),
444
+ vec: db.prepare("INSERT INTO chunk_vectors_bit (chunk_id, v) VALUES (?, ?)"),
575
445
  int8: db.prepare("INSERT INTO chunk_vectors_int8 (chunk_id, v) VALUES (?, ?)")
576
446
  };
577
447
  vecStmts.set(db, stmts);
@@ -618,7 +488,7 @@ async function compileKnowledgeCatalog(opts) {
618
488
  const db = shardDb(shardId);
619
489
  shardDocCounts.set(shardId, (shardDocCounts.get(shardId) ?? 0) + 1);
620
490
  const topics = shardTopicIds.get(shardId) ?? /* @__PURE__ */ new Set();
621
- topics.add(p.doc.topicPath[0]);
491
+ topics.add(p.leafTopicId);
622
492
  shardTopicIds.set(shardId, topics);
623
493
  const insertChunk = db.prepare(
624
494
  `INSERT INTO chunks (chunk_uid, document_id, ordinal, title, heading_path, heading_text,
@@ -650,7 +520,7 @@ async function compileKnowledgeCatalog(opts) {
650
520
  const header = buildContextHeader(
651
521
  p.doc.title,
652
522
  chunk.headingPath,
653
- opts.chunkingProfile.contextHeader.maxTokens,
523
+ opts.chunkingProfile.contextHeader.max,
654
524
  opts.countTokens
655
525
  );
656
526
  pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
@@ -659,6 +529,30 @@ async function compileKnowledgeCatalog(opts) {
659
529
  }
660
530
  }
661
531
  await flushEmbeds();
532
+ for (let id = 0; id < shardCount; id++) {
533
+ if (!embedded)
534
+ putMeta(shardDb(id), {
535
+ shard_id: String(id),
536
+ chunk_count: String(shardChunkCounts.get(id) ?? 0)
537
+ });
538
+ }
539
+ if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
540
+ const finalDir = join(opts.workDir, "final");
541
+ mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
542
+ const vacuumed = [];
543
+ const vacuumInto = (db, archivePath) => {
544
+ db.exec("COMMIT");
545
+ const dest = join(finalDir, archivePath);
546
+ db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
547
+ vacuumed.push({ archivePath, absPath: dest });
548
+ return dest;
549
+ };
550
+ const shardBytes = /* @__PURE__ */ new Map();
551
+ if (!embedded) {
552
+ for (let id = 0; id < shardCount; id++) {
553
+ shardBytes.set(id, statSync(vacuumInto(shardDb(id), shardPath(id))).size);
554
+ }
555
+ }
662
556
  {
663
557
  const insertShard = router.prepare(
664
558
  "INSERT INTO shards (id, path, chunk_count, document_count, topic_ids_json, centroid_count, bytes) VALUES (?, ?, ?, ?, ?, ?, ?)"
@@ -681,35 +575,14 @@ async function compileKnowledgeCatalog(opts) {
681
575
  BigInt(shardDocCounts.get(id) ?? 0),
682
576
  JSON.stringify([...shardTopicIds.get(id) ?? []].sort()),
683
577
  BigInt(centroids.length),
684
- BigInt(0)
685
- // authoritative sizes live in the manifest (post-VACUUM)
578
+ BigInt(shardBytes.get(id) ?? 0)
686
579
  );
687
580
  for (const c of centroids) {
688
581
  insertCentroid.run(BigInt(id), Buffer.from(c.centroid.buffer.slice(0)), BigInt(c.weight));
689
582
  }
690
583
  }
691
584
  }
692
- for (let id = 0; id < shardCount; id++) {
693
- if (!embedded)
694
- putMeta(shardDb(id), {
695
- shard_id: String(id),
696
- chunk_count: String(shardChunkCounts.get(id) ?? 0)
697
- });
698
- }
699
- if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
700
- const finalDir = join(opts.workDir, "final");
701
- mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
702
- const vacuumed = [];
703
- const vacuumInto = (db, archivePath) => {
704
- db.exec("COMMIT");
705
- const dest = join(finalDir, archivePath);
706
- db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
707
- vacuumed.push({ archivePath, absPath: dest });
708
- };
709
585
  vacuumInto(router, ROUTER_DB_PATH);
710
- if (!embedded) {
711
- for (let id = 0; id < shardCount; id++) vacuumInto(shardDb(id), shardPath(id));
712
- }
713
586
  router.close();
714
587
  for (const db of shardDbs.values()) db.close();
715
588
  let smokeQueries = opts.smokeQueries;
@@ -765,17 +638,25 @@ async function compileKnowledgeCatalog(opts) {
765
638
  });
766
639
  }
767
640
  }
768
- const extraFiles = opts.extraFiles ?? {};
641
+ const extraFiles = {
642
+ [README_PATH]: defaultReadme(opts),
643
+ [LICENSE_NOTICE_PATH]: defaultLicenseNotice(opts),
644
+ ...opts.extraFiles ?? {}
645
+ };
769
646
  for (const [path, content] of Object.entries(extraFiles)) {
770
647
  const bytes = Buffer.from(content, "utf8");
771
648
  files.push({
772
649
  path,
773
650
  sizeBytes: bytes.length,
774
- sha256: createHash3("sha256").update(bytes).digest("hex")
651
+ sha256: createHash("sha256").update(bytes).digest("hex")
775
652
  });
776
653
  }
654
+ for (const asset of preparedAssets) {
655
+ files.push({ path: asset.path, sizeBytes: asset.sizeBytes, sha256: asset.sha256 });
656
+ }
657
+ assertAssetReferencesDeclared(prepared, preparedAssets);
777
658
  const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
778
- kind: "gezel-knowledge-catalog",
659
+ kind: GEZK_MANIFEST_KIND,
779
660
  formatVersion: GEZK_FORMAT_VERSION,
780
661
  indexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION,
781
662
  id: opts.catalog.id,
@@ -786,20 +667,34 @@ async function compileKnowledgeCatalog(opts) {
786
667
  publisher: opts.catalog.publisher,
787
668
  createdAt: opts.catalog.createdAt,
788
669
  ...opts.catalog.sourceSnapshot ? { sourceSnapshot: opts.catalog.sourceSnapshot } : {},
789
- license: opts.catalog.license,
670
+ license: {
671
+ ...opts.catalog.license,
672
+ noticePath: opts.catalog.license.noticePath ?? LICENSE_NOTICE_PATH
673
+ },
790
674
  embedding: profile,
791
675
  chunking: opts.chunkingProfile,
792
- topics: opts.topics.map((t) => ({ id: t.id, name: t.name })),
676
+ topics: opts.topics.map((t) => ({
677
+ id: t.id,
678
+ name: t.name,
679
+ ...t.parentId ? { parentId: t.parentId } : {},
680
+ sortKey: t.sortKey ?? t.name.toLowerCase(),
681
+ ...t.description ? { description: t.description } : {}
682
+ })),
793
683
  router: {
794
684
  shardTargetChunks: shardTarget,
795
685
  shards: routerShardStats.sort((a, b) => a.id - b.id),
796
686
  totalCentroids: routerShardStats.reduce((sum, s) => sum + s.centroids, 0)
797
687
  },
798
- counts: { documents: prepared.length, chunks: totalChunks, shards: shardCount },
688
+ counts: {
689
+ documents: prepared.length,
690
+ chunks: totalChunks,
691
+ shards: shardCount,
692
+ assets: preparedAssets.length
693
+ },
799
694
  files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
800
- compatibility: { maximumIndexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION },
695
+ requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
801
696
  ...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
802
- ...opts.toolchain ? { toolchain: opts.toolchain } : {}
697
+ toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
803
698
  });
804
699
  const manifest = opts.finalizeManifest ? KnowledgeCatalogManifestSchema.parse(opts.finalizeManifest(unsignedManifest)) : unsignedManifest;
805
700
  const archiveEntries = [
@@ -812,7 +707,8 @@ async function compileKnowledgeCatalog(opts) {
812
707
  ...Object.entries(extraFiles).map(([path, content]) => ({
813
708
  path,
814
709
  content: Buffer.from(content, "utf8")
815
- }))
710
+ })),
711
+ ...preparedAssets.map((asset) => ({ path: asset.path, content: asset.bytes }))
816
712
  ];
817
713
  await writeGezkArchive(opts.outputPath, archiveEntries);
818
714
  const archiveBytes = (await stat(opts.outputPath)).size;
@@ -828,15 +724,166 @@ async function compileKnowledgeCatalog(opts) {
828
724
  closeAll();
829
725
  }
830
726
  }
727
+ function assertTopicForest(topics) {
728
+ const byId = /* @__PURE__ */ new Map();
729
+ for (const topic of topics) {
730
+ if (byId.has(topic.id)) throw new Error(`duplicate topic id: ${topic.id}`);
731
+ byId.set(topic.id, topic);
732
+ }
733
+ for (const topic of topics) {
734
+ const seen = /* @__PURE__ */ new Set();
735
+ let depth = 0;
736
+ let current = topic;
737
+ while (current) {
738
+ if (seen.has(current.id)) throw new Error(`topic '${topic.id}' sits in a parent cycle`);
739
+ seen.add(current.id);
740
+ depth += 1;
741
+ if (depth > MAX_KNOWLEDGE_TOPIC_DEPTH) {
742
+ throw new Error(`topic '${topic.id}' is deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH}`);
743
+ }
744
+ if (current.parentId === void 0) break;
745
+ const parent = byId.get(current.parentId);
746
+ if (!parent) {
747
+ throw new Error(`topic '${current.id}' names an undeclared parent '${current.parentId}'`);
748
+ }
749
+ current = parent;
750
+ }
751
+ }
752
+ return byId;
753
+ }
754
+ function assertTopicPath(topicById, docId, path) {
755
+ for (let i = 0; i < path.length; i++) {
756
+ const id = path[i];
757
+ const topic = topicById.get(id);
758
+ if (!topic) {
759
+ throw new Error(
760
+ `document ${docId} names unknown topic '${id}' (segment ${i} of ${path.join("/")})`
761
+ );
762
+ }
763
+ const expectedParent = i === 0 ? void 0 : path[i - 1];
764
+ if (topic.parentId !== expectedParent) {
765
+ throw new Error(
766
+ `document ${docId}: topic path ${path.join("/")} does not follow the declared tree at '${id}' (its parent is '${topic.parentId ?? "(root)"}')`
767
+ );
768
+ }
769
+ }
770
+ return path[path.length - 1];
771
+ }
772
+ function encodeDocumentMeta(doc) {
773
+ if (!doc.meta || Object.keys(doc.meta).length === 0) return null;
774
+ let json;
775
+ try {
776
+ json = canonicalizeJson(doc.meta);
777
+ } catch (error) {
778
+ throw new Error(
779
+ `document ${doc.id} has metadata that cannot be serialized: ${error instanceof Error ? error.message : String(error)}`
780
+ );
781
+ }
782
+ const bytes = Buffer.byteLength(json, "utf8");
783
+ if (bytes > MAX_KNOWLEDGE_DOCUMENT_META_BYTES) {
784
+ throw new Error(
785
+ `document ${doc.id} has ${bytes} bytes of metadata; the limit is ${MAX_KNOWLEDGE_DOCUMENT_META_BYTES}`
786
+ );
787
+ }
788
+ return json;
789
+ }
790
+ function prepareAssets(assets, extraFiles) {
791
+ for (const path of Object.keys(extraFiles)) {
792
+ if (path.startsWith("assets/")) {
793
+ throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
794
+ }
795
+ }
796
+ if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
797
+ throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
798
+ }
799
+ const seen = /* @__PURE__ */ new Set();
800
+ const prepared = [];
801
+ let total = 0;
802
+ for (const asset of assets) {
803
+ const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
804
+ if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
805
+ const key = asset.path.toLowerCase();
806
+ if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
807
+ seen.add(key);
808
+ if (asset.content === void 0 === (asset.absPath === void 0)) {
809
+ throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
810
+ }
811
+ const bytes = asset.content ?? readFileSync(asset.absPath);
812
+ if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
813
+ throw new Error(
814
+ `asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
815
+ );
816
+ }
817
+ total += bytes.byteLength;
818
+ if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
819
+ throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
820
+ }
821
+ const ext = assetExtension(asset.path);
822
+ const expected = ext ? assetKindForExtension(ext) : null;
823
+ const actual = sniffAssetType(bytes);
824
+ if (expected === null || actual !== expected) {
825
+ throw new Error(
826
+ `asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
827
+ );
828
+ }
829
+ if (ext === "svg") {
830
+ const problem = svgInertnessProblem(bytes);
831
+ if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
832
+ }
833
+ prepared.push({
834
+ path: asset.path,
835
+ bytes,
836
+ sizeBytes: bytes.byteLength,
837
+ sha256: createHash("sha256").update(bytes).digest("hex")
838
+ });
839
+ }
840
+ return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
841
+ }
842
+ function assertAssetReferencesDeclared(prepared, assets) {
843
+ const declared = new Set(assets.map((a) => a.path));
844
+ for (const p of prepared) {
845
+ for (const match of p.doc.markdown.matchAll(/\]\(\s*<?(assets\/[^)\s>]+)/g)) {
846
+ const target = match[1];
847
+ if (!declared.has(target)) {
848
+ throw new Error(
849
+ `document ${p.doc.id} references '${target}', which is not among the catalog's assets`
850
+ );
851
+ }
852
+ }
853
+ }
854
+ }
831
855
  function createCatalogDb(absPath) {
832
- const db = new DatabaseSync(absPath, { allowExtension: true });
833
- loadVecExtension(db);
856
+ const db = new DatabaseSync(absPath);
834
857
  db.exec("PRAGMA page_size=8192");
835
858
  db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
836
859
  db.exec(`PRAGMA user_version=${GEZK_INDEX_SCHEMA_VERSION}`);
837
860
  db.exec("PRAGMA journal_mode=DELETE");
838
861
  return db;
839
862
  }
863
+ function defaultReadme(opts) {
864
+ const c = opts.catalog;
865
+ return [
866
+ `# ${c.name}`,
867
+ "",
868
+ ...c.description ? [c.description, ""] : [],
869
+ `A gezk knowledge catalog (${c.id}@${c.version}) published by ${c.publisher.name}.`,
870
+ `Language: ${c.language}. License: ${c.license.name} (see ${c.license.noticePath ?? LICENSE_NOTICE_PATH}).`,
871
+ ...c.sourceSnapshot ? [`Source snapshot: ${c.sourceSnapshot.name} (${c.sourceSnapshot.date}).`] : [],
872
+ ""
873
+ ].join("\n");
874
+ }
875
+ function defaultLicenseNotice(opts) {
876
+ const c = opts.catalog;
877
+ const publisher = c.publisher.url ? `${c.publisher.name} <${c.publisher.url}>` : c.publisher.name;
878
+ const license = c.license.spdx ? `${c.license.name} (SPDX: ${c.license.spdx})` : c.license.name;
879
+ return [
880
+ `${c.name} (${c.id}@${c.version})`,
881
+ `Publisher: ${publisher}`,
882
+ `License: ${license}`,
883
+ ...c.license.attributionRequired ? ["Attribution is required when redistributing this content."] : [],
884
+ ""
885
+ ].join("\n");
886
+ }
840
887
  function normalizeMarkdown(md) {
841
888
  return md.replace(/\r\n/g, "\n").normalize("NFC");
842
889
  }
@@ -854,7 +901,7 @@ ${path.join(" > ")}
854
901
  return header;
855
902
  }
856
903
  function seedFor(catalogId, shardId) {
857
- const digest = createHash3("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
904
+ const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
858
905
  return digest.readBigUInt64LE(24);
859
906
  }
860
907
  function makePrng(seed) {
@@ -939,14 +986,14 @@ function kMeans(samples, k, seed) {
939
986
  }
940
987
 
941
988
  // src/archive/read.ts
942
- import { createHash as createHash4 } from "crypto";
989
+ import { createHash as createHash2 } from "crypto";
943
990
  import { createWriteStream as createWriteStream2 } from "fs";
944
991
  import { mkdir as mkdir2 } from "fs/promises";
945
992
  import { createRequire as createRequire4 } from "module";
946
993
  import { dirname as dirname2, isAbsolute, normalize, relative, resolve } from "path";
947
994
  import { Transform } from "stream";
948
995
  import { pipeline } from "stream/promises";
949
- import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/gezel";
996
+ import { GEZK_MANIFEST_KIND as GEZK_MANIFEST_KIND2, KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/gezk";
950
997
  var nodeRequire4 = createRequire4(import.meta.url);
951
998
  var GEZK_ARCHIVE_LIMITS = {
952
999
  maxEntries: 16384,
@@ -1036,7 +1083,7 @@ async function inspectEntries(archivePath) {
1036
1083
  function reconcileEntries(manifest, entries) {
1037
1084
  const declared = new Map(manifest.files.map((file) => [file.path, file]));
1038
1085
  for (const path of entries.keys()) {
1039
- if (path === MANIFEST_PATH) continue;
1086
+ if (path === MANIFEST_PATH || path === MIMETYPE_PATH) continue;
1040
1087
  if (!declared.has(path)) {
1041
1088
  throw new GezkArchiveError(`archive contains undeclared file: ${path}`, "undeclared-file");
1042
1089
  }
@@ -1066,14 +1113,88 @@ async function inspectGezkArchive(archivePath) {
1066
1113
  totalUncompressedBytes: inspected.totalUncompressedBytes
1067
1114
  };
1068
1115
  }
1116
+ function assertMimetypeEntry(entry, position) {
1117
+ if (position !== 0) {
1118
+ throw new GezkArchiveError("the mimetype entry must be the first archive entry", "mimetype");
1119
+ }
1120
+ if (entry.compressionMethod !== 0 || entry.uncompressedSize !== GEZK_MIME_TYPE.length) {
1121
+ throw new GezkArchiveError("the mimetype entry must be stored, not compressed", "mimetype");
1122
+ }
1123
+ }
1124
+ function assertSupportedFormat(raw) {
1125
+ const kind = raw?.kind;
1126
+ const version = raw?.formatVersion;
1127
+ if (kind !== GEZK_MANIFEST_KIND2 || !isSupportedFormatVersion(version)) {
1128
+ throw new GezkArchiveError(
1129
+ `unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${GEZK_SUPPORTED_FORMAT_VERSIONS.join(", ")} \u2014 catalogs built for another version need a matching reader`,
1130
+ "format-version"
1131
+ );
1132
+ }
1133
+ }
1134
+ function readEntryText(zip, entry) {
1135
+ return new Promise((resolveText, rejectText) => {
1136
+ zip.openReadStream(entry, (err, stream) => {
1137
+ if (err || !stream) {
1138
+ rejectText(new GezkArchiveError(`${entry.fileName} read failed: ${err?.message}`, "io"));
1139
+ return;
1140
+ }
1141
+ const parts = [];
1142
+ stream.on("data", (d) => parts.push(d));
1143
+ stream.on("end", () => resolveText(Buffer.concat(parts).toString("utf8")));
1144
+ stream.on(
1145
+ "error",
1146
+ (streamErr) => rejectText(
1147
+ new GezkArchiveError(`${entry.fileName} read failed: ${streamErr.message}`, "io")
1148
+ )
1149
+ );
1150
+ });
1151
+ });
1152
+ }
1069
1153
  async function readGezkManifest(archivePath) {
1070
1154
  const zip = await openZip(archivePath);
1071
1155
  return new Promise((resolvePromise, reject) => {
1156
+ let position = 0;
1157
+ let mimetypeSeen = false;
1158
+ const fail = (err) => {
1159
+ zip.close();
1160
+ reject(
1161
+ err instanceof GezkArchiveError ? err : new GezkArchiveError(err instanceof Error ? err.message : String(err), "io")
1162
+ );
1163
+ };
1072
1164
  zip.on("entry", (entry) => {
1165
+ const index = position++;
1166
+ if (entry.fileName === MIMETYPE_PATH) {
1167
+ try {
1168
+ assertMimetypeEntry(entry, index);
1169
+ } catch (err) {
1170
+ fail(err);
1171
+ return;
1172
+ }
1173
+ readEntryText(zip, entry).then((text) => {
1174
+ if (text !== GEZK_MIME_TYPE) {
1175
+ fail(
1176
+ new GezkArchiveError(
1177
+ `not a gezk archive (mimetype ${JSON.stringify(text)})`,
1178
+ "mimetype"
1179
+ )
1180
+ );
1181
+ return;
1182
+ }
1183
+ mimetypeSeen = true;
1184
+ zip.readEntry();
1185
+ }, fail);
1186
+ return;
1187
+ }
1073
1188
  if (entry.fileName !== MANIFEST_PATH) {
1074
1189
  zip.readEntry();
1075
1190
  return;
1076
1191
  }
1192
+ if (!mimetypeSeen) {
1193
+ fail(
1194
+ new GezkArchiveError("archive does not start with the gezk mimetype entry", "mimetype")
1195
+ );
1196
+ return;
1197
+ }
1077
1198
  if (entry.uncompressedSize > GEZK_ARCHIVE_LIMITS.maxManifestBytes) {
1078
1199
  zip.close();
1079
1200
  reject(new GezkArchiveError("manifest exceeds size limit", "limits"));
@@ -1090,12 +1211,14 @@ async function readGezkManifest(archivePath) {
1090
1211
  stream.on("end", () => {
1091
1212
  zip.close();
1092
1213
  try {
1093
- resolvePromise(
1094
- KnowledgeCatalogManifestSchema2.parse(
1095
- JSON.parse(Buffer.concat(parts).toString("utf8"))
1096
- )
1097
- );
1214
+ const raw = JSON.parse(Buffer.concat(parts).toString("utf8"));
1215
+ assertSupportedFormat(raw);
1216
+ resolvePromise(KnowledgeCatalogManifestSchema2.parse(raw));
1098
1217
  } catch (parseErr) {
1218
+ if (parseErr instanceof GezkArchiveError) {
1219
+ reject(parseErr);
1220
+ return;
1221
+ }
1099
1222
  reject(
1100
1223
  new GezkArchiveError(
1101
1224
  `invalid manifest: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,
@@ -1153,7 +1276,7 @@ async function extractGezkVerified(archivePath, destDir) {
1153
1276
  pipeline(stream, createWriteStream2(target)).then(res, rej);
1154
1277
  return;
1155
1278
  }
1156
- const hasher = createHash4("sha256");
1279
+ const hasher = createHash2("sha256");
1157
1280
  let sizeBytes = 0;
1158
1281
  const verify = new Transform({
1159
1282
  transform(chunk, _encoding, callback) {
@@ -1214,9 +1337,7 @@ async function extractGezkVerified(archivePath, destDir) {
1214
1337
  }
1215
1338
 
1216
1339
  // src/reader/open.ts
1217
- import { createRequire as createRequire5 } from "module";
1218
1340
  import { pathToFileURL } from "url";
1219
- var nodeRequire5 = createRequire5(import.meta.url);
1220
1341
  var CatalogOpenError = class extends Error {
1221
1342
  constructor(message, reason) {
1222
1343
  super(message);
@@ -1229,7 +1350,7 @@ function openCatalogDatabase(absPath) {
1229
1350
  const uri = `${pathToFileURL(absPath).href}?immutable=1`;
1230
1351
  let db;
1231
1352
  try {
1232
- db = new DatabaseSync(uri, { readOnly: true, allowExtension: true });
1353
+ db = new DatabaseSync(uri, { readOnly: true });
1233
1354
  } catch (err) {
1234
1355
  throw new CatalogOpenError(
1235
1356
  `cannot open catalog database ${absPath}: ${err instanceof Error ? err.message : String(err)}`,
@@ -1237,20 +1358,6 @@ function openCatalogDatabase(absPath) {
1237
1358
  );
1238
1359
  }
1239
1360
  try {
1240
- let vecLoaded = false;
1241
- try {
1242
- db.enableLoadExtension(true);
1243
- const sqliteVec = nodeRequire5("sqlite-vec");
1244
- db.loadExtension(sqliteVec.getLoadablePath());
1245
- vecLoaded = true;
1246
- } catch {
1247
- vecLoaded = false;
1248
- } finally {
1249
- try {
1250
- db.enableLoadExtension(false);
1251
- } catch {
1252
- }
1253
- }
1254
1361
  const appId = readPragmaNumber(db, "application_id");
1255
1362
  if (appId !== GEZK_APPLICATION_ID) {
1256
1363
  throw new CatalogOpenError(
@@ -1259,19 +1366,17 @@ function openCatalogDatabase(absPath) {
1259
1366
  );
1260
1367
  }
1261
1368
  const schemaVersion = readPragmaNumber(db, "user_version");
1262
- if (schemaVersion !== GEZK_INDEX_SCHEMA_VERSION) {
1369
+ if (!isSupportedIndexSchemaVersion(schemaVersion)) {
1263
1370
  throw new CatalogOpenError(
1264
- `unsupported index schema version ${schemaVersion} (reader supports ${GEZK_INDEX_SCHEMA_VERSION})`,
1371
+ `unsupported index schema version ${schemaVersion} (this reader supports ${GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.join(", ")}; catalogs built for another generation need a matching reader)`,
1265
1372
  "schema-version"
1266
1373
  );
1267
1374
  }
1268
- if (!vecLoaded) {
1269
- throw new CatalogOpenError("sqlite-vec extension failed to load", "vec-unavailable");
1270
- }
1271
1375
  db.exec("PRAGMA mmap_size=536870912");
1272
1376
  db.exec("PRAGMA cache_size=-8192");
1273
1377
  return {
1274
1378
  db,
1379
+ schemaVersion,
1275
1380
  close: () => {
1276
1381
  try {
1277
1382
  db.close();
@@ -1294,12 +1399,94 @@ function readPragmaNumber(db, pragma) {
1294
1399
  }
1295
1400
 
1296
1401
  // src/reader/catalog-handle.ts
1402
+ import { readFileSync as readFileSync2 } from "fs";
1297
1403
  import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1298
1404
  import { brotliDecompressSync } from "zlib";
1405
+ import {
1406
+ MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
1407
+ assetContentType,
1408
+ isKnowledgeAssetPath
1409
+ } from "@bendyline/gezk";
1410
+
1411
+ // src/reader/bit-scan.ts
1412
+ var POPCOUNT8 = new Uint8Array(256);
1413
+ for (let i = 1; i < 256; i++) POPCOUNT8[i] = (i & 1) + POPCOUNT8[i >> 1];
1414
+ function popcount32(x) {
1415
+ let v = x - (x >>> 1 & 1431655765);
1416
+ v = (v & 858993459) + (v >>> 2 & 858993459);
1417
+ return Math.imul(v + (v >>> 4) & 252645135, 16843009) >>> 24;
1418
+ }
1419
+ function hammingAll(index, query, out) {
1420
+ const { bits, bytesPerRow, rows } = index;
1421
+ if (bytesPerRow % 4 === 0 && bits.byteOffset % 4 === 0) {
1422
+ const perRow = bytesPerRow / 4;
1423
+ const words = new Uint32Array(bits.buffer, bits.byteOffset, rows * perRow);
1424
+ const aligned = new Uint8Array(bytesPerRow);
1425
+ aligned.set(query);
1426
+ const qwords = new Uint32Array(aligned.buffer, 0, perRow);
1427
+ let offset2 = 0;
1428
+ for (let r = 0; r < rows; r++) {
1429
+ let d = 0;
1430
+ for (let w = 0; w < perRow; w++) {
1431
+ d += popcount32((words[offset2 + w] ^ qwords[w]) >>> 0);
1432
+ }
1433
+ out[r] = d;
1434
+ offset2 += perRow;
1435
+ }
1436
+ return;
1437
+ }
1438
+ let offset = 0;
1439
+ for (let r = 0; r < rows; r++) {
1440
+ let d = 0;
1441
+ for (let b = 0; b < bytesPerRow; b++) {
1442
+ d += POPCOUNT8[bits[offset + b] ^ query[b]];
1443
+ }
1444
+ out[r] = d;
1445
+ offset += bytesPerRow;
1446
+ }
1447
+ }
1448
+ function hammingTopK(index, query, k) {
1449
+ if (query.length !== index.bytesPerRow) {
1450
+ throw new Error(`query has ${query.length} bytes, shard rows have ${index.bytesPerRow}`);
1451
+ }
1452
+ const limit = Math.min(k, index.rows);
1453
+ if (limit <= 0) return [];
1454
+ const distances = new Uint16Array(index.rows);
1455
+ hammingAll(index, query, distances);
1456
+ const maxDistance = index.bytesPerRow * 8;
1457
+ const histogram = new Uint32Array(maxDistance + 1);
1458
+ for (let r = 0; r < index.rows; r++) {
1459
+ const d = distances[r];
1460
+ histogram[d] = histogram[d] + 1;
1461
+ }
1462
+ let threshold = 0;
1463
+ let seen = 0;
1464
+ for (; threshold <= maxDistance; threshold++) {
1465
+ seen += histogram[threshold];
1466
+ if (seen >= limit) break;
1467
+ }
1468
+ let atThreshold = limit - (seen - histogram[threshold]);
1469
+ const hits = [];
1470
+ for (let r = 0; r < index.rows && hits.length < limit; r++) {
1471
+ const d = distances[r];
1472
+ if (d < threshold) hits.push({ chunkId: r + 1, distance: d });
1473
+ else if (d === threshold && atThreshold > 0) {
1474
+ hits.push({ chunkId: r + 1, distance: d });
1475
+ atThreshold--;
1476
+ }
1477
+ }
1478
+ hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
1479
+ return hits;
1480
+ }
1481
+
1482
+ // src/reader/catalog-handle.ts
1483
+ var TOPIC_WALK_MAX_DEPTH = 32;
1484
+ var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
1299
1485
  var CatalogHandle = class _CatalogHandle {
1300
1486
  constructor(rootDir, router) {
1301
1487
  this.rootDir = rootDir;
1302
1488
  this.router = router;
1489
+ this.schemaVersion = router.schemaVersion;
1303
1490
  this.meta = {};
1304
1491
  for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
1305
1492
  this.meta[row.key] = row.value;
@@ -1310,8 +1497,12 @@ var CatalogHandle = class _CatalogHandle {
1310
1497
  rootDir;
1311
1498
  router;
1312
1499
  connections = /* @__PURE__ */ new Map();
1500
+ bitIndexes = /* @__PURE__ */ new Map();
1501
+ assetsByPath = null;
1313
1502
  meta;
1314
1503
  shards;
1504
+ /** The router's `PRAGMA user_version` generation; decides which columns exist. */
1505
+ schemaVersion;
1315
1506
  /** Open a catalog's extracted version directory (the dir holding manifest.json). */
1316
1507
  static open(rootDir) {
1317
1508
  const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
@@ -1320,6 +1511,7 @@ var CatalogHandle = class _CatalogHandle {
1320
1511
  close() {
1321
1512
  for (const conn of this.connections.values()) conn.close();
1322
1513
  this.connections.clear();
1514
+ this.bitIndexes.clear();
1323
1515
  }
1324
1516
  /** Integrity check for the router connection, used by deep validation. */
1325
1517
  routerQuickCheck() {
@@ -1350,6 +1542,57 @@ var CatalogHandle = class _CatalogHandle {
1350
1542
  }
1351
1543
  return conn.db;
1352
1544
  }
1545
+ /**
1546
+ * A shard's sign-bit rows as one contiguous array, loaded on first use and
1547
+ * kept while the resident budget allows. Chunk ids must be dense from 1 —
1548
+ * the row index IS the id — so a gap is a corrupt shard, not a sparse one.
1549
+ */
1550
+ shardBits(shard) {
1551
+ const cached = this.bitIndexes.get(shard.path);
1552
+ if (cached) return cached;
1553
+ const db = this.shardDb(shard);
1554
+ const span = db.prepare(
1555
+ "SELECT COUNT(*) AS n, MIN(chunk_id) AS lo, MAX(chunk_id) AS hi FROM chunk_vectors_bit"
1556
+ ).get();
1557
+ const rows = Number(span.n);
1558
+ if (rows > 0 && (Number(span.lo) !== 1 || Number(span.hi) !== rows)) {
1559
+ throw new CatalogOpenError(`chunk ids are not dense in ${shard.path}`, "corrupt");
1560
+ }
1561
+ const bytesPerRow = Math.ceil(this.dimensions() / 8);
1562
+ const bits = new Uint8Array(rows * bytesPerRow);
1563
+ const rowsIter = db.prepare("SELECT chunk_id, v FROM chunk_vectors_bit ORDER BY chunk_id").iterate();
1564
+ for (const row of rowsIter) {
1565
+ if (row.v.byteLength !== bytesPerRow) {
1566
+ throw new CatalogOpenError(
1567
+ `bit vector width ${row.v.byteLength} != ${bytesPerRow} in ${shard.path}`,
1568
+ "corrupt"
1569
+ );
1570
+ }
1571
+ bits.set(row.v, (Number(row.chunk_id) - 1) * bytesPerRow);
1572
+ }
1573
+ const index = { bits, bytesPerRow, rows };
1574
+ let resident = bits.byteLength;
1575
+ for (const other of this.bitIndexes.values()) resident += other.bits.byteLength;
1576
+ for (const [key, other] of this.bitIndexes) {
1577
+ if (resident <= BIT_INDEX_BUDGET_BYTES) break;
1578
+ this.bitIndexes.delete(key);
1579
+ resident -= other.bits.byteLength;
1580
+ }
1581
+ this.bitIndexes.set(shard.path, index);
1582
+ return index;
1583
+ }
1584
+ /** The embedding dimension from the router's profile echo. */
1585
+ dimensions() {
1586
+ const raw = this.meta.embedding_profile_json;
1587
+ if (raw) {
1588
+ try {
1589
+ const dims = JSON.parse(raw).dimensions;
1590
+ if (typeof dims === "number" && dims > 0) return dims;
1591
+ } catch {
1592
+ }
1593
+ }
1594
+ throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
1595
+ }
1353
1596
  /** Resolve a manifest/router path without allowing it to escape the catalog. */
1354
1597
  resolveCatalogPath(path) {
1355
1598
  const root = resolve2(this.rootDir);
@@ -1361,40 +1604,124 @@ var CatalogHandle = class _CatalogHandle {
1361
1604
  return target;
1362
1605
  }
1363
1606
  // ── browsing (the shipped table of contents) ──────────────────────────────
1607
+ /** The format version the router was written with (`meta.format_version`). */
1608
+ get formatVersion() {
1609
+ return this.meta.format_version ?? "";
1610
+ }
1611
+ documentColumns() {
1612
+ const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
1613
+ return this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
1614
+ }
1615
+ documentOrder() {
1616
+ return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
1617
+ }
1618
+ /**
1619
+ * Every topic with its direct count and a rollup over its subtree. On a
1620
+ * 0.5 catalog every document sits at a root, so both counts agree there.
1621
+ */
1364
1622
  topics() {
1365
1623
  return this.router.db.prepare(
1366
- "SELECT id, parent_id, name, description, sort_key, document_count FROM topics ORDER BY sort_key"
1624
+ `WITH RECURSIVE sub(root, id, depth) AS (
1625
+ SELECT id, id, 0 FROM topics
1626
+ UNION ALL
1627
+ SELECT sub.root, t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
1628
+ WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
1629
+ ),
1630
+ rollup AS (
1631
+ SELECT sub.root AS id, SUM(t.document_count) AS total
1632
+ FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
1633
+ )
1634
+ SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
1635
+ rollup.total AS total_document_count
1636
+ FROM topics t JOIN rollup ON rollup.id = t.id
1637
+ ORDER BY t.sort_key, t.id`
1367
1638
  ).all().map((r) => ({
1368
1639
  id: r.id,
1369
1640
  parentId: r.parent_id,
1370
1641
  name: r.name,
1371
1642
  description: r.description,
1372
1643
  sortKey: r.sort_key,
1373
- documentCount: Number(r.document_count)
1644
+ documentCount: Number(r.document_count),
1645
+ totalDocumentCount: Number(r.total_document_count)
1374
1646
  }));
1375
1647
  }
1648
+ /**
1649
+ * A page of documents, by default including those filed under the topic's
1650
+ * descendants (`descendants: false` lists only the topic's own).
1651
+ */
1376
1652
  documentsPage(opts = {}) {
1377
1653
  const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
1378
1654
  const offset = Math.max(0, opts.offset ?? 0);
1379
- const where = opts.topicId ? "WHERE topic_id = ?" : "";
1380
- const params = opts.topicId ? [opts.topicId] : [];
1655
+ const descendants = opts.descendants ?? true;
1656
+ let scope = "";
1657
+ let where = "";
1658
+ const params = [];
1659
+ if (opts.topicId && descendants) {
1660
+ scope = `WITH RECURSIVE sub(id, depth) AS (
1661
+ SELECT ?, 0
1662
+ UNION ALL
1663
+ SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
1664
+ WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
1665
+ )`;
1666
+ where = "WHERE topic_id IN (SELECT id FROM sub)";
1667
+ params.push(opts.topicId);
1668
+ } else if (opts.topicId) {
1669
+ where = "WHERE topic_id = ?";
1670
+ params.push(opts.topicId);
1671
+ }
1381
1672
  const total = Number(
1382
- this.router.db.prepare(`SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
1673
+ this.router.db.prepare(`${scope} SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
1383
1674
  );
1384
1675
  const documents = this.router.db.prepare(
1385
- `SELECT id, title, slug, summary, language, topic_id, source_url, source_revision,
1386
- source_updated_at, attribution_json
1387
- FROM documents ${where} ORDER BY slug, id LIMIT ? OFFSET ?`
1676
+ `${scope} SELECT ${this.documentColumns()}
1677
+ FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
1388
1678
  ).all(...params, limit, offset).map(rowToDocumentMeta);
1389
1679
  return { documents, total };
1390
1680
  }
1391
1681
  getDocument(id) {
1392
1682
  const row = this.router.db.prepare(
1393
- `SELECT id, title, slug, summary, language, topic_id, source_url, source_revision,
1394
- source_updated_at, attribution_json, body_codec, body_blob
1683
+ `SELECT ${this.documentColumns()}, body_codec, body_blob
1395
1684
  FROM documents WHERE id = ?`
1396
1685
  ).get(id);
1397
1686
  if (!row) return null;
1687
+ return { ...rowToDocumentMeta(row), markdown: this.decodeBody(id, row) };
1688
+ }
1689
+ /** Documents whose `topic_id` names no declared topic (a validator check). */
1690
+ documentsWithUndeclaredTopic() {
1691
+ return Number(
1692
+ this.router.db.prepare(
1693
+ "SELECT COUNT(*) AS n FROM documents d LEFT JOIN topics t ON t.id = d.topic_id WHERE t.id IS NULL"
1694
+ ).get().n
1695
+ );
1696
+ }
1697
+ /**
1698
+ * Validator support: every `meta_json` must be a JSON object within the
1699
+ * byte limit. Returns the offending ids (capped) rather than throwing.
1700
+ */
1701
+ checkDocumentMeta(maxBytes, cap = 5) {
1702
+ const invalid = [];
1703
+ const oversize = [];
1704
+ if (this.schemaVersion < 3) return { invalid, oversize };
1705
+ const rows = this.router.db.prepare("SELECT id, meta_json FROM documents WHERE meta_json IS NOT NULL").iterate();
1706
+ for (const row of rows) {
1707
+ if (invalid.length >= cap && oversize.length >= cap) break;
1708
+ if (Buffer.byteLength(row.meta_json, "utf8") > maxBytes) {
1709
+ if (oversize.length < cap) oversize.push(row.id);
1710
+ continue;
1711
+ }
1712
+ if (parseJsonObject(row.meta_json) === null && invalid.length < cap) invalid.push(row.id);
1713
+ }
1714
+ return { invalid, oversize };
1715
+ }
1716
+ /** Validator support: decoded bodies, streamed one at a time. */
1717
+ *documentBodies() {
1718
+ const rows = this.router.db.prepare("SELECT id, body_codec, body_blob FROM documents ORDER BY id").iterate();
1719
+ for (const row of rows) {
1720
+ const id = row.id;
1721
+ yield { id, markdown: this.decodeBody(id, row) };
1722
+ }
1723
+ }
1724
+ decodeBody(id, row) {
1398
1725
  const blob = row.body_blob;
1399
1726
  if (blob.byteLength > MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024) {
1400
1727
  throw new CatalogOpenError(`document body exceeds the stored-size limit: ${id}`, "corrupt");
@@ -1417,21 +1744,82 @@ var CatalogHandle = class _CatalogHandle {
1417
1744
  } else {
1418
1745
  throw new CatalogOpenError(`unknown document body codec for ${id}`, "corrupt");
1419
1746
  }
1420
- const markdown = body.toString("utf8");
1421
- return { ...rowToDocumentMeta(row), markdown };
1747
+ return body.toString("utf8");
1422
1748
  }
1423
- // ── search ────────────────────────────────────────────────────────────────
1424
- /** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
1425
- searchDocumentsFts(query, limit = 10) {
1426
- const match = ftsQuery(query);
1427
- if (!match) return [];
1749
+ // ── assets ────────────────────────────────────────────────────────────────
1750
+ /**
1751
+ * The manifest's `files` entries under `assets/`, read lazily: the
1752
+ * declaration in the manifest is what authorizes serving a file, since
1753
+ * extraction reconciled and hashed every declared entry.
1754
+ */
1755
+ assetIndex() {
1756
+ if (this.assetsByPath) return this.assetsByPath;
1757
+ const index = /* @__PURE__ */ new Map();
1758
+ let raw;
1428
1759
  try {
1429
- return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
1430
- documentId,
1431
- rank: i
1432
- }));
1433
- } catch {
1434
- return [];
1760
+ raw = JSON.parse(readFileSync2(join2(this.rootDir, MANIFEST_PATH), "utf8"));
1761
+ } catch (error) {
1762
+ if (error.code === "ENOENT") {
1763
+ this.assetsByPath = index;
1764
+ return index;
1765
+ }
1766
+ throw new CatalogOpenError(
1767
+ `cannot read ${MANIFEST_PATH}: ${error instanceof Error ? error.message : String(error)}`,
1768
+ "corrupt"
1769
+ );
1770
+ }
1771
+ const files = raw?.files;
1772
+ if (Array.isArray(files)) {
1773
+ for (const file of files) {
1774
+ const path = file?.path;
1775
+ if (typeof path !== "string" || !isKnowledgeAssetPath(path)) continue;
1776
+ const contentType = assetContentType(path);
1777
+ if (!contentType || typeof file.sizeBytes !== "number" || typeof file.sha256 !== "string") {
1778
+ continue;
1779
+ }
1780
+ index.set(path, { path, contentType, sizeBytes: file.sizeBytes, sha256: file.sha256 });
1781
+ }
1782
+ }
1783
+ this.assetsByPath = index;
1784
+ return index;
1785
+ }
1786
+ /** Declared assets, sorted by path. */
1787
+ assets() {
1788
+ return [...this.assetIndex().values()].sort((a, b) => a.path < b.path ? -1 : 1);
1789
+ }
1790
+ /** One declared asset's bytes, or null when the catalog declares no such asset. */
1791
+ readAsset(path) {
1792
+ const info = this.assetIndex().get(path);
1793
+ if (!info) return null;
1794
+ if (info.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES2) {
1795
+ throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
1796
+ }
1797
+ let bytes;
1798
+ try {
1799
+ bytes = readFileSync2(this.resolveCatalogPath(path));
1800
+ } catch (error) {
1801
+ throw new CatalogOpenError(
1802
+ `cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
1803
+ "corrupt"
1804
+ );
1805
+ }
1806
+ if (bytes.byteLength !== info.sizeBytes) {
1807
+ throw new CatalogOpenError(`asset size differs from the manifest: ${path}`, "corrupt");
1808
+ }
1809
+ return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
1810
+ }
1811
+ // ── search ────────────────────────────────────────────────────────────────
1812
+ /** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
1813
+ searchDocumentsFts(query, limit = 10) {
1814
+ const match = ftsQuery(query);
1815
+ if (!match) return [];
1816
+ try {
1817
+ return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
1818
+ documentId,
1819
+ rank: i
1820
+ }));
1821
+ } catch {
1822
+ return [];
1435
1823
  }
1436
1824
  }
1437
1825
  /**
@@ -1487,16 +1875,16 @@ var CatalogHandle = class _CatalogHandle {
1487
1875
  * after global cross-catalog routing has already spent the S budget.
1488
1876
  */
1489
1877
  searchShards(queryVector, shardIds, finalK) {
1490
- const queryBits = Buffer.from(quantizeBinary(queryVector));
1878
+ const queryBits = quantizeBinary(queryVector);
1491
1879
  const hits = [];
1492
1880
  for (const shardId of shardIds) {
1493
1881
  const shard = this.shards.find((s) => s.id === shardId);
1494
1882
  if (!shard) continue;
1495
1883
  const db = this.shardDb(shard);
1496
1884
  const k = rerankK(finalK, shard.chunkCount);
1497
- const candidates = db.prepare(
1498
- "SELECT chunk_id FROM vec_chunks WHERE embedding MATCH vec_bit(?) AND k = ? ORDER BY distance"
1499
- ).all(queryBits, BigInt(k));
1885
+ const candidates = hammingTopK(this.shardBits(shard), queryBits, k).map((hit) => ({
1886
+ chunk_id: hit.chunkId
1887
+ }));
1500
1888
  if (candidates.length === 0) continue;
1501
1889
  const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
1502
1890
  const reranked = candidates.map((c) => {
@@ -1563,35 +1951,35 @@ var CatalogHandle = class _CatalogHandle {
1563
1951
  return hits;
1564
1952
  }
1565
1953
  /**
1566
- * Embedder-free integrity smoke (§6.3): chunk 1's own bit vector must be
1567
- * its own nearest neighbor — proves the extension parses this file's vec
1568
- * blobs. Returns false instead of throwing (callers report a reason).
1954
+ * Embedder-free integrity smoke: chunk 1's own bit vector must be its own
1955
+ * nearest neighbor — proves the shard's vectors load and scan. Returns
1956
+ * false instead of throwing (callers report a reason).
1569
1957
  */
1570
1958
  selfKnnSmoke(shardId) {
1571
1959
  const shard = this.shards.find((s) => s.id === shardId);
1572
1960
  if (!shard) return false;
1573
1961
  try {
1574
- const db = this.shardDb(shard);
1575
- const first = db.prepare("SELECT embedding FROM vec_chunks WHERE chunk_id = 1").get();
1576
- if (!first) return false;
1577
- const nearest = db.prepare(
1578
- "SELECT chunk_id FROM vec_chunks WHERE embedding MATCH vec_bit(?) AND k = 1 ORDER BY distance"
1579
- ).get(Buffer.from(first.embedding));
1580
- return nearest !== void 0 && Number(nearest.chunk_id) === 1;
1962
+ const index = this.shardBits(shard);
1963
+ if (index.rows === 0) return false;
1964
+ const first = index.bits.subarray(0, index.bytesPerRow);
1965
+ const nearest = hammingTopK(index, first, 1)[0];
1966
+ return nearest !== void 0 && nearest.chunkId === 1 && nearest.distance === 0;
1581
1967
  } catch {
1582
1968
  return false;
1583
1969
  }
1584
1970
  }
1585
1971
  };
1586
- function rowToDocumentMeta(row) {
1587
- let attribution = null;
1588
- if (typeof row.attribution_json === "string") {
1589
- try {
1590
- attribution = JSON.parse(row.attribution_json);
1591
- } catch {
1592
- attribution = null;
1593
- }
1972
+ function parseJsonObject(raw) {
1973
+ if (typeof raw !== "string") return null;
1974
+ try {
1975
+ const parsed = JSON.parse(raw);
1976
+ return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
1977
+ } catch {
1978
+ return null;
1594
1979
  }
1980
+ }
1981
+ function rowToDocumentMeta(row) {
1982
+ const ordinal = row.ordinal;
1595
1983
  return {
1596
1984
  id: row.id,
1597
1985
  title: row.title,
@@ -1599,10 +1987,12 @@ function rowToDocumentMeta(row) {
1599
1987
  summary: row.summary ?? null,
1600
1988
  language: row.language,
1601
1989
  topicId: row.topic_id,
1990
+ ordinal: typeof ordinal === "number" || typeof ordinal === "bigint" ? Number(ordinal) : null,
1602
1991
  sourceUrl: row.source_url ?? null,
1603
1992
  sourceRevision: row.source_revision ?? null,
1604
1993
  sourceUpdatedAt: row.source_updated_at ?? null,
1605
- attribution
1994
+ attribution: parseJsonObject(row.attribution_json),
1995
+ meta: parseJsonObject(row.meta_json)
1606
1996
  };
1607
1997
  }
1608
1998
  function parseHeadingPath(raw) {
@@ -1618,7 +2008,17 @@ var ftsQuery = sanitizeFtsQuery;
1618
2008
  // src/reader/validate.ts
1619
2009
  import { readFile } from "fs/promises";
1620
2010
  import { join as join3 } from "path";
1621
- import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/gezel";
2011
+ import {
2012
+ KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3,
2013
+ MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES as MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
2014
+ MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES3,
2015
+ MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
2016
+ assetExtension as assetExtension2,
2017
+ assetKindForExtension as assetKindForExtension2,
2018
+ isKnowledgeAssetPath as isKnowledgeAssetPath2,
2019
+ sniffAssetType as sniffAssetType2,
2020
+ svgInertnessProblem as svgInertnessProblem2
2021
+ } from "@bendyline/gezk";
1622
2022
  async function validateExtractedCatalog(rootDir, opts = {}) {
1623
2023
  const checks = [];
1624
2024
  const check = (name, ok, detail) => {
@@ -1685,8 +2085,48 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1685
2085
  handle.meta.embedding_profile_id === manifest.embedding.id,
1686
2086
  `router meta profile ${handle.meta.embedding_profile_id}`
1687
2087
  );
2088
+ check(
2089
+ "meta-format",
2090
+ handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
2091
+ `router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
2092
+ );
1688
2093
  const topics = handle.topics();
1689
2094
  check("toc-present", topics.length >= 1, `${topics.length} topics`);
2095
+ check("topics-tree", ...topicTreeProblem(topics));
2096
+ const undeclared = handle.documentsWithUndeclaredTopic();
2097
+ check(
2098
+ "documents-topic-declared",
2099
+ undeclared === 0,
2100
+ `${undeclared} documents are filed under a topic the router does not declare`
2101
+ );
2102
+ const assetFiles = manifest.files.filter((f) => f.path.startsWith("assets/"));
2103
+ if (manifest.formatVersion === "0.5") {
2104
+ check(
2105
+ "assets-not-in-0.5",
2106
+ assetFiles.length === 0,
2107
+ `${assetFiles.length} assets/ entries in a 0.5 catalog`
2108
+ );
2109
+ } else {
2110
+ const badPaths = assetFiles.filter((f) => !isKnowledgeAssetPath2(f.path)).map((f) => f.path);
2111
+ check("assets-paths", badPaths.length === 0, `invalid asset paths: ${badPaths.join(", ")}`);
2112
+ const oversize = assetFiles.filter((f) => f.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES3);
2113
+ const totalBytes = assetFiles.reduce((sum, f) => sum + f.sizeBytes, 0);
2114
+ check(
2115
+ "assets-limits",
2116
+ assetFiles.length <= MAX_KNOWLEDGE_ASSET_COUNT2 && oversize.length === 0 && totalBytes <= MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
2117
+ `${assetFiles.length} assets, ${totalBytes} bytes, ${oversize.length} over the per-asset limit`
2118
+ );
2119
+ check(
2120
+ "counts-assets",
2121
+ (manifest.counts.assets ?? 0) === assetFiles.length,
2122
+ `manifest counts ${manifest.counts.assets ?? 0} assets, files declare ${assetFiles.length}`
2123
+ );
2124
+ }
2125
+ check(
2126
+ "license-notice",
2127
+ manifest.files.some((f) => f.path === manifest.license.noticePath),
2128
+ `manifest.files lacks the declared notice ${manifest.license.noticePath}`
2129
+ );
1690
2130
  const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
1691
2131
  check(
1692
2132
  "counts-documents",
@@ -1737,7 +2177,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1737
2177
  db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n
1738
2178
  );
1739
2179
  const vecCount = Number(
1740
- db.prepare("SELECT COUNT(*) AS n FROM vec_chunks").get().n
2180
+ db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit").get().n
1741
2181
  );
1742
2182
  const int8Count = Number(
1743
2183
  db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8").get().n
@@ -1747,11 +2187,68 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1747
2187
  chunkCount === shard.chunkCount && vecCount === chunkCount && int8Count === chunkCount,
1748
2188
  `chunks ${chunkCount}, vec ${vecCount}, int8 ${int8Count}, shards row ${shard.chunkCount}`
1749
2189
  );
2190
+ const dims = manifest.embedding.dimensions;
2191
+ const badBit = Number(
2192
+ db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit WHERE length(v) != ?").get(Math.ceil(dims / 8)).n
2193
+ );
2194
+ const badInt8 = Number(
2195
+ db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8 WHERE length(v) != ?").get(dims).n
2196
+ );
2197
+ check(
2198
+ `vector-widths:${shard.path}`,
2199
+ badBit === 0 && badInt8 === 0,
2200
+ `${badBit} bit rows and ${badInt8} int8 rows have the wrong width for ${dims} dimensions`
2201
+ );
2202
+ const span = db.prepare("SELECT MIN(id) AS lo, MAX(id) AS hi FROM chunks").get();
2203
+ check(
2204
+ `chunk-ids-dense:${shard.path}`,
2205
+ chunkCount === 0 || Number(span.lo) === 1 && Number(span.hi) === chunkCount,
2206
+ `ids span ${String(span.lo)}..${String(span.hi)} for ${chunkCount} chunks`
2207
+ );
1750
2208
  } finally {
1751
2209
  conn.close();
1752
2210
  }
1753
2211
  check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
1754
2212
  }
2213
+ if (handle.schemaVersion >= 3) {
2214
+ const meta = handle.checkDocumentMeta(MAX_KNOWLEDGE_DOCUMENT_META_BYTES);
2215
+ check(
2216
+ "document-meta-json",
2217
+ meta.invalid.length === 0 && meta.oversize.length === 0,
2218
+ `${meta.invalid.length > 0 ? `not a JSON object: ${meta.invalid.join(", ")}` : ""}${meta.oversize.length > 0 ? ` over ${MAX_KNOWLEDGE_DOCUMENT_META_BYTES} bytes: ${meta.oversize.join(", ")}` : ""}`.trim()
2219
+ );
2220
+ const declared = new Set(assetFiles.map((f) => f.path));
2221
+ for (const file of assetFiles) {
2222
+ const ext = assetExtension2(file.path);
2223
+ if (!ext) continue;
2224
+ const bytes = await readFile(join3(rootDir, file.path));
2225
+ const kind = sniffAssetType2(bytes);
2226
+ check(
2227
+ `asset-type:${file.path}`,
2228
+ kind === assetKindForExtension2(ext),
2229
+ `leading bytes say ${kind ?? "unknown"}, the extension says ${assetKindForExtension2(ext)}`
2230
+ );
2231
+ if (ext === "svg") {
2232
+ const problem = svgInertnessProblem2(bytes);
2233
+ check(`asset-svg-inert:${file.path}`, problem === null, problem ?? void 0);
2234
+ }
2235
+ }
2236
+ if (declared.size > 0) {
2237
+ const missing = [];
2238
+ for (const doc of handle.documentBodies()) {
2239
+ for (const target of assetReferences(doc.markdown)) {
2240
+ if (!declared.has(target)) missing.push(`${doc.id} \u2192 ${target}`);
2241
+ if (missing.length >= 5) break;
2242
+ }
2243
+ if (missing.length >= 5) break;
2244
+ }
2245
+ check(
2246
+ "document-asset-refs",
2247
+ missing.length === 0,
2248
+ `undeclared asset references: ${missing.join("; ")}`
2249
+ );
2250
+ }
2251
+ }
1755
2252
  for (const smoke of manifest.smokeQueries ?? []) {
1756
2253
  const hits = handle.searchDocumentsFts(smoke.query, SMOKE_QUERY_TOP_N).map((h) => h.documentId);
1757
2254
  const missing = smoke.expectedDocumentIds.filter((id) => !hits.includes(id));
@@ -1767,6 +2264,528 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1767
2264
  }
1768
2265
  return report(manifest);
1769
2266
  }
2267
+ function assetReferences(markdown) {
2268
+ const found = /* @__PURE__ */ new Set();
2269
+ for (const match of markdown.matchAll(/\]\(\s*<?(assets\/[^)\s>]+)/g)) {
2270
+ const target = match[1];
2271
+ if (target) found.add(target);
2272
+ }
2273
+ return [...found];
2274
+ }
2275
+ function topicTreeProblem(topics) {
2276
+ const byId = new Map(topics.map((t) => [t.id, t]));
2277
+ for (const topic of topics) {
2278
+ if (topic.parentId !== null && !byId.has(topic.parentId)) {
2279
+ return [false, `topic '${topic.id}' names an undeclared parent '${topic.parentId}'`];
2280
+ }
2281
+ }
2282
+ const depthOf = /* @__PURE__ */ new Map();
2283
+ for (const topic of topics) {
2284
+ const seen = /* @__PURE__ */ new Set();
2285
+ let depth = 0;
2286
+ let current = topic;
2287
+ while (current) {
2288
+ if (seen.has(current.id)) return [false, `topic '${topic.id}' sits in a parent cycle`];
2289
+ seen.add(current.id);
2290
+ depth += 1;
2291
+ if (depth > MAX_KNOWLEDGE_TOPIC_DEPTH) {
2292
+ return [false, `topic '${topic.id}' is deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH}`];
2293
+ }
2294
+ current = current.parentId === null ? void 0 : byId.get(current.parentId);
2295
+ }
2296
+ depthOf.set(topic.id, depth);
2297
+ }
2298
+ return [true, `${topics.length} topics, max depth ${Math.max(0, ...depthOf.values())}`];
2299
+ }
2300
+
2301
+ // src/export/duckdb.ts
2302
+ import { spawn } from "child_process";
2303
+ import { access } from "fs/promises";
2304
+ var DuckdbError = class extends Error {
2305
+ constructor(message, stderr) {
2306
+ super(message);
2307
+ this.stderr = stderr;
2308
+ this.name = "DuckdbError";
2309
+ }
2310
+ stderr;
2311
+ };
2312
+ async function duckdbVersion(binaryPath) {
2313
+ const { stdout } = await runDuckdbProcess(binaryPath, ["--version"], "");
2314
+ const match = /v?(\d+\.\d+\.\d+)/.exec(stdout);
2315
+ if (!match?.[1])
2316
+ throw new DuckdbError(`could not read the DuckDB version from: ${stdout.trim()}`, "");
2317
+ return match[1];
2318
+ }
2319
+ async function assertDuckdbCli(cli) {
2320
+ await access(cli.binaryPath).catch(() => {
2321
+ throw new DuckdbError(`DuckDB CLI not found at ${cli.binaryPath}`, "");
2322
+ });
2323
+ const version = await duckdbVersion(cli.binaryPath);
2324
+ if (cli.expectedVersion && version !== cli.expectedVersion) {
2325
+ throw new DuckdbError(
2326
+ `DuckDB ${cli.expectedVersion} expected, but ${cli.binaryPath} is ${version}; Parquet output is only reproducible against the pinned release`,
2327
+ ""
2328
+ );
2329
+ }
2330
+ return version;
2331
+ }
2332
+ async function runDuckdbScript(cli, sql) {
2333
+ const { stdout } = await runDuckdbProcess(cli.binaryPath, ["-batch", ":memory:"], sql);
2334
+ return stdout;
2335
+ }
2336
+ function runDuckdbProcess(binaryPath, args, stdin) {
2337
+ return new Promise((resolve5, reject) => {
2338
+ const child = spawn(binaryPath, args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
2339
+ let stdout = "";
2340
+ let stderr = "";
2341
+ child.stdout.setEncoding("utf8");
2342
+ child.stderr.setEncoding("utf8");
2343
+ child.stdout.on("data", (chunk) => {
2344
+ stdout += chunk;
2345
+ });
2346
+ child.stderr.on("data", (chunk) => {
2347
+ stderr += chunk;
2348
+ });
2349
+ child.on(
2350
+ "error",
2351
+ (err) => reject(new DuckdbError(`could not start DuckDB: ${err.message}`, stderr))
2352
+ );
2353
+ child.on("close", (code) => {
2354
+ if (code === 0) resolve5({ stdout });
2355
+ else
2356
+ reject(
2357
+ new DuckdbError(
2358
+ `DuckDB exited with code ${code}: ${stderr.trim() || stdout.trim()}`,
2359
+ stderr
2360
+ )
2361
+ );
2362
+ });
2363
+ child.stdin.on("error", () => {
2364
+ });
2365
+ child.stdin.end(stdin);
2366
+ });
2367
+ }
2368
+
2369
+ // src/export/parquet.ts
2370
+ import { createHash as createHash3 } from "crypto";
2371
+ import { createWriteStream as createWriteStream3 } from "fs";
2372
+ import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
2373
+ import { join as join4, resolve as resolve3 } from "path";
2374
+ import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
2375
+ import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
2376
+ var PARQUET_EXPORT_VERSION = 2;
2377
+ var DEFAULT_ROW_GROUP_SIZE = 32768;
2378
+ var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
2379
+ var PARQUET_REPORT_PATH = "parquet-manifest.json";
2380
+ async function exportCatalogParquet(opts) {
2381
+ const duckdbVersion2 = await assertDuckdbCli(opts.duckdb);
2382
+ const outDir = resolve3(opts.outDir);
2383
+ await mkdir3(outDir, { recursive: true });
2384
+ const work = join4(outDir, `.export-${process.pid}-${Date.now().toString(36)}`);
2385
+ await mkdir3(work, { recursive: true });
2386
+ try {
2387
+ let rootDir;
2388
+ if ("archivePath" in opts.source) {
2389
+ opts.onProgress?.({ phase: "extract" });
2390
+ rootDir = join4(work, "catalog");
2391
+ await extractGezkVerified(opts.source.archivePath, rootDir);
2392
+ } else {
2393
+ rootDir = resolve3(opts.source.rootDir);
2394
+ }
2395
+ const manifest = KnowledgeCatalogManifestSchema4.parse(
2396
+ JSON.parse(await readFile2(join4(rootDir, MANIFEST_PATH), "utf8"))
2397
+ );
2398
+ const staging = join4(work, "ndjson");
2399
+ await mkdir3(staging, { recursive: true });
2400
+ const files = [];
2401
+ const metadata = kvMetadata(manifest);
2402
+ const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
2403
+ const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
2404
+ try {
2405
+ const topicPaths = topicPathsOf(router);
2406
+ const topicRows = await stageTopics(router, join4(staging, "topics.ndjson"));
2407
+ opts.onProgress?.({ phase: "stage", table: "topics", rows: topicRows });
2408
+ files.push(
2409
+ await writeParquet(opts.duckdb, {
2410
+ outDir,
2411
+ file: "topics.parquet",
2412
+ table: "topics",
2413
+ rows: topicRows,
2414
+ sql: topicsCopy(
2415
+ join4(staging, "topics.ndjson"),
2416
+ join4(outDir, "topics.parquet"),
2417
+ rowGroupSize,
2418
+ metadata
2419
+ ),
2420
+ onProgress: opts.onProgress
2421
+ })
2422
+ );
2423
+ for (const shard of manifest.router.shards) {
2424
+ const tag = String(shard.id).padStart(3, "0");
2425
+ const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
2426
+ const documentRows = await stageDocuments(router, shard.id, topicPaths, documentsNdjson);
2427
+ opts.onProgress?.({
2428
+ phase: "stage",
2429
+ table: "documents",
2430
+ shardId: shard.id,
2431
+ rows: documentRows
2432
+ });
2433
+ files.push(
2434
+ await writeParquet(opts.duckdb, {
2435
+ outDir,
2436
+ file: `documents-${tag}.parquet`,
2437
+ table: "documents",
2438
+ shardId: shard.id,
2439
+ rows: documentRows,
2440
+ sql: documentsCopy(
2441
+ documentsNdjson,
2442
+ join4(outDir, `documents-${tag}.parquet`),
2443
+ rowGroupSize,
2444
+ metadata
2445
+ ),
2446
+ onProgress: opts.onProgress
2447
+ })
2448
+ );
2449
+ const shardDb = openCatalogDatabase(join4(rootDir, shard.path));
2450
+ const chunksNdjson = join4(staging, `chunks-${tag}.ndjson`);
2451
+ let chunkRows;
2452
+ try {
2453
+ chunkRows = await stageChunks(
2454
+ shardDb,
2455
+ shard.id,
2456
+ manifest.embedding.dimensions,
2457
+ chunksNdjson
2458
+ );
2459
+ } finally {
2460
+ shardDb.close();
2461
+ }
2462
+ opts.onProgress?.({ phase: "stage", table: "chunks", shardId: shard.id, rows: chunkRows });
2463
+ files.push(
2464
+ await writeParquet(opts.duckdb, {
2465
+ outDir,
2466
+ file: `chunks-${tag}.parquet`,
2467
+ table: "chunks",
2468
+ shardId: shard.id,
2469
+ rows: chunkRows,
2470
+ sql: chunksCopy(
2471
+ chunksNdjson,
2472
+ join4(outDir, `chunks-${tag}.parquet`),
2473
+ manifest.embedding.dimensions,
2474
+ rowGroupSize,
2475
+ metadata
2476
+ ),
2477
+ onProgress: opts.onProgress
2478
+ })
2479
+ );
2480
+ }
2481
+ } finally {
2482
+ router.close();
2483
+ }
2484
+ const report = {
2485
+ exportVersion: PARQUET_EXPORT_VERSION,
2486
+ catalogId: manifest.id,
2487
+ version: manifest.version,
2488
+ publisherId: manifest.publisher.id,
2489
+ formatVersion: manifest.formatVersion,
2490
+ embeddingProfileId: manifest.embedding.id,
2491
+ dimensions: manifest.embedding.dimensions,
2492
+ files,
2493
+ duckdbVersion: duckdbVersion2,
2494
+ exporter: { name: KNOWLEDGE_TOOLCHAIN.name, version: KNOWLEDGE_TOOLCHAIN.version }
2495
+ };
2496
+ const { writeFile } = await import("fs/promises");
2497
+ await writeFile(
2498
+ join4(outDir, PARQUET_REPORT_PATH),
2499
+ `${JSON.stringify(report, null, 2)}
2500
+ `,
2501
+ "utf8"
2502
+ );
2503
+ return report;
2504
+ } finally {
2505
+ await rm(work, { recursive: true, force: true }).catch(() => {
2506
+ });
2507
+ }
2508
+ }
2509
+ function ndjsonWriter(path) {
2510
+ const stream = createWriteStream3(path, { encoding: "utf8" });
2511
+ return {
2512
+ write(row) {
2513
+ const line = `${JSON.stringify(row)}
2514
+ `;
2515
+ if (stream.write(line)) return Promise.resolve();
2516
+ return new Promise((resolveDrain) => stream.once("drain", () => resolveDrain()));
2517
+ },
2518
+ close() {
2519
+ return new Promise((resolveClose, reject) => {
2520
+ stream.on("error", reject);
2521
+ stream.end(() => resolveClose());
2522
+ });
2523
+ }
2524
+ };
2525
+ }
2526
+ function topicPathsOf(router) {
2527
+ const parents = /* @__PURE__ */ new Map();
2528
+ for (const row of router.db.prepare("SELECT id, parent_id FROM topics").all()) {
2529
+ parents.set(row.id, row.parent_id);
2530
+ }
2531
+ const paths = /* @__PURE__ */ new Map();
2532
+ for (const id of parents.keys()) {
2533
+ const path = [];
2534
+ let cursor = id;
2535
+ const seen = /* @__PURE__ */ new Set();
2536
+ while (cursor && !seen.has(cursor)) {
2537
+ seen.add(cursor);
2538
+ path.unshift(cursor);
2539
+ cursor = parents.get(cursor) ?? null;
2540
+ }
2541
+ paths.set(id, path);
2542
+ }
2543
+ return paths;
2544
+ }
2545
+ async function stageTopics(router, path) {
2546
+ const out = ndjsonWriter(path);
2547
+ let rows = 0;
2548
+ const statement = router.db.prepare(
2549
+ "SELECT id, parent_id, name, description, sort_key, document_count FROM topics ORDER BY sort_key, id"
2550
+ );
2551
+ for (const row of statement.iterate()) {
2552
+ await out.write({
2553
+ id: row.id,
2554
+ parent_id: row.parent_id ?? null,
2555
+ name: row.name,
2556
+ description: row.description ?? null,
2557
+ sort_key: row.sort_key,
2558
+ document_count: Number(row.document_count)
2559
+ });
2560
+ rows++;
2561
+ }
2562
+ await out.close();
2563
+ return rows;
2564
+ }
2565
+ async function stageDocuments(router, shardId, topicPaths, path) {
2566
+ const out = ndjsonWriter(path);
2567
+ let rows = 0;
2568
+ const aliasesOf = router.db.prepare(
2569
+ "SELECT alias FROM aliases WHERE document_id = ? ORDER BY alias"
2570
+ );
2571
+ const extra = router.schemaVersion >= 3 ? ", ordinal, meta_json" : ", NULL AS ordinal, NULL AS meta_json";
2572
+ const statement = router.db.prepare(
2573
+ `SELECT id, title, slug, summary, language, topic_id, shard_id, chunk_count,
2574
+ source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob${extra}
2575
+ FROM documents WHERE shard_id = ? ORDER BY id`
2576
+ );
2577
+ for (const row of statement.iterate(shardId)) {
2578
+ const aliases = aliasesOf.all(row.id).map(
2579
+ (a) => a.alias
2580
+ );
2581
+ await out.write({
2582
+ id: row.id,
2583
+ title: row.title,
2584
+ slug: row.slug,
2585
+ summary: row.summary ?? null,
2586
+ language: row.language,
2587
+ topic_id: row.topic_id,
2588
+ topic_path: topicPaths.get(row.topic_id) ?? [row.topic_id],
2589
+ shard_id: Number(row.shard_id),
2590
+ chunk_count: Number(row.chunk_count),
2591
+ markdown: decodeBody(row.body_codec, row.body_blob),
2592
+ source_url: row.source_url ?? null,
2593
+ source_revision: row.source_revision ?? null,
2594
+ source_updated_at: row.source_updated_at ?? null,
2595
+ attribution: row.attribution_json ?? null,
2596
+ ordinal: row.ordinal == null ? null : Number(row.ordinal),
2597
+ meta: row.meta_json ?? null,
2598
+ aliases
2599
+ });
2600
+ rows++;
2601
+ }
2602
+ await out.close();
2603
+ return rows;
2604
+ }
2605
+ function decodeBody(codec, blob) {
2606
+ const bytes = Buffer.from(blob);
2607
+ if (codec === "br") {
2608
+ return brotliDecompressSync2(bytes, { maxOutputLength: MAX_KNOWLEDGE_DOCUMENT_BYTES }).toString(
2609
+ "utf8"
2610
+ );
2611
+ }
2612
+ if (codec === "none") return bytes.toString("utf8");
2613
+ throw new Error(`unknown body codec ${codec}`);
2614
+ }
2615
+ async function stageChunks(shard, shardId, dimensions, path) {
2616
+ const out = ndjsonWriter(path);
2617
+ const bitBytes = Math.ceil(dimensions / 8);
2618
+ let rows = 0;
2619
+ const statement = shard.db.prepare(
2620
+ `SELECT c.id, c.chunk_uid, c.document_id, c.ordinal, c.title, c.heading_path, c.heading_text,
2621
+ c.text, c.token_count, c.content_hash, c.line_start, c.line_end,
2622
+ i.v AS int8, b.v AS bits
2623
+ FROM chunks c
2624
+ JOIN chunk_vectors_int8 i ON i.chunk_id = c.id
2625
+ JOIN chunk_vectors_bit b ON b.chunk_id = c.id
2626
+ ORDER BY c.id`
2627
+ );
2628
+ for (const row of statement.iterate()) {
2629
+ const int8 = Buffer.from(row.int8);
2630
+ const bits = Buffer.from(row.bits);
2631
+ if (int8.length !== dimensions || bits.length !== bitBytes) {
2632
+ throw new Error(
2633
+ `chunk ${row.chunk_uid}: vector widths ${int8.length}/${bits.length} do not match the ${dimensions}-dimension profile`
2634
+ );
2635
+ }
2636
+ await out.write({
2637
+ id: Number(row.id),
2638
+ chunk_uid: row.chunk_uid,
2639
+ document_id: row.document_id,
2640
+ ordinal: Number(row.ordinal),
2641
+ title: row.title,
2642
+ heading_path: parseJsonStrings(row.heading_path),
2643
+ heading_text: row.heading_text,
2644
+ text: row.text,
2645
+ token_count: Number(row.token_count),
2646
+ content_hash: row.content_hash,
2647
+ line_start: Number(row.line_start),
2648
+ line_end: Number(row.line_end),
2649
+ shard_id: shardId,
2650
+ embedding: Array.from(new Int8Array(int8.buffer, int8.byteOffset, int8.length)),
2651
+ embedding_bit: bits.toString("base64")
2652
+ });
2653
+ rows++;
2654
+ }
2655
+ await out.close();
2656
+ return rows;
2657
+ }
2658
+ function parseJsonStrings(raw) {
2659
+ try {
2660
+ const parsed = JSON.parse(raw);
2661
+ return Array.isArray(parsed) ? parsed.filter((p) => typeof p === "string") : [];
2662
+ } catch {
2663
+ return [];
2664
+ }
2665
+ }
2666
+ function sqlString(value) {
2667
+ return `'${value.replace(/'/g, "''")}'`;
2668
+ }
2669
+ function kvMetadata(manifest) {
2670
+ return {
2671
+ gezk_catalog_id: manifest.id,
2672
+ gezk_catalog_version: manifest.version,
2673
+ gezk_publisher_id: manifest.publisher.id,
2674
+ gezk_format_version: manifest.formatVersion,
2675
+ gezk_embedding_profile: manifest.embedding.id,
2676
+ gezk_embedding_model: manifest.embedding.model.repo,
2677
+ gezk_embedding_dimensions: String(manifest.embedding.dimensions),
2678
+ gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
2679
+ gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
2680
+ gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
2681
+ gezk_license: manifest.license.name
2682
+ };
2683
+ }
2684
+ function copyOptions(rowGroupSize, metadata) {
2685
+ const kv = Object.entries(metadata).map(([key, value]) => `${key}: ${sqlString(value)}`).join(", ");
2686
+ return `(FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 3, ROW_GROUP_SIZE ${rowGroupSize}, KV_METADATA {${kv}})`;
2687
+ }
2688
+ var PRELUDE = "SET threads = 1;\nSET preserve_insertion_order = true;\n";
2689
+ function readNdjson(path, columns) {
2690
+ const spec = Object.entries(columns).map(([name, type]) => `${name}: ${sqlString(type)}`).join(", ");
2691
+ return `read_ndjson(${sqlString(path)}, columns = {${spec}}, maximum_object_size = ${NDJSON_MAX_OBJECT_BYTES})`;
2692
+ }
2693
+ function topicsCopy(ndjson, out, rowGroupSize, metadata) {
2694
+ const source = readNdjson(ndjson, {
2695
+ id: "VARCHAR",
2696
+ parent_id: "VARCHAR",
2697
+ name: "VARCHAR",
2698
+ description: "VARCHAR",
2699
+ sort_key: "VARCHAR",
2700
+ document_count: "BIGINT"
2701
+ });
2702
+ return `${PRELUDE}COPY (SELECT id, parent_id, name, description, sort_key, document_count FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2703
+ `;
2704
+ }
2705
+ function documentsCopy(ndjson, out, rowGroupSize, metadata) {
2706
+ const source = readNdjson(ndjson, {
2707
+ id: "VARCHAR",
2708
+ title: "VARCHAR",
2709
+ slug: "VARCHAR",
2710
+ summary: "VARCHAR",
2711
+ language: "VARCHAR",
2712
+ topic_id: "VARCHAR",
2713
+ topic_path: "VARCHAR[]",
2714
+ shard_id: "INTEGER",
2715
+ chunk_count: "INTEGER",
2716
+ markdown: "VARCHAR",
2717
+ source_url: "VARCHAR",
2718
+ source_revision: "VARCHAR",
2719
+ source_updated_at: "VARCHAR",
2720
+ attribution: "VARCHAR",
2721
+ ordinal: "INTEGER",
2722
+ meta: "VARCHAR",
2723
+ aliases: "VARCHAR[]"
2724
+ });
2725
+ return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, ordinal, meta, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2726
+ `;
2727
+ }
2728
+ function chunksCopy(ndjson, out, dimensions, rowGroupSize, metadata) {
2729
+ const source = readNdjson(ndjson, {
2730
+ id: "BIGINT",
2731
+ chunk_uid: "VARCHAR",
2732
+ document_id: "VARCHAR",
2733
+ ordinal: "INTEGER",
2734
+ title: "VARCHAR",
2735
+ heading_path: "VARCHAR[]",
2736
+ heading_text: "VARCHAR",
2737
+ text: "VARCHAR",
2738
+ token_count: "INTEGER",
2739
+ content_hash: "VARCHAR",
2740
+ line_start: "INTEGER",
2741
+ line_end: "INTEGER",
2742
+ shard_id: "INTEGER",
2743
+ embedding: "TINYINT[]",
2744
+ embedding_bit: "VARCHAR"
2745
+ });
2746
+ return `${PRELUDE}COPY (SELECT id, chunk_uid, document_id, ordinal, title, heading_path, heading_text, text, token_count, content_hash, line_start, line_end, shard_id, embedding::TINYINT[${dimensions}] AS embedding, from_base64(embedding_bit) AS embedding_bit FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2747
+ `;
2748
+ }
2749
+ async function writeParquet(cli, spec) {
2750
+ spec.onProgress?.({ phase: "write", file: spec.file });
2751
+ const target = join4(spec.outDir, spec.file);
2752
+ await rm(target, { force: true });
2753
+ await runDuckdbScript(cli, spec.sql);
2754
+ const bytes = await readFile2(target);
2755
+ return {
2756
+ path: spec.file,
2757
+ table: spec.table,
2758
+ ...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
2759
+ rows: spec.rows,
2760
+ sizeBytes: (await stat2(target)).size,
2761
+ sha256: createHash3("sha256").update(bytes).digest("hex")
2762
+ };
2763
+ }
2764
+ async function findDuckdbBinary(env = process.env) {
2765
+ const configured = env.GEZEL_DUCKDB_BIN?.trim();
2766
+ if (configured) return configured;
2767
+ const { readdir: readdir2 } = await import("fs/promises");
2768
+ const { homedir } = await import("os");
2769
+ const name = process.platform === "win32" ? "duckdb.exe" : "duckdb";
2770
+ const homes = [env.GEZEL_HOME, join4(homedir(), ".gezel-dev"), join4(homedir(), ".gezel")].filter(
2771
+ (h) => Boolean(h)
2772
+ );
2773
+ for (const home of homes) {
2774
+ const engines = join4(home, "engines", "duckdb");
2775
+ const versions = (await readdir2(engines).catch(() => [])).sort().reverse();
2776
+ for (const version of versions) {
2777
+ const candidate = join4(engines, version, name);
2778
+ if (await stat2(candidate).catch(() => null)) return candidate;
2779
+ }
2780
+ }
2781
+ const pathDirs = (env.PATH ?? "").split(process.platform === "win32" ? ";" : ":");
2782
+ for (const dir of pathDirs) {
2783
+ if (!dir) continue;
2784
+ const candidate = join4(dir, name);
2785
+ if (await stat2(candidate).catch(() => null)) return candidate;
2786
+ }
2787
+ return null;
2788
+ }
1770
2789
 
1771
2790
  // src/profiles/registry.ts
1772
2791
  var SHARED_ENCODING = {
@@ -1774,50 +2793,152 @@ var SHARED_ENCODING = {
1774
2793
  normalized: true,
1775
2794
  dimensions: 384,
1776
2795
  maxTokens: 512,
1777
- vectorEncoding: "bit384+int8",
2796
+ vectorEncoding: "bit+int8",
1778
2797
  distance: { stage1: "hamming", stage2: "cosine" },
1779
2798
  quantization: {
1780
2799
  int8: { method: "symmetric-linear", scale: 127 },
1781
2800
  binary: { method: "sign", threshold: 0, packing: "lsb-first" }
1782
2801
  }
1783
2802
  };
1784
- var GEZEL_MULTILINGUAL_E5_SMALL_1 = {
1785
- id: "gezel-multilingual-e5-small@1",
2803
+ var MULTILINGUAL_E5_SMALL_1 = {
2804
+ id: "multilingual-e5-small@1",
1786
2805
  model: {
1787
2806
  repo: "Xenova/multilingual-e5-small",
1788
- revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78"
2807
+ revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
2808
+ onnxFile: "onnx/model.onnx",
2809
+ onnxDigest: "sha256:4aa845c27760e06e9a686b9d8b5d440eae4b6612cd09e5b522b716d3941f77ff"
2810
+ },
2811
+ tokenizer: {
2812
+ kind: "sentencepiece-xlmr",
2813
+ file: "tokenizer.json",
2814
+ digest: "sha256:0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39"
1789
2815
  },
1790
- tokenizer: { kind: "sentencepiece-xlmr" },
1791
2816
  queryInstruction: "query: ",
1792
2817
  passageInstruction: "passage: ",
1793
2818
  ...SHARED_ENCODING
1794
2819
  };
1795
- var GEZEL_BGE_SMALL_EN_V15_1 = {
1796
- id: "gezel-bge-small-en-v1.5@1",
2820
+ var BGE_SMALL_EN_V15_1 = {
2821
+ id: "bge-small-en-v1.5@1",
1797
2822
  model: {
1798
2823
  repo: "Xenova/bge-small-en-v1.5",
1799
- revision: "ea104dacec62c0de699686887e3f920caeb4f3e3"
2824
+ revision: "ea104dacec62c0de699686887e3f920caeb4f3e3",
2825
+ onnxFile: "onnx/model.onnx",
2826
+ onnxDigest: "sha256:828e1496d7fabb79cfa4dcd84fa38625c0d3d21da474a00f08db0f559940cf35"
2827
+ },
2828
+ tokenizer: {
2829
+ kind: "wordpiece-bert",
2830
+ file: "tokenizer.json",
2831
+ digest: "sha256:d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66"
1800
2832
  },
1801
- tokenizer: { kind: "wordpiece-bert" },
1802
2833
  queryInstruction: "Represent this sentence for searching relevant passages: ",
1803
2834
  passageInstruction: "",
1804
2835
  ...SHARED_ENCODING
1805
2836
  };
1806
2837
  var KNOWLEDGE_EMBEDDING_PROFILES = [
1807
- GEZEL_MULTILINGUAL_E5_SMALL_1,
1808
- GEZEL_BGE_SMALL_EN_V15_1
2838
+ MULTILINGUAL_E5_SMALL_1,
2839
+ BGE_SMALL_EN_V15_1
1809
2840
  ];
1810
2841
  function knowledgeEmbeddingProfile(id) {
1811
2842
  return KNOWLEDGE_EMBEDDING_PROFILES.find((p) => p.id === id) ?? null;
1812
2843
  }
1813
- var GEZEL_MARKDOWN_CHUNKS_2 = {
1814
- id: "gezel-markdown-chunks@2",
2844
+ var MARKDOWN_CHUNKS_2 = {
2845
+ id: "markdown-chunks@2",
1815
2846
  unit: "tokens",
1816
2847
  tokenizer: "profile",
1817
- targetTokens: 420,
1818
- overlapTokens: 64,
1819
- contextHeader: { maxTokens: 64 }
2848
+ target: 420,
2849
+ overlap: 64,
2850
+ contextHeader: { max: 64 }
1820
2851
  };
2852
+ var GEZEL_MULTILINGUAL_E5_SMALL_1 = MULTILINGUAL_E5_SMALL_1;
2853
+ var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
2854
+ var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
2855
+
2856
+ // src/embedding/profile-embedder.ts
2857
+ import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
2858
+
2859
+ // src/embedding/artifact-verify.ts
2860
+ import { createHash as createHash4 } from "crypto";
2861
+ import { createReadStream } from "fs";
2862
+ import { stat as stat3 } from "fs/promises";
2863
+ import { join as join5 } from "path";
2864
+ import { embeddingProfileArtifacts } from "@bendyline/gezk";
2865
+ var EmbedderArtifactError = class extends Error {
2866
+ constructor(reason, profileId, checks) {
2867
+ super(describeFailure(reason, profileId, checks));
2868
+ this.reason = reason;
2869
+ this.profileId = profileId;
2870
+ this.checks = checks;
2871
+ this.name = "EmbedderArtifactError";
2872
+ }
2873
+ reason;
2874
+ profileId;
2875
+ checks;
2876
+ isActionable = true;
2877
+ };
2878
+ function describeFailure(reason, profileId, checks) {
2879
+ const detail = checks.filter((c) => c.actual !== c.expected).map(
2880
+ (c) => c.actual === null ? `${c.file} not found at ${c.path}` : `${c.file} is ${c.actual}, profile pins ${c.expected}`
2881
+ ).join("; ");
2882
+ return reason === "missing" ? `cannot verify the model files for profile ${profileId}: ${detail}` : `the model files loaded for profile ${profileId} differ from the ones it pins: ${detail}`;
2883
+ }
2884
+ function transformersCachePath(cacheDir, repo, revision, file) {
2885
+ return revision === "main" ? join5(cacheDir, repo, file) : join5(cacheDir, repo, revision, file);
2886
+ }
2887
+ var digestMemo = /* @__PURE__ */ new Map();
2888
+ var DIGEST_MEMO_MAX = 32;
2889
+ async function fileDigest(path) {
2890
+ let size;
2891
+ let mtimeMs;
2892
+ try {
2893
+ const info = await stat3(path);
2894
+ if (!info.isFile()) return null;
2895
+ size = info.size;
2896
+ mtimeMs = info.mtimeMs;
2897
+ } catch {
2898
+ return null;
2899
+ }
2900
+ const key = `${path} ${size} ${mtimeMs}`;
2901
+ const memo = digestMemo.get(key);
2902
+ if (memo) return memo;
2903
+ const digest = await new Promise((resolve5, reject) => {
2904
+ const hash = createHash4("sha256");
2905
+ createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
2906
+ });
2907
+ if (digestMemo.size >= DIGEST_MEMO_MAX) {
2908
+ const oldest = digestMemo.keys().next().value;
2909
+ if (oldest !== void 0) digestMemo.delete(oldest);
2910
+ }
2911
+ digestMemo.set(key, digest);
2912
+ return digest;
2913
+ }
2914
+ async function verifyProfileArtifacts(profile, opts) {
2915
+ const artifacts = embeddingProfileArtifacts(profile);
2916
+ const revision = opts.revision ?? profile.model.revision;
2917
+ const pinned = [];
2918
+ if (artifacts.onnxDigest) {
2919
+ pinned.push({ role: "onnx", file: artifacts.onnxFile, expected: artifacts.onnxDigest });
2920
+ }
2921
+ if (artifacts.tokenizerDigest) {
2922
+ pinned.push({
2923
+ role: "tokenizer",
2924
+ file: artifacts.tokenizerFile,
2925
+ expected: artifacts.tokenizerDigest
2926
+ });
2927
+ }
2928
+ if (pinned.length === 0) return { status: "unpinned", checks: [] };
2929
+ const checks = [];
2930
+ for (const { role, file, expected } of pinned) {
2931
+ const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file);
2932
+ checks.push({ role, file, path, expected, actual: await fileDigest(path) });
2933
+ }
2934
+ if (checks.some((c) => c.actual === null)) {
2935
+ throw new EmbedderArtifactError("missing", profile.id, checks);
2936
+ }
2937
+ if (checks.some((c) => c.actual !== c.expected)) {
2938
+ throw new EmbedderArtifactError("mismatch", profile.id, checks);
2939
+ }
2940
+ return { status: "verified", checks };
2941
+ }
1821
2942
 
1822
2943
  // src/embedding/profile-embedder.ts
1823
2944
  var MAX_BATCH = 8;
@@ -1829,6 +2950,42 @@ var EmbedderUnavailableError = class extends Error {
1829
2950
  this.name = "EmbedderUnavailableError";
1830
2951
  }
1831
2952
  };
2953
+ var ONNX_DTYPE_BY_SUFFIX = [
2954
+ ["_q4f16", "q4f16"],
2955
+ ["_quantized", "q8"],
2956
+ ["_uint8", "uint8"],
2957
+ ["_int8", "int8"],
2958
+ ["_bnb4", "bnb4"],
2959
+ ["_fp16", "fp16"],
2960
+ ["_q4", "q4"]
2961
+ ];
2962
+ var POOLING_BY_PROFILE = {
2963
+ mean: "mean",
2964
+ cls: "cls"
2965
+ };
2966
+ function resolveTransformersModelOptions(profile) {
2967
+ const { onnxFile, tokenizerFile } = embeddingProfileArtifacts2(profile);
2968
+ if (tokenizerFile !== "tokenizer.json") {
2969
+ throw new EmbedderUnavailableError(
2970
+ `profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
2971
+ );
2972
+ }
2973
+ if (!onnxFile.endsWith(".onnx")) {
2974
+ throw new EmbedderUnavailableError(
2975
+ `profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
2976
+ );
2977
+ }
2978
+ const slash = onnxFile.lastIndexOf("/");
2979
+ const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
2980
+ const base = onnxFile.slice(slash + 1, -".onnx".length);
2981
+ const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
2982
+ const [suffix, dtype] = variant ?? ["", "fp32"];
2983
+ const modelFileName = base.slice(0, base.length - suffix.length);
2984
+ if (modelFileName === "") {
2985
+ throw new EmbedderUnavailableError(`profile ${profile.id} pins an unnamed graph ${onnxFile}`);
2986
+ }
2987
+ return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
2988
+ }
1832
2989
  async function loadTransformers() {
1833
2990
  try {
1834
2991
  const specifier = "@huggingface/transformers";
@@ -1844,33 +3001,52 @@ async function loadTransformers() {
1844
3001
  }
1845
3002
  }
1846
3003
  async function createProfileEmbedder(profile, opts = {}) {
1847
- const transformers = await loadTransformers();
3004
+ const pooling = POOLING_BY_PROFILE[profile.pooling];
3005
+ if (!pooling) {
3006
+ throw new EmbedderUnavailableError(
3007
+ `profile ${profile.id} uses ${profile.pooling} pooling, which this runtime does not implement`
3008
+ );
3009
+ }
3010
+ const modelOptions = resolveTransformersModelOptions(profile);
3011
+ const transformers = opts.transformers ?? await loadTransformers();
1848
3012
  const cacheDir = opts.cacheDir ?? process.env.GEZEL_HF_CACHE_DIR;
1849
3013
  if (cacheDir) {
1850
3014
  transformers.env.cacheDir = cacheDir;
1851
3015
  transformers.env.useFSCache = true;
1852
3016
  transformers.env.allowRemoteModels = true;
1853
3017
  }
1854
- const modelOpts = {
1855
- revision: profile.model.revision,
1856
- ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
1857
- };
1858
3018
  const [pipe, tokenizer] = await Promise.all([
1859
- transformers.pipeline("feature-extraction", profile.model.repo, modelOpts),
1860
- transformers.AutoTokenizer.from_pretrained(profile.model.repo, modelOpts)
3019
+ transformers.pipeline("feature-extraction", profile.model.repo, {
3020
+ ...modelOptions,
3021
+ ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
3022
+ }),
3023
+ transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
3024
+ revision: modelOptions.revision
3025
+ })
1861
3026
  ]);
3027
+ let verification;
3028
+ try {
3029
+ verification = await verifyProfileArtifacts(profile, {
3030
+ cacheDir: cacheDir ?? transformers.env.cacheDir ?? ""
3031
+ });
3032
+ } catch (err) {
3033
+ await pipe.dispose?.().catch(() => {
3034
+ });
3035
+ throw err;
3036
+ }
1862
3037
  const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
1863
3038
  const embed = async (texts) => {
1864
3039
  const out = [];
1865
3040
  for (let i = 0; i < texts.length; i += MAX_BATCH) {
1866
3041
  const slice = texts.slice(i, i + MAX_BATCH).map(cap);
1867
- const result = await pipe(slice, { pooling: "mean", normalize: true });
3042
+ const result = await pipe(slice, { pooling, normalize: true });
1868
3043
  out.push(...result.tolist());
1869
3044
  }
1870
3045
  return out;
1871
3046
  };
1872
3047
  return {
1873
3048
  profile,
3049
+ verification,
1874
3050
  embed,
1875
3051
  embedQuery: async (text) => {
1876
3052
  const [vector] = await embed([`${profile.queryInstruction}${text}`]);
@@ -1884,29 +3060,427 @@ async function createProfileEmbedder(profile, opts = {}) {
1884
3060
  }
1885
3061
 
1886
3062
  // src/markdown-adapter/load.ts
1887
- import { readFile as readFile2, readdir } from "fs/promises";
1888
- import { join as join4, relative as relative3, sep } from "path";
1889
- import { CatalogDocumentSchema } from "@bendyline/gezel";
3063
+ import { existsSync } from "fs";
3064
+ import { readFile as readFile3, readdir } from "fs/promises";
3065
+ import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix2, relative as relative3, resolve as resolve4, sep } from "path";
3066
+ import {
3067
+ CatalogDocumentSchema,
3068
+ KnowledgeDocumentIdSchema,
3069
+ KnowledgeOrdinalSchema,
3070
+ assetExtension as assetExtension3,
3071
+ formatKnowledgeUri,
3072
+ topicSortKeyForOrder
3073
+ } from "@bendyline/gezk";
3074
+
3075
+ // src/markdown-adapter/frontmatter.ts
3076
+ import { parseDocument } from "yaml";
3077
+ var FRONTMATTER_MAX_BYTES = 64 * 1024;
3078
+ var OUTLINE_MAX_BYTES = 1024 * 1024;
3079
+ function parseYaml(text, opts) {
3080
+ const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
3081
+ if (Buffer.byteLength(text, "utf8") > maxBytes) {
3082
+ throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
3083
+ }
3084
+ const doc = parseDocument(text, {
3085
+ version: "1.2",
3086
+ schema: "core",
3087
+ uniqueKeys: true,
3088
+ logLevel: "silent"
3089
+ });
3090
+ const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
3091
+ if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
3092
+ return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
3093
+ }
3094
+ var OPEN = /^---[ \t]*\r?\n/;
3095
+ var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
3096
+ function parseMarkdownFrontMatter(source) {
3097
+ const open = OPEN.exec(source);
3098
+ if (!open) return { data: {}, body: source };
3099
+ const lines = source.slice(open[0].length).split(/\r?\n/);
3100
+ const closeAt = lines.findIndex((line) => CLOSE.test(line));
3101
+ if (closeAt === -1) throw new Error("front matter is not closed with ---");
3102
+ const yamlText = lines.slice(0, closeAt).join("\n");
3103
+ const body = lines.slice(closeAt + 1).join("\n");
3104
+ const parsed = parseYaml(yamlText, { what: "front matter" });
3105
+ if (parsed === null || parsed === void 0) return { data: {}, body };
3106
+ if (typeof parsed !== "object" || Array.isArray(parsed)) {
3107
+ throw new Error("front matter must be a mapping");
3108
+ }
3109
+ return { data: parsed, body };
3110
+ }
3111
+
3112
+ // src/markdown-adapter/outline.ts
3113
+ import { posix } from "path";
3114
+ var HAS_SCHEME = /^[a-z][a-z0-9+.-]*:/i;
3115
+ var MARKDOWN_EXT = /\.(md|markdown)$/i;
3116
+ function newOutlineTopic(name) {
3117
+ return { name, children: [], entries: [] };
3118
+ }
3119
+ function isRecord(value) {
3120
+ return typeof value === "object" && value !== null && !Array.isArray(value);
3121
+ }
3122
+ function isExternal(target) {
3123
+ return HAS_SCHEME.test(target) || target.startsWith("//");
3124
+ }
3125
+ function safeDecode(value) {
3126
+ try {
3127
+ return decodeURIComponent(value);
3128
+ } catch {
3129
+ return value;
3130
+ }
3131
+ }
3132
+ function stem(file) {
3133
+ return posix.basename(file).replace(MARKDOWN_EXT, "");
3134
+ }
3135
+ function resolveOutlinePath(baseDir, target) {
3136
+ const cleaned = safeDecode(
3137
+ target.replace(/\\/g, "/").replace(/[#?].*$/, "").trim()
3138
+ );
3139
+ if (cleaned === "" || posix.isAbsolute(cleaned)) return null;
3140
+ const joined = posix.normalize(posix.join(baseDir, cleaned));
3141
+ if (joined === ".." || joined.startsWith("../")) return null;
3142
+ return joined === "." ? "" : joined;
3143
+ }
3144
+ function assignOutlineOrders(topic) {
3145
+ topic.entries.forEach((entry, i) => {
3146
+ if (entry.order === void 0) entry.order = i + 1;
3147
+ });
3148
+ topic.children.forEach((child, i) => {
3149
+ if (child.order === void 0) child.order = i + 1;
3150
+ assignOutlineOrders(child);
3151
+ });
3152
+ }
3153
+ function titleFromFolderName(name) {
3154
+ const words = name.replace(/[_-]+/g, " ").trim().split(/\s+/).filter(Boolean);
3155
+ if (words.length === 0) return name;
3156
+ return words.map((word) => /[A-Z]/.test(word) ? word : word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
3157
+ }
3158
+ var HEADING2 = /^(#{1,6})\s+(.*?)\s*#*\s*$/;
3159
+ var LIST_ITEM = /^([ \t]*)(?:[-*+]|\d+[.)])\s+(.+?)\s*$/;
3160
+ var ITEM_LINK = /^\[([^\]]*)\]\(\s*<?([^)>\s]*)>?(?:\s+"[^"]*")?\s*\)$/;
3161
+ function materializeSummaryTopic(slot) {
3162
+ if (slot.topic) return slot.topic;
3163
+ const topic = newOutlineTopic(slot.title || "Untitled");
3164
+ if (slot.entry) {
3165
+ const at = slot.container.entries.indexOf(slot.entry);
3166
+ if (at >= 0) slot.container.entries.splice(at, 1);
3167
+ topic.entries.push(slot.entry);
3168
+ }
3169
+ slot.container.children.push(topic);
3170
+ slot.topic = topic;
3171
+ return topic;
3172
+ }
3173
+ function parseGitbookSummary(text, summaryFile) {
3174
+ const dir = posix.dirname(summaryFile);
3175
+ const baseDir = dir === "." ? "" : dir;
3176
+ const outline = {
3177
+ format: "gitbook",
3178
+ root: newOutlineTopic(""),
3179
+ consumed: [summaryFile],
3180
+ warnings: []
3181
+ };
3182
+ let container = outline.root;
3183
+ const stack = [];
3184
+ let fence = null;
3185
+ for (const rawLine of text.split(/\r?\n/)) {
3186
+ const line = rawLine.replace(/\t/g, " ");
3187
+ const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
3188
+ if (fenceMatch) {
3189
+ const marker = fenceMatch[1][0];
3190
+ if (fence === null) fence = marker;
3191
+ else if (marker === fence) fence = null;
3192
+ continue;
3193
+ }
3194
+ if (fence !== null) continue;
3195
+ const heading = HEADING2.exec(line);
3196
+ if (heading) {
3197
+ if (heading[1].length === 1) continue;
3198
+ const part = newOutlineTopic(heading[2].replace(/[*_`]/g, "").trim() || "Part");
3199
+ outline.root.children.push(part);
3200
+ container = part;
3201
+ stack.length = 0;
3202
+ continue;
3203
+ }
3204
+ const item = LIST_ITEM.exec(line);
3205
+ if (!item) continue;
3206
+ const indent = item[1].length;
3207
+ while (stack.length > 0 && stack[stack.length - 1].indent >= indent) {
3208
+ stack.pop();
3209
+ }
3210
+ const parent = stack[stack.length - 1];
3211
+ const parentTopic = parent ? materializeSummaryTopic(parent) : container;
3212
+ const content = item[2].trim();
3213
+ const link = ITEM_LINK.exec(content);
3214
+ const title = (link ? link[1] : content).replace(/[*_`]/g, "").trim();
3215
+ const slot = { indent, title, container: parentTopic };
3216
+ const target = link?.[2] ?? "";
3217
+ if (link && target !== "" && !isExternal(target)) {
3218
+ const file = resolveOutlinePath(baseDir, target);
3219
+ if (file === null) {
3220
+ outline.warnings.push(`${summaryFile}: '${target}' leaves the catalog tree; skipped`);
3221
+ } else if (!MARKDOWN_EXT.test(file)) {
3222
+ outline.warnings.push(`${summaryFile}: '${target}' is not a Markdown page; skipped`);
3223
+ } else {
3224
+ slot.entry = { file, ...title ? { title } : {} };
3225
+ parentTopic.entries.push(slot.entry);
3226
+ }
3227
+ }
3228
+ stack.push(slot);
3229
+ }
3230
+ assignOutlineOrders(outline.root);
3231
+ return outline;
3232
+ }
3233
+ function parseMkdocsNav(config, docsDir, configFile = "mkdocs.yml") {
3234
+ if (!isRecord(config)) throw new Error(`${configFile}: not a mapping`);
3235
+ const nav = config.nav;
3236
+ if (nav === void 0 || nav === null) return null;
3237
+ if (!Array.isArray(nav)) throw new Error(`${configFile}: 'nav' must be a list`);
3238
+ const outline = {
3239
+ format: "mkdocs",
3240
+ root: newOutlineTopic(""),
3241
+ consumed: [],
3242
+ warnings: []
3243
+ };
3244
+ const add = (container, target, title) => {
3245
+ if (isExternal(target)) return;
3246
+ const file = resolveOutlinePath(docsDir, target);
3247
+ if (file === null) {
3248
+ outline.warnings.push(
3249
+ `${configFile}: nav entry '${target}' leaves the docs directory; skipped`
3250
+ );
3251
+ return;
3252
+ }
3253
+ if (!MARKDOWN_EXT.test(file)) {
3254
+ outline.warnings.push(`${configFile}: nav entry '${target}' is not a Markdown page; skipped`);
3255
+ return;
3256
+ }
3257
+ container.entries.push({ file, ...title ? { title } : {} });
3258
+ };
3259
+ const walk = (items, container) => {
3260
+ for (const item of items) {
3261
+ if (typeof item === "string") {
3262
+ add(container, item);
3263
+ continue;
3264
+ }
3265
+ if (!isRecord(item)) {
3266
+ outline.warnings.push(`${configFile}: unrecognized nav entry ${JSON.stringify(item)}`);
3267
+ continue;
3268
+ }
3269
+ for (const [title, value] of Object.entries(item)) {
3270
+ if (typeof value === "string") {
3271
+ add(container, value, title.trim());
3272
+ } else if (Array.isArray(value)) {
3273
+ const section = newOutlineTopic(title.trim() || "Section");
3274
+ walk(value, section);
3275
+ container.children.push(section);
3276
+ } else {
3277
+ outline.warnings.push(
3278
+ `${configFile}: nav entry '${title}' is neither a page nor a section`
3279
+ );
3280
+ }
3281
+ }
3282
+ }
3283
+ };
3284
+ walk(nav, outline.root);
3285
+ assignOutlineOrders(outline.root);
3286
+ return outline;
3287
+ }
3288
+ function globToRegExp(pattern) {
3289
+ let source = "";
3290
+ for (let i = 0; i < pattern.length; i++) {
3291
+ const ch = pattern[i];
3292
+ if (ch === "*") {
3293
+ if (pattern[i + 1] === "*") {
3294
+ source += ".*";
3295
+ i++;
3296
+ } else source += "[^/]*";
3297
+ } else if (ch === "?") source += "[^/]";
3298
+ else source += ch.replace(/[.+^${}()|[\]\\]/g, "\\$&");
3299
+ }
3300
+ return new RegExp(`^${source}$`);
3301
+ }
3302
+ function parseJupyterBookToc(doc, tocDir, files, tocFile = "_toc.yml") {
3303
+ const outline = {
3304
+ format: "jupyter-book",
3305
+ root: newOutlineTopic(""),
3306
+ consumed: [],
3307
+ warnings: []
3308
+ };
3309
+ const known = new Set(files.map((f) => f.normalize("NFC")));
3310
+ const resolveFile = (spec) => {
3311
+ const candidates = MARKDOWN_EXT.test(spec) ? [spec] : [`${spec}.md`, `${spec}.markdown`];
3312
+ for (const candidate of candidates) {
3313
+ const path = resolveOutlinePath(tocDir, candidate);
3314
+ if (path !== null && known.has(path.normalize("NFC"))) return path;
3315
+ }
3316
+ return null;
3317
+ };
3318
+ const expandGlob = (pattern) => {
3319
+ const regex = globToRegExp(pattern.replace(/\\/g, "/"));
3320
+ const prefix = tocDir ? `${tocDir}/` : "";
3321
+ return files.filter((file) => {
3322
+ if (!file.startsWith(prefix)) return false;
3323
+ const rel = file.slice(prefix.length);
3324
+ return regex.test(rel) || regex.test(rel.replace(MARKDOWN_EXT, ""));
3325
+ }).sort();
3326
+ };
3327
+ const walkItems = (items, container, where) => {
3328
+ if (items === void 0 || items === null) return;
3329
+ if (!Array.isArray(items)) {
3330
+ outline.warnings.push(`${tocFile}: ${where} must be a list`);
3331
+ return;
3332
+ }
3333
+ for (const item of items) {
3334
+ if (!isRecord(item)) {
3335
+ outline.warnings.push(`${tocFile}: ${where} holds an unrecognized entry`);
3336
+ continue;
3337
+ }
3338
+ const title = typeof item.title === "string" ? item.title.trim() : void 0;
3339
+ if (typeof item.url === "string") continue;
3340
+ if (typeof item.glob === "string") {
3341
+ for (const file2 of expandGlob(item.glob)) container.entries.push({ file: file2 });
3342
+ continue;
3343
+ }
3344
+ if (typeof item.file !== "string") {
3345
+ outline.warnings.push(`${tocFile}: ${where} holds an entry without 'file'`);
3346
+ continue;
3347
+ }
3348
+ const file = resolveFile(item.file);
3349
+ if (file === null) {
3350
+ outline.warnings.push(
3351
+ `${tocFile}: '${item.file}' is not a Markdown file in the tree (notebooks are not supported); skipped`
3352
+ );
3353
+ continue;
3354
+ }
3355
+ const entry = { file, ...title ? { title } : {} };
3356
+ const children = item.sections ?? item.chapters ?? item.entries;
3357
+ if (Array.isArray(children) && children.length > 0) {
3358
+ const topic = newOutlineTopic(title ?? titleFromFolderName(stem(file)));
3359
+ if (!title) topic.titleFromFile = file;
3360
+ topic.entries.push(entry);
3361
+ walkItems(children, topic, `'${item.file}'`);
3362
+ container.children.push(topic);
3363
+ } else {
3364
+ container.entries.push(entry);
3365
+ }
3366
+ }
3367
+ };
3368
+ if (Array.isArray(doc)) {
3369
+ for (const item of doc) {
3370
+ if (isRecord(item) && typeof item.part === "string") {
3371
+ const part = newOutlineTopic(item.part.trim() || "Part");
3372
+ walkItems(item.chapters ?? item.sections, part, `part '${item.part}'`);
3373
+ outline.root.children.push(part);
3374
+ } else {
3375
+ walkItems([item], outline.root, "the list");
3376
+ }
3377
+ }
3378
+ } else if (isRecord(doc)) {
3379
+ if (typeof doc.root === "string") {
3380
+ const file = resolveFile(doc.root);
3381
+ if (file === null) {
3382
+ outline.warnings.push(
3383
+ `${tocFile}: root '${doc.root}' is not a Markdown file in the tree; skipped`
3384
+ );
3385
+ } else {
3386
+ outline.root.entries.push({ file });
3387
+ }
3388
+ }
3389
+ if (Array.isArray(doc.parts)) {
3390
+ for (const part of doc.parts) {
3391
+ if (!isRecord(part)) continue;
3392
+ const caption = typeof part.caption === "string" ? part.caption.trim() : "";
3393
+ const topic = newOutlineTopic(caption || "Part");
3394
+ walkItems(part.chapters ?? part.sections, topic, `part '${topic.name}'`);
3395
+ outline.root.children.push(topic);
3396
+ }
3397
+ }
3398
+ walkItems(doc.chapters, outline.root, "chapters");
3399
+ walkItems(doc.sections, outline.root, "sections");
3400
+ } else {
3401
+ throw new Error(`${tocFile}: not a mapping or a list`);
3402
+ }
3403
+ assignOutlineOrders(outline.root);
3404
+ return outline;
3405
+ }
3406
+
3407
+ // src/markdown-adapter/load.ts
1890
3408
  var ROOT_TOPIC_ID = "general";
1891
3409
  var SUMMARY_MAX_CHARS = 280;
1892
- function parseFrontMatter(markdown) {
1893
- if (!markdown.startsWith("---\n") && !markdown.startsWith("---\r\n")) return { body: markdown };
1894
- const end = markdown.indexOf("\n---", 3);
1895
- if (end === -1) return { body: markdown };
1896
- const head = markdown.slice(markdown.indexOf("\n") + 1, end);
1897
- const body = markdown.slice(markdown.indexOf("\n", end + 1) + 1);
1898
- const out = { body };
1899
- for (const line of head.split("\n")) {
1900
- const at = line.indexOf(":");
1901
- if (at === -1) continue;
1902
- const key = line.slice(0, at).trim();
1903
- const value = line.slice(at + 1).trim();
1904
- if (!value) continue;
1905
- if (key === "title") out.title = value;
1906
- else if (key === "summary") out.summary = value;
1907
- else if (key === "aliases") {
1908
- out.aliases = value.split(",").map((a) => a.trim()).filter(Boolean);
3410
+ var TOPIC_SIDECAR = "_topic.yaml";
3411
+ var HUGO_SECTION_PAGE = "_index";
3412
+ var SECTION_PAGE_ORDINAL = -2147483648;
3413
+ var RESERVED_KEYS = /* @__PURE__ */ new Set(["title", "summary", "aliases", "id", "order", "subcategory"]);
3414
+ function readFrontMatter(raw, file, hugo) {
3415
+ let data;
3416
+ let body;
3417
+ try {
3418
+ ({ data, body } = parseMarkdownFrontMatter(raw));
3419
+ } catch (error) {
3420
+ throw new Error(`${file}: ${error instanceof Error ? error.message : String(error)}`);
3421
+ }
3422
+ const out = { meta: {}, body };
3423
+ const str = (key) => {
3424
+ const value = data[key];
3425
+ if (value === void 0 || value === null) return void 0;
3426
+ if (typeof value === "number") return String(value);
3427
+ if (typeof value !== "string")
3428
+ throw new Error(`${file}: front matter '${key}' must be a string`);
3429
+ return value.trim() || void 0;
3430
+ };
3431
+ out.title = str("title");
3432
+ out.summary = str("summary");
3433
+ out.id = str("id");
3434
+ const aliases = data.aliases;
3435
+ if (typeof aliases === "string") {
3436
+ out.aliases = aliases.split(",").map((a) => a.trim()).filter(Boolean);
3437
+ } else if (Array.isArray(aliases)) {
3438
+ out.aliases = aliases.map((a) => String(a).trim()).filter(Boolean);
3439
+ } else if (aliases !== void 0 && aliases !== null) {
3440
+ throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
3441
+ }
3442
+ if (data.order !== void 0 && data.order !== null) {
3443
+ const order = KnowledgeOrdinalSchema.safeParse(data.order);
3444
+ if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
3445
+ out.order = order.data;
3446
+ }
3447
+ const sub = data.subcategory;
3448
+ if (sub !== void 0 && sub !== null) {
3449
+ if (typeof sub !== "object" || Array.isArray(sub)) {
3450
+ throw new Error(`${file}: front matter 'subcategory' must be a mapping with an id`);
3451
+ }
3452
+ const record = sub;
3453
+ if (typeof record.id !== "string" || !record.id.trim()) {
3454
+ throw new Error(`${file}: front matter 'subcategory' must be a mapping with an id`);
3455
+ }
3456
+ const subcategory = { id: record.id.trim() };
3457
+ if (record.title !== void 0 && record.title !== null) {
3458
+ if (typeof record.title !== "string") {
3459
+ throw new Error(`${file}: front matter 'subcategory.title' must be a string`);
3460
+ }
3461
+ subcategory.title = record.title.trim();
3462
+ }
3463
+ if (record.order !== void 0 && record.order !== null) {
3464
+ const order = KnowledgeOrdinalSchema.safeParse(record.order);
3465
+ if (!order.success)
3466
+ throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
3467
+ subcategory.order = order.data;
3468
+ }
3469
+ out.subcategory = subcategory;
3470
+ }
3471
+ if (hugo) {
3472
+ if (data.draft === true) out.unpublished = "draft";
3473
+ else if (data.headless === true) out.unpublished = "headless";
3474
+ if (data.weight !== void 0 && data.weight !== null) {
3475
+ const weight = KnowledgeOrdinalSchema.safeParse(data.weight);
3476
+ if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
3477
+ if (weight.data > 0) out.weight = weight.data;
1909
3478
  }
3479
+ out.description = str("description");
3480
+ }
3481
+ for (const [key, value] of Object.entries(data)) {
3482
+ if (RESERVED_KEYS.has(key) || value === void 0) continue;
3483
+ out.meta[key] = value;
1910
3484
  }
1911
3485
  return out;
1912
3486
  }
@@ -1940,7 +3514,7 @@ async function walkMarkdownFiles(root) {
1940
3514
  );
1941
3515
  for (const entry of entries) {
1942
3516
  if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
1943
- const abs = join4(dir, entry.name);
3517
+ const abs = join6(dir, entry.name);
1944
3518
  if (entry.isDirectory()) await visit(abs);
1945
3519
  else if (entry.isFile() && /\.(md|markdown)$/i.test(entry.name)) out.push(abs);
1946
3520
  }
@@ -1948,17 +3522,233 @@ async function walkMarkdownFiles(root) {
1948
3522
  await visit(root);
1949
3523
  return out;
1950
3524
  }
1951
- async function loadMarkdownCatalog(rootDir, opts) {
3525
+ function isRecord2(value) {
3526
+ return typeof value === "object" && value !== null && !Array.isArray(value);
3527
+ }
3528
+ async function readTopicSidecar(dir) {
3529
+ const path = join6(dir, TOPIC_SIDECAR);
3530
+ if (!existsSync(path)) return void 0;
3531
+ const raw = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
3532
+ const data = parseYaml(raw, { what: path });
3533
+ if (data === null || data === void 0) return {};
3534
+ if (!isRecord2(data)) throw new Error(`${path}: must be a mapping`);
3535
+ const out = {};
3536
+ if (typeof data.name === "string" && data.name.trim()) out.name = data.name.trim();
3537
+ if (typeof data.description === "string" && data.description.trim()) {
3538
+ out.description = data.description.trim();
3539
+ }
3540
+ if (data.order !== void 0 && data.order !== null) {
3541
+ const order = KnowledgeOrdinalSchema.safeParse(data.order);
3542
+ if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
3543
+ out.order = order.data;
3544
+ }
3545
+ return out;
3546
+ }
3547
+ function applyOverride(topic, override) {
3548
+ if (!topic || !override) return;
3549
+ if (override.name) topic.name = override.name;
3550
+ if (override.description) topic.description = override.description;
3551
+ if (override.order !== void 0) topic.sortKey = topicSortKeyForOrder(override.order);
3552
+ }
3553
+ function insideRoot(rootDir, abs) {
3554
+ const rel = relative3(rootDir, abs).split(sep).join("/");
3555
+ if (rel === ".." || rel.startsWith("../") || isAbsolute3(rel)) return null;
3556
+ return rel;
3557
+ }
3558
+ function firstExisting(candidates) {
3559
+ return candidates.find((candidate) => existsSync(candidate));
3560
+ }
3561
+ async function readMkdocsDocsDir(mkdocsPath) {
3562
+ const config = parseYaml((await readFile3(mkdocsPath, "utf8")).replace(/^\uFEFF/, ""), {
3563
+ what: basename(mkdocsPath),
3564
+ maxBytes: OUTLINE_MAX_BYTES,
3565
+ tolerateUnknownTags: true,
3566
+ maxAliasCount: 100
3567
+ });
3568
+ const docsDir = isRecord2(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
3569
+ return resolve4(dirname3(mkdocsPath), docsDir);
3570
+ }
3571
+ async function detectTableOfContents(rootDir, projectDir = rootDir) {
3572
+ const summary = join6(rootDir, "SUMMARY.md");
3573
+ if (existsSync(summary)) return { format: "gitbook", path: summary };
3574
+ const jupyter = firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
3575
+ if (jupyter) return { format: "jupyter-book", path: jupyter };
3576
+ const mkdocs = firstExisting(
3577
+ [rootDir, projectDir, dirname3(rootDir)].flatMap((dir) => [
3578
+ join6(dir, "mkdocs.yml"),
3579
+ join6(dir, "mkdocs.yaml")
3580
+ ])
3581
+ );
3582
+ if (mkdocs) return { format: "mkdocs", path: mkdocs };
1952
3583
  const files = await walkMarkdownFiles(rootDir);
3584
+ if (files.some((file) => basename(file).replace(/\.(md|markdown)$/i, "") === HUGO_SECTION_PAGE)) {
3585
+ return { format: "hugo" };
3586
+ }
3587
+ return { format: "folders" };
3588
+ }
3589
+ async function readOutlineText(path) {
3590
+ const text = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
3591
+ if (Buffer.byteLength(text, "utf8") > OUTLINE_MAX_BYTES) {
3592
+ throw new Error(`${basename(path)} exceeds ${OUTLINE_MAX_BYTES} bytes`);
3593
+ }
3594
+ return text;
3595
+ }
3596
+ async function loadOutline(rootDir, toc, files) {
3597
+ if (toc.format === "folders" || toc.format === "hugo") return { outline: null };
3598
+ const explicit = toc.path ? isAbsolute3(toc.path) ? toc.path : join6(rootDir, toc.path) : void 0;
3599
+ if (explicit && !existsSync(explicit)) {
3600
+ throw new Error(`${toc.format} table of contents not found: ${explicit}`);
3601
+ }
3602
+ if (toc.format === "gitbook") {
3603
+ const path2 = explicit ?? join6(rootDir, "SUMMARY.md");
3604
+ const rel = insideRoot(rootDir, path2);
3605
+ if (!existsSync(path2)) throw new Error(`gitbook table of contents not found: ${path2}`);
3606
+ if (rel === null)
3607
+ throw new Error(`${basename(path2)} must live inside the content root ${rootDir}`);
3608
+ return { outline: parseGitbookSummary(await readOutlineText(path2), rel), path: path2 };
3609
+ }
3610
+ if (toc.format === "mkdocs") {
3611
+ const path2 = explicit ?? firstExisting(
3612
+ [rootDir, dirname3(rootDir)].flatMap((dir) => [
3613
+ join6(dir, "mkdocs.yml"),
3614
+ join6(dir, "mkdocs.yaml")
3615
+ ])
3616
+ );
3617
+ if (!path2) throw new Error(`mkdocs.yml not found in ${rootDir} or its parent`);
3618
+ const config = parseYaml(await readOutlineText(path2), {
3619
+ what: basename(path2),
3620
+ maxBytes: OUTLINE_MAX_BYTES,
3621
+ tolerateUnknownTags: true,
3622
+ maxAliasCount: 100
3623
+ });
3624
+ const docsDir = await readMkdocsDocsDir(path2);
3625
+ const docsRel = insideRoot(rootDir, docsDir);
3626
+ if (docsRel === null) {
3627
+ throw new Error(
3628
+ `${basename(path2)}: docs_dir ${docsDir} is outside the content root ${rootDir}`
3629
+ );
3630
+ }
3631
+ return { outline: parseMkdocsNav(config, docsRel, basename(path2)), path: path2 };
3632
+ }
3633
+ const path = explicit ?? firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
3634
+ if (!path) throw new Error(`_toc.yml not found in ${rootDir}`);
3635
+ const tocDir = insideRoot(rootDir, dirname3(path));
3636
+ if (tocDir === null)
3637
+ throw new Error(`${basename(path)} must live inside the content root ${rootDir}`);
3638
+ const doc = parseYaml(await readOutlineText(path), {
3639
+ what: basename(path),
3640
+ maxBytes: OUTLINE_MAX_BYTES
3641
+ });
3642
+ return { outline: parseJupyterBookToc(doc, tocDir, files, basename(path)), path };
3643
+ }
3644
+ var INLINE_LINK = /(!?)\[([^\]]*)\]\(\s*(<[^>]*>|[^)\s]+)((?:\s+"[^"]*")?)\s*\)/g;
3645
+ var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
3646
+ function resolveInTree(ctx, target) {
3647
+ const resolved = posix2.normalize(posix2.join(ctx.docDir, target));
3648
+ if (resolved === ".." || resolved.startsWith("../") || posix2.isAbsolute(resolved)) return null;
3649
+ return resolved === "." ? "" : resolved;
3650
+ }
3651
+ function rewriteTarget(ctx, isImage, rawTarget) {
3652
+ const target = rawTarget.startsWith("<") ? rawTarget.slice(1, -1) : rawTarget;
3653
+ if (target === "" || HAS_SCHEME2.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
3654
+ return null;
3655
+ }
3656
+ const hashAt = target.search(/[#?]/);
3657
+ const pathPart = hashAt === -1 ? target : target.slice(0, hashAt);
3658
+ const decoded = safeDecode2(pathPart);
3659
+ const resolved = resolveInTree(ctx, decoded);
3660
+ if (resolved === null) {
3661
+ ctx.warn(`${ctx.file}: link '${target}' leaves the catalog tree; left as is`);
3662
+ return null;
3663
+ }
3664
+ if (isImage) {
3665
+ if (!assetExtension3(resolved)) {
3666
+ ctx.warn(`${ctx.file}: image '${target}' is not a supported asset type; left as is`);
3667
+ return null;
3668
+ }
3669
+ const abs = join6(ctx.rootDir, ...resolved.split("/"));
3670
+ if (!existsSync(abs)) {
3671
+ const message = `${ctx.file}: image '${target}' does not exist in the catalog tree`;
3672
+ if (ctx.missingAssets === "error") throw new Error(message);
3673
+ ctx.warn(`${message}; left as is`);
3674
+ return null;
3675
+ }
3676
+ const archivePath = resolved.startsWith("assets/") ? resolved : `assets/${resolved}`;
3677
+ if (!ctx.assets.has(archivePath))
3678
+ ctx.assets.set(archivePath, { path: archivePath, absPath: abs });
3679
+ return archivePath;
3680
+ }
3681
+ if (!/\.(md|markdown)$/i.test(resolved)) return null;
3682
+ const documentId = ctx.idByRel.get(resolved.replace(/\.(md|markdown)$/i, "").normalize("NFC"));
3683
+ if (!documentId) {
3684
+ ctx.warn(`${ctx.file}: link '${target}' does not name a document in the catalog; left as is`);
3685
+ return null;
3686
+ }
3687
+ if (!ctx.uri) return null;
3688
+ return formatKnowledgeUri({
3689
+ publisherId: ctx.uri.publisherId,
3690
+ catalogId: ctx.uri.catalogId,
3691
+ documentId
3692
+ });
3693
+ }
3694
+ function safeDecode2(value) {
3695
+ try {
3696
+ return decodeURIComponent(value);
3697
+ } catch {
3698
+ return value;
3699
+ }
3700
+ }
3701
+ function rewriteLinks(ctx, markdown) {
3702
+ const lines = markdown.split("\n");
3703
+ let fence = null;
3704
+ return lines.map((line) => {
3705
+ const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
3706
+ if (fenceMatch) {
3707
+ const marker = fenceMatch[1];
3708
+ if (fence === null) fence = marker[0];
3709
+ else if (marker[0] === fence) fence = null;
3710
+ return line;
3711
+ }
3712
+ if (fence !== null) return line;
3713
+ return line.replace(
3714
+ INLINE_LINK,
3715
+ (whole, bang, text, target, title) => {
3716
+ const rewritten = rewriteTarget(ctx, bang === "!", target);
3717
+ return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
3718
+ }
3719
+ );
3720
+ }).join("\n");
3721
+ }
3722
+ async function loadMarkdownCatalog(rootDir, opts) {
3723
+ const warn = opts.onWarning ?? (() => {
3724
+ });
3725
+ const toc = opts.toc ?? { format: "folders" };
3726
+ const hugo = toc.format === "hugo";
3727
+ const ignore = new Set(opts.ignore ?? []);
3728
+ const relOf = (abs) => relative3(rootDir, abs).split(sep).join("/");
3729
+ let files = (await walkMarkdownFiles(rootDir)).filter((abs) => !ignore.has(relOf(abs)));
3730
+ const { outline, path: outlinePath } = await loadOutline(rootDir, toc, files.map(relOf));
3731
+ for (const message of outline?.warnings ?? []) warn(message);
3732
+ if (outline) {
3733
+ const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
3734
+ files = files.filter((abs) => !consumed.has(relOf(abs).normalize("NFC")));
3735
+ }
1953
3736
  if (files.length === 0) {
1954
3737
  throw new Error(`no Markdown files found under ${rootDir}`);
1955
3738
  }
1956
3739
  const takenTopicIds = /* @__PURE__ */ new Set();
1957
3740
  const topicIdByDir = /* @__PURE__ */ new Map();
1958
3741
  const topics = [];
3742
+ const topicById = /* @__PURE__ */ new Map();
1959
3743
  const rootTopicId = opts.rootTopicId ?? ROOT_TOPIC_ID;
1960
3744
  let rootTopicUsed = false;
1961
- const ensureTopicChain = (relDirPosix) => {
3745
+ let rootOverride;
3746
+ const addTopic = (topic) => {
3747
+ topics.push(topic);
3748
+ topicById.set(topic.id, topic);
3749
+ return topic;
3750
+ };
3751
+ const ensureTopicChain = async (relDirPosix) => {
1962
3752
  if (!relDirPosix) {
1963
3753
  rootTopicUsed = true;
1964
3754
  return [rootTopicId];
@@ -1972,32 +3762,141 @@ async function loadMarkdownCatalog(rootDir, opts) {
1972
3762
  if (!id) {
1973
3763
  id = topicIdFor(segment, takenTopicIds);
1974
3764
  topicIdByDir.set(prefix, id);
1975
- topics.push({
3765
+ const topic = addTopic({
1976
3766
  id,
1977
- name: segment,
3767
+ name: titleFromFolderName(segment),
1978
3768
  ...path.length > 0 ? { parentId: path[path.length - 1] } : {}
1979
3769
  });
3770
+ applyOverride(topic, await readTopicSidecar(join6(rootDir, ...prefix.split("/"))));
3771
+ applyOverride(topic, opts.topics?.[prefix]);
1980
3772
  }
1981
3773
  path.push(id);
1982
3774
  }
1983
3775
  return path;
1984
3776
  };
1985
- const documents = [];
1986
- const seenIds = /* @__PURE__ */ new Set();
3777
+ const placements = /* @__PURE__ */ new Map();
3778
+ if (outline) {
3779
+ const walk = (topic, chain) => {
3780
+ for (const entry of topic.entries) {
3781
+ const key = entry.file.normalize("NFC");
3782
+ if (placements.has(key)) {
3783
+ warn(
3784
+ `${entry.file}: listed more than once in the table of contents; the first place wins`
3785
+ );
3786
+ } else {
3787
+ placements.set(key, { chain, entry });
3788
+ }
3789
+ }
3790
+ for (const child of topic.children) walk(child, [...chain, child]);
3791
+ };
3792
+ walk(outline.root, []);
3793
+ }
3794
+ const outlineTopicIds = /* @__PURE__ */ new Map();
3795
+ const ensureOutlineChain = (chain) => {
3796
+ if (chain.length === 0) {
3797
+ rootTopicUsed = true;
3798
+ return [rootTopicId];
3799
+ }
3800
+ const ids = [];
3801
+ for (const node of chain) {
3802
+ let id = outlineTopicIds.get(node);
3803
+ if (!id) {
3804
+ id = topicIdFor(node.name, takenTopicIds);
3805
+ outlineTopicIds.set(node, id);
3806
+ const topic = {
3807
+ id,
3808
+ name: node.name,
3809
+ ...ids.length > 0 ? { parentId: ids[ids.length - 1] } : {}
3810
+ };
3811
+ if (node.description) topic.description = node.description;
3812
+ if (node.order !== void 0) topic.sortKey = topicSortKeyForOrder(node.order);
3813
+ addTopic(topic);
3814
+ }
3815
+ ids.push(id);
3816
+ }
3817
+ return ids;
3818
+ };
3819
+ const shelves = /* @__PURE__ */ new Map();
3820
+ const ensureShelf = (parentId, sub, file) => {
3821
+ const key = `${parentId}/${sub.id}`;
3822
+ const existing = shelves.get(key);
3823
+ if (existing) {
3824
+ if (existing.title !== sub.title || existing.order !== sub.order) {
3825
+ throw new Error(
3826
+ `${file}: subcategory '${sub.id}' is declared with a different title or order in ${existing.file}`
3827
+ );
3828
+ }
3829
+ return existing.topicId;
3830
+ }
3831
+ const topicId = topicIdFor(`${parentId}-${sub.id}`, takenTopicIds);
3832
+ const topic = { id: topicId, name: sub.title ?? sub.id, parentId };
3833
+ if (sub.order !== void 0) topic.sortKey = topicSortKeyForOrder(sub.order);
3834
+ addTopic(topic);
3835
+ shelves.set(key, { topicId, title: sub.title, order: sub.order, file });
3836
+ return topicId;
3837
+ };
3838
+ const loaded = [];
3839
+ const fileById = /* @__PURE__ */ new Map();
3840
+ const idByRel = /* @__PURE__ */ new Map();
3841
+ const seenRel = /* @__PURE__ */ new Set();
1987
3842
  for (const abs of files) {
1988
- const rel = relative3(rootDir, abs).split(sep).join("/");
3843
+ const rel = relOf(abs);
1989
3844
  const relDir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
1990
- const topicPath = ensureTopicChain(relDir);
1991
- const id = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
1992
- if (seenIds.has(id)) throw new Error(`duplicate document id from path: ${rel}`);
1993
- seenIds.add(id);
1994
- const raw = (await readFile2(abs, "utf8")).replace(/^\uFEFF/, "");
1995
- const fm = parseFrontMatter(raw);
1996
- const stem = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
1997
- const title = fm.title ?? firstHeading(fm.body) ?? stem;
1998
- const summary = fm.summary ?? firstParagraph(fm.body);
1999
- documents.push(
2000
- CatalogDocumentSchema.parse({
3845
+ const relNoExt = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
3846
+ const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
3847
+ const fm = readFrontMatter(raw, rel, hugo);
3848
+ if (fm.unpublished) {
3849
+ warn(`${rel}: skipped, ${fm.unpublished} page`);
3850
+ continue;
3851
+ }
3852
+ seenRel.add(rel.normalize("NFC"));
3853
+ const id = fm.id ?? relNoExt;
3854
+ const idCheck = KnowledgeDocumentIdSchema.safeParse(id);
3855
+ if (!idCheck.success) throw new Error(`${rel}: front matter 'id' is not a valid document id`);
3856
+ const previous = fileById.get(id);
3857
+ if (previous) throw new Error(`${rel}: document id '${id}' is already used by ${previous}`);
3858
+ fileById.set(id, rel);
3859
+ idByRel.set(relNoExt, id);
3860
+ const stem2 = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
3861
+ let title = fm.title ?? firstHeading(fm.body) ?? stem2;
3862
+ let ordinal = fm.order;
3863
+ let topicPath;
3864
+ const placement = placements.get(rel.normalize("NFC"));
3865
+ if (placement) {
3866
+ topicPath = ensureOutlineChain(placement.chain);
3867
+ if (placement.entry.title) title = placement.entry.title;
3868
+ if (placement.entry.order !== void 0) ordinal = placement.entry.order;
3869
+ } else {
3870
+ if (outline)
3871
+ warn(`${rel}: not in the ${outline.format} table of contents; filed under its folder`);
3872
+ topicPath = await ensureTopicChain(relDir);
3873
+ if (hugo && ordinal === void 0) ordinal = fm.weight;
3874
+ }
3875
+ if (hugo && stem2 === HUGO_SECTION_PAGE) {
3876
+ const override = {
3877
+ ...fm.title ? { name: fm.title } : {},
3878
+ ...fm.description ?? fm.summary ? { description: fm.description ?? fm.summary } : {},
3879
+ ...fm.weight !== void 0 ? { order: fm.weight } : {}
3880
+ };
3881
+ if (relDir === "") {
3882
+ rootOverride = override;
3883
+ } else {
3884
+ const leaf = topicById.get(topicPath[topicPath.length - 1]);
3885
+ applyOverride(leaf, override);
3886
+ applyOverride(leaf, opts.topics?.[relDir]);
3887
+ }
3888
+ if (!fm.body.trim()) continue;
3889
+ ordinal = SECTION_PAGE_ORDINAL;
3890
+ }
3891
+ if (fm.subcategory) {
3892
+ const parentId = topicPath[topicPath.length - 1];
3893
+ topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
3894
+ }
3895
+ const summary = fm.summary ?? (hugo ? fm.description : void 0) ?? firstParagraph(fm.body);
3896
+ loaded.push({
3897
+ rel,
3898
+ file: rel,
3899
+ doc: {
2001
3900
  id,
2002
3901
  title,
2003
3902
  slug: documentSlug(title),
@@ -2005,9 +3904,33 @@ async function loadMarkdownCatalog(rootDir, opts) {
2005
3904
  language: opts.language,
2006
3905
  topicPath,
2007
3906
  markdown: fm.body,
2008
- ...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {}
2009
- })
3907
+ ...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
3908
+ ...ordinal !== void 0 ? { ordinal } : {},
3909
+ ...Object.keys(fm.meta).length > 0 ? { meta: fm.meta } : {}
3910
+ }
3911
+ });
3912
+ }
3913
+ if (outline) {
3914
+ const titleByRel = new Map(
3915
+ loaded.map((entry) => [entry.rel.normalize("NFC"), entry.doc.title])
2010
3916
  );
3917
+ const nameFromPages = (topic) => {
3918
+ for (const child of topic.children) {
3919
+ const id = outlineTopicIds.get(child);
3920
+ const title = child.titleFromFile ? titleByRel.get(child.titleFromFile.normalize("NFC")) : void 0;
3921
+ const compiled = id ? topicById.get(id) : void 0;
3922
+ if (compiled && title) compiled.name = title;
3923
+ nameFromPages(child);
3924
+ }
3925
+ };
3926
+ nameFromPages(outline.root);
3927
+ for (const [file, placement] of placements) {
3928
+ if (!seenRel.has(file)) {
3929
+ warn(
3930
+ `${placement.entry.file}: named by the table of contents but not found among the Markdown files`
3931
+ );
3932
+ }
3933
+ }
2011
3934
  }
2012
3935
  if (rootTopicUsed) {
2013
3936
  if (takenTopicIds.has(rootTopicId)) {
@@ -2015,137 +3938,54 @@ async function loadMarkdownCatalog(rootDir, opts) {
2015
3938
  `root topic id '${rootTopicId}' collides with a folder-derived topic; pass rootTopicId`
2016
3939
  );
2017
3940
  }
2018
- topics.unshift({ id: rootTopicId, name: opts.rootTopicName ?? "General" });
3941
+ const root = { id: rootTopicId, name: opts.rootTopicName ?? "General" };
3942
+ applyOverride(root, await readTopicSidecar(rootDir));
3943
+ applyOverride(root, rootOverride);
3944
+ applyOverride(root, opts.topics?.[""]);
3945
+ topics.unshift(root);
3946
+ topicById.set(root.id, root);
2019
3947
  }
2020
- return { topics, documents };
3948
+ const assets = /* @__PURE__ */ new Map();
3949
+ const documents = [];
3950
+ for (const entry of loaded) {
3951
+ const ctx = {
3952
+ rootDir,
3953
+ docDir: entry.rel.includes("/") ? entry.rel.slice(0, entry.rel.lastIndexOf("/")) : "",
3954
+ file: entry.file,
3955
+ idByRel,
3956
+ uri: opts.uri,
3957
+ assets,
3958
+ missingAssets: opts.missingAssets ?? "error",
3959
+ warn
3960
+ };
3961
+ documents.push(
3962
+ CatalogDocumentSchema.parse({
3963
+ ...entry.doc,
3964
+ markdown: rewriteLinks(ctx, entry.doc.markdown)
3965
+ })
3966
+ );
3967
+ }
3968
+ return {
3969
+ topics,
3970
+ documents,
3971
+ assets: [...assets.values()].sort((a, b) => a.path < b.path ? -1 : 1),
3972
+ toc: { format: toc.format, ...outlinePath ? { path: outlinePath } : {} }
3973
+ };
2021
3974
  }
2022
3975
 
2023
3976
  // src/registry-client/fetch.ts
2024
- import { awakeTimeoutSignal } from "@bendyline/gezel";
2025
- import { KnowledgeRegistryIndexSchema } from "@bendyline/gezel";
3977
+ import { KnowledgeRegistryIndexSchema } from "@bendyline/gezk";
2026
3978
 
2027
3979
  // src/signatures/signing.ts
2028
3980
  import {
2029
- createHash as createHash5,
2030
- createPrivateKey,
2031
- createPublicKey,
2032
- sign as cryptoSign,
2033
- verify as cryptoVerify,
2034
- generateKeyPairSync
2035
- } from "crypto";
2036
-
2037
- // src/signatures/jcs.ts
2038
- function canonicalizeJson(value) {
2039
- return serialize(value, "$");
2040
- }
2041
- function serialize(value, path) {
2042
- if (value === null) return "null";
2043
- switch (typeof value) {
2044
- case "boolean":
2045
- return value ? "true" : "false";
2046
- case "number":
2047
- if (!Number.isFinite(value)) throw new Error(`non-finite number at ${path}`);
2048
- return JSON.stringify(value);
2049
- case "string":
2050
- return JSON.stringify(value);
2051
- case "object":
2052
- break;
2053
- default:
2054
- throw new Error(`value of type ${typeof value} at ${path} has no JSON identity`);
2055
- }
2056
- if (Array.isArray(value)) {
2057
- return `[${value.map((item, i) => serialize(item ?? null, `${path}[${i}]`)).join(",")}]`;
2058
- }
2059
- const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => compareUtf16(a, b));
2060
- return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${serialize(v, `${path}.${k}`)}`).join(",")}}`;
2061
- }
2062
- function compareUtf16(a, b) {
2063
- const n = Math.min(a.length, b.length);
2064
- for (let i = 0; i < n; i++) {
2065
- const d = a.charCodeAt(i) - b.charCodeAt(i);
2066
- if (d !== 0) return d;
2067
- }
2068
- return a.length - b.length;
2069
- }
2070
-
2071
- // src/signatures/signing.ts
2072
- function knowledgeKeyId(publicKeyPem) {
2073
- const der = createPublicKey(publicKeyPem).export({ type: "spki", format: "der" });
2074
- return createHash5("sha256").update(der).digest("hex").slice(0, 16);
2075
- }
2076
- function generateKnowledgeSigningKeyPair() {
2077
- const { publicKey, privateKey } = generateKeyPairSync("ed25519");
2078
- const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }).toString();
2079
- return {
2080
- publicKeyPem,
2081
- privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
2082
- keyId: knowledgeKeyId(publicKeyPem)
2083
- };
2084
- }
2085
- function manifestSigningPayload(manifest) {
2086
- const { signature: _omitted, ...unsigned } = manifest;
2087
- return Buffer.from(canonicalizeJson(unsigned), "utf8");
2088
- }
2089
- function signManifest(manifest, privateKeyPem) {
2090
- const key = createPrivateKey(privateKeyPem);
2091
- const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
2092
- const value = cryptoSign(null, manifestSigningPayload(manifest), key).toString("base64");
2093
- return {
2094
- ...manifest,
2095
- signature: {
2096
- algorithm: "ed25519",
2097
- keyId: knowledgeKeyId(publicKeyPem),
2098
- canonicalization: "rfc8785",
2099
- value
2100
- }
2101
- };
2102
- }
2103
- function verifyManifestSignature(manifest, anchors) {
2104
- return verifySignedDocument(manifest, anchors);
2105
- }
2106
- function signRegistryIndex(index, privateKeyPem) {
2107
- const key = createPrivateKey(privateKeyPem);
2108
- const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
2109
- const { signature: _omitted, ...unsigned } = index;
2110
- const value = cryptoSign(null, Buffer.from(canonicalizeJson(unsigned), "utf8"), key).toString(
2111
- "base64"
2112
- );
2113
- return {
2114
- ...index,
2115
- signature: {
2116
- algorithm: "ed25519",
2117
- keyId: knowledgeKeyId(publicKeyPem),
2118
- canonicalization: "rfc8785",
2119
- value
2120
- }
2121
- };
2122
- }
2123
- function verifyRegistryIndex(index, anchors) {
2124
- return verifySignedDocument(index, anchors);
2125
- }
2126
- function verifySignedDocument(document, anchors) {
2127
- const signature = document.signature;
2128
- if (!signature) return { ok: false, reason: "unsigned" };
2129
- const anchor = anchors.find((a) => a.keyId === signature.keyId);
2130
- if (!anchor) {
2131
- return { ok: false, reason: "unknown-key", detail: `no trust anchor for ${signature.keyId}` };
2132
- }
2133
- try {
2134
- if (knowledgeKeyId(anchor.publicKeyPem) !== signature.keyId) {
2135
- return { ok: false, reason: "unknown-key", detail: "anchor keyId does not match its key" };
2136
- }
2137
- const { signature: _omitted, ...unsigned } = document;
2138
- const ok = cryptoVerify(
2139
- null,
2140
- Buffer.from(canonicalizeJson(unsigned), "utf8"),
2141
- createPublicKey(anchor.publicKeyPem),
2142
- Buffer.from(signature.value, "base64")
2143
- );
2144
- return ok ? { ok: true, keyId: signature.keyId } : { ok: false, reason: "bad-signature" };
2145
- } catch (err) {
2146
- return { ok: false, reason: "error", detail: err instanceof Error ? err.message : String(err) };
2147
- }
2148
- }
3981
+ generateKnowledgeSigningKeyPair,
3982
+ knowledgeKeyId,
3983
+ manifestSigningPayload,
3984
+ signManifest,
3985
+ signRegistryIndex,
3986
+ verifyManifestSignature,
3987
+ verifyRegistryIndex
3988
+ } from "@bendyline/gezk/node";
2149
3989
 
2150
3990
  // src/registry-client/fetch.ts
2151
3991
  var DEFAULT_MAX_REGISTRY_BYTES = 4 * 1024 * 1024;
@@ -2164,7 +4004,7 @@ async function fetchKnowledgeRegistry(url, opts) {
2164
4004
  let response;
2165
4005
  try {
2166
4006
  response = await fetchImpl(url, {
2167
- signal: awakeTimeoutSignal(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
4007
+ signal: opts.signal ?? AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
2168
4008
  redirect: "follow",
2169
4009
  headers: { accept: "application/json" }
2170
4010
  });
@@ -2197,7 +4037,7 @@ async function fetchKnowledgeRegistry(url, opts) {
2197
4037
  } catch (err) {
2198
4038
  throw new KnowledgeRegistryFetchError(
2199
4039
  "invalid",
2200
- `registry is not a valid gezel-knowledge-registry document: ${err instanceof Error ? err.message : String(err)}`
4040
+ `registry is not a valid gezk-registry document: ${err instanceof Error ? err.message : String(err)}`
2201
4041
  );
2202
4042
  }
2203
4043
  const verdict = verifyRegistryIndex(registry, opts.anchors);
@@ -2243,7 +4083,11 @@ function newerRegistryEntries(registry, installed) {
2243
4083
 
2244
4084
  // src/signatures/anchors.ts
2245
4085
  var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
4086
+
4087
+ // src/signatures/jcs.ts
4088
+ import { canonicalizeJson as canonicalizeJson2 } from "@bendyline/gezk";
2246
4089
  export {
4090
+ BGE_SMALL_EN_V15_1,
2247
4091
  BODY_CODEC_MIN_BYTES,
2248
4092
  BUILTIN_KNOWLEDGE_TRUST_ANCHORS,
2249
4093
  CENTROID_CHUNKS_PER,
@@ -2252,19 +4096,35 @@ export {
2252
4096
  CENTROID_SAMPLE_MAX,
2253
4097
  CatalogHandle,
2254
4098
  CatalogOpenError,
4099
+ DuckdbError,
4100
+ EmbedderArtifactError,
2255
4101
  EmbedderUnavailableError,
2256
4102
  GEZEL_BGE_SMALL_EN_V15_1,
2257
4103
  GEZEL_MARKDOWN_CHUNKS_2,
2258
4104
  GEZEL_MULTILINGUAL_E5_SMALL_1,
2259
4105
  GEZK_APPLICATION_ID,
2260
4106
  GEZK_ARCHIVE_LIMITS,
4107
+ GEZK_FORMAT_GENERATIONS,
2261
4108
  GEZK_FORMAT_VERSION,
2262
4109
  GEZK_INDEX_SCHEMA_VERSION,
4110
+ GEZK_MIME_TYPE,
4111
+ GEZK_SUPPORTED_FORMAT_VERSIONS,
4112
+ GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
2263
4113
  GezkArchiveError,
2264
4114
  KNOWLEDGE_EMBEDDING_PROFILES,
4115
+ KNOWLEDGE_TOOLCHAIN,
2265
4116
  KnowledgeRegistryFetchError,
4117
+ LICENSE_NOTICE_PATH,
2266
4118
  MANIFEST_PATH,
4119
+ MARKDOWN_CHUNKS_2,
2267
4120
  MAX_KNOWLEDGE_DOCUMENT_BYTES,
4121
+ MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
4122
+ MAX_KNOWLEDGE_TOPIC_DEPTH,
4123
+ MIMETYPE_PATH,
4124
+ MULTILINGUAL_E5_SMALL_1,
4125
+ PARQUET_EXPORT_VERSION,
4126
+ PARQUET_REPORT_PATH,
4127
+ README_PATH,
2268
4128
  RERANK_FINAL_K,
2269
4129
  ROUTER_DB_PATH,
2270
4130
  ROUTER_DDL,
@@ -2273,37 +4133,58 @@ export {
2273
4133
  SHARD_DDL,
2274
4134
  SHARD_MAX_CHUNKS,
2275
4135
  SHARD_TARGET_CHUNKS,
4136
+ SOURCE_NOTICES_PATH,
2276
4137
  ZIP_FIXED_MTIME,
2277
- canonicalizeJson,
4138
+ assertDuckdbCli,
4139
+ assetReferences,
4140
+ assignOutlineOrders,
4141
+ canonicalizeJson2 as canonicalizeJson,
2278
4142
  chunkContentHash,
2279
4143
  chunkMarkdownProfile,
2280
4144
  chunkUid,
2281
4145
  compareCatalogVersions,
2282
4146
  compileKnowledgeCatalog,
2283
4147
  createProfileEmbedder,
4148
+ detectTableOfContents,
2284
4149
  documentSlug,
4150
+ duckdbVersion,
4151
+ exportCatalogParquet,
2285
4152
  extractGezkVerified,
2286
4153
  fetchKnowledgeRegistry,
4154
+ findDuckdbBinary,
2287
4155
  findRegistryEntry,
2288
4156
  generateKnowledgeSigningKeyPair,
4157
+ hammingTopK,
2289
4158
  inspectGezkArchive,
4159
+ isSupportedFormatVersion,
4160
+ isSupportedIndexSchemaVersion,
2290
4161
  knowledgeEmbeddingProfile,
2291
4162
  knowledgeKeyId,
2292
4163
  l2Normalize,
2293
4164
  loadMarkdownCatalog,
2294
4165
  manifestSigningPayload,
4166
+ newOutlineTopic,
2295
4167
  newerRegistryEntries,
2296
4168
  openCatalogDatabase,
4169
+ parseGitbookSummary,
4170
+ parseJupyterBookToc,
4171
+ parseMkdocsNav,
2297
4172
  quantizeBinary,
2298
4173
  quantizeInt8,
2299
4174
  readGezkManifest,
4175
+ readMkdocsDocsDir,
2300
4176
  rerankK,
2301
4177
  rerankScore,
4178
+ resolveOutlinePath,
4179
+ resolveTransformersModelOptions,
4180
+ runDuckdbScript,
2302
4181
  signManifest,
2303
4182
  signRegistryIndex,
4183
+ titleFromFolderName,
4184
+ transformersCachePath,
2304
4185
  validateExtractedCatalog,
2305
- vecChunksDdl,
2306
4186
  verifyManifestSignature,
4187
+ verifyProfileArtifacts,
2307
4188
  verifyRegistryIndex,
2308
4189
  writeGezkArchive
2309
4190
  };