@bendyline/gezel-knowledge 1.0.6 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +28 -17
  2. package/dist/index.d.ts +482 -137
  3. package/dist/index.js +1233 -397
  4. package/package.json +6 -5
package/dist/index.js CHANGED
@@ -1,13 +1,21 @@
1
1
  // src/format/constants.ts
2
- var GEZK_FORMAT_VERSION = 1;
3
- var GEZK_INDEX_SCHEMA_VERSION = 1;
4
- var GEZK_APPLICATION_ID = 1195727435;
5
- var ROUTER_DB_PATH = "index/router.db";
6
- var MANIFEST_PATH = "manifest.json";
2
+ import {
3
+ BODY_CODEC_MIN_BYTES,
4
+ GEZK_APPLICATION_ID,
5
+ GEZK_FORMAT_VERSION,
6
+ GEZK_INDEX_SCHEMA_VERSION,
7
+ GEZK_MIME_TYPE,
8
+ LICENSE_NOTICE_PATH,
9
+ MANIFEST_PATH,
10
+ MAX_KNOWLEDGE_DOCUMENT_BYTES,
11
+ MIMETYPE_PATH,
12
+ README_PATH,
13
+ ROUTER_DB_PATH,
14
+ SOURCE_NOTICES_PATH,
15
+ ZIP_FIXED_MTIME
16
+ } from "@bendyline/gezk";
7
17
  var SHARD_TARGET_CHUNKS = 2e5;
8
18
  var SHARD_MAX_CHUNKS = 25e4;
9
- var BODY_CODEC_MIN_BYTES = 512;
10
- var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
11
19
  var CENTROID_CHUNKS_PER = 12500;
12
20
  var CENTROID_MAX_PER_SHARD = 32;
13
21
  var CENTROID_KMEANS_MAX_ITER = 25;
@@ -18,144 +26,16 @@ var RERANK_FINAL_K = 24;
18
26
  function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
19
27
  return Math.min(512, Math.max(128, 8 * finalK), chunkCount);
20
28
  }
21
- var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
22
29
 
23
30
  // src/format/ddl.ts
24
- var ROUTER_DDL = `
25
- CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
26
-
27
- CREATE TABLE topics(
28
- id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,
29
- description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL
30
- );
31
-
32
- CREATE TABLE documents(
33
- id TEXT PRIMARY KEY,
34
- title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
35
- language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
36
- shard_id INTEGER NOT NULL,
37
- chunk_count INTEGER NOT NULL,
38
- source_url TEXT, source_revision TEXT, source_updated_at TEXT,
39
- attribution_json TEXT,
40
- body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
41
- body_blob BLOB NOT NULL
42
- );
43
- CREATE INDEX documents_topic ON documents(topic_id, slug);
44
- CREATE INDEX documents_shard ON documents(shard_id);
45
-
46
- CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
47
- PRIMARY KEY (alias, document_id)) WITHOUT ROWID;
48
-
49
- CREATE TABLE shards(
50
- id INTEGER PRIMARY KEY,
51
- path TEXT NOT NULL,
52
- chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,
53
- topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,
54
- bytes INTEGER NOT NULL
55
- );
56
-
57
- CREATE TABLE route_centroids(
58
- id INTEGER PRIMARY KEY,
59
- shard_id INTEGER NOT NULL REFERENCES shards(id),
60
- embedding BLOB NOT NULL,
61
- weight INTEGER NOT NULL
62
- );
63
- CREATE INDEX route_centroids_shard ON route_centroids(shard_id);
64
-
65
- CREATE VIRTUAL TABLE fts_documents USING fts5(
66
- title, summary, aliases, document_id UNINDEXED,
67
- tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
68
- );
69
- `;
70
- var SHARD_DDL = `
71
- CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
72
-
73
- CREATE TABLE chunks(
74
- id INTEGER PRIMARY KEY,
75
- chunk_uid TEXT NOT NULL,
76
- document_id TEXT NOT NULL,
77
- ordinal INTEGER NOT NULL,
78
- title TEXT NOT NULL,
79
- heading_path TEXT NOT NULL,
80
- heading_text TEXT NOT NULL,
81
- line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,
82
- token_count INTEGER NOT NULL,
83
- content_hash TEXT NOT NULL,
84
- text TEXT NOT NULL
85
- );
86
- CREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);
87
- CREATE INDEX chunks_document ON chunks(document_id, ordinal);
88
-
89
- CREATE VIRTUAL TABLE fts_chunks USING fts5(
90
- title, heading_text, text,
91
- content = 'chunks', content_rowid = 'id',
92
- tokenize = 'unicode61 remove_diacritics 2'
93
- );
94
-
95
- CREATE TABLE chunk_vectors_int8(
96
- chunk_id INTEGER PRIMARY KEY,
97
- v BLOB NOT NULL
98
- );
99
- `;
100
- function vecChunksDdl(dim) {
101
- return `CREATE VIRTUAL TABLE vec_chunks USING vec0(
102
- chunk_id INTEGER PRIMARY KEY,
103
- embedding bit[${dim}],
104
- chunk_size=1024
105
- );`;
106
- }
31
+ import { ROUTER_DDL, SHARD_DDL } from "@bendyline/gezk";
107
32
 
108
33
  // src/format/ids.ts
109
- import { createHash } from "crypto";
110
- function chunkUid(documentId, ordinal, chunkText) {
111
- const textHash = createHash("sha256").update(chunkText, "utf8").digest();
112
- return createHash("sha256").update(documentId, "utf8").update(Buffer.from([0])).update(String(ordinal), "utf8").update(Buffer.from([0])).update(textHash).digest("hex").slice(0, 32);
113
- }
114
- function chunkContentHash(chunkText) {
115
- return createHash("sha256").update(chunkText, "utf8").digest("hex");
116
- }
117
- function documentSlug(title) {
118
- const slug = title.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
119
- return slug || "untitled";
120
- }
34
+ import { documentSlug } from "@bendyline/gezk";
35
+ import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
121
36
 
122
37
  // src/format/quantize.ts
123
- function quantizeInt8(unitVector) {
124
- const out = new Int8Array(unitVector.length);
125
- for (let i = 0; i < unitVector.length; i++) {
126
- const q = Math.round(unitVector[i] * 127);
127
- out[i] = q > 127 ? 127 : q < -127 ? -127 : q;
128
- }
129
- return out;
130
- }
131
- function quantizeBinary(unitVector) {
132
- const bytes = Math.ceil(unitVector.length / 8);
133
- const out = new Uint8Array(bytes);
134
- for (let i = 0; i < unitVector.length; i++) {
135
- if (unitVector[i] > 0) {
136
- out[i >> 3] = out[i >> 3] | 1 << (i & 7);
137
- }
138
- }
139
- return out;
140
- }
141
- function rerankScore(query, passageInt8) {
142
- let dot = 0;
143
- const n = Math.min(query.length, passageInt8.length);
144
- for (let i = 0; i < n; i++) dot += query[i] * (passageInt8[i] / 127);
145
- return dot;
146
- }
147
- function l2Normalize(vector) {
148
- let sum = 0;
149
- for (let i = 0; i < vector.length; i++) {
150
- const v = vector[i];
151
- sum += v * v;
152
- }
153
- const norm = Math.sqrt(sum);
154
- if (!Number.isFinite(norm) || norm === 0) throw new Error("degenerate vector (zero norm)");
155
- const out = new Float32Array(vector.length);
156
- for (let i = 0; i < vector.length; i++) out[i] = vector[i] / norm;
157
- return out;
158
- }
38
+ import { l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from "@bendyline/gezk";
159
39
 
160
40
  // src/chunking/markdown-chunker.ts
161
41
  var HEADING = /^(#{1,6})\s+(.*)$/;
@@ -284,12 +164,12 @@ function splitOversizeLine(line, opts) {
284
164
  }
285
165
 
286
166
  // src/compiler/compile.ts
287
- import { createHash as createHash3 } from "crypto";
288
- import { mkdirSync, rmSync } from "fs";
167
+ import { createHash } from "crypto";
168
+ import { mkdirSync, rmSync, statSync } from "fs";
289
169
  import { stat } from "fs/promises";
290
170
  import { join } from "path";
291
171
  import { brotliCompressSync, constants as zlibConstants } from "zlib";
292
- import { KnowledgeCatalogManifestSchema } from "@bendyline/gezel";
172
+ import { GEZK_MANIFEST_KIND, KnowledgeCatalogManifestSchema } from "@bendyline/gezk";
293
173
 
294
174
  // src/archive/write.ts
295
175
  import { createWriteStream } from "fs";
@@ -301,9 +181,13 @@ async function writeGezkArchive(outputPath, entries) {
301
181
  const yazl = nodeRequire("yazl");
302
182
  await mkdir(dirname(outputPath), { recursive: true });
303
183
  const zip = new yazl.ZipFile();
304
- const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress: true };
184
+ const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress: false };
185
+ zip.addBuffer(Buffer.from(GEZK_MIME_TYPE, "utf8"), MIMETYPE_PATH, options);
305
186
  const sorted = [...entries].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
306
187
  for (const entry of sorted) {
188
+ if (entry.path === MIMETYPE_PATH) {
189
+ throw new Error(`the writer owns the ${MIMETYPE_PATH} entry`);
190
+ }
307
191
  if (entry.path.includes("\\") || entry.path.startsWith("/") || entry.path.includes("..")) {
308
192
  throw new Error(`unsafe archive path: ${entry.path}`);
309
193
  }
@@ -311,10 +195,10 @@ async function writeGezkArchive(outputPath, entries) {
311
195
  else if (entry.absPath) zip.addFile(entry.absPath, entry.path, options);
312
196
  else throw new Error(`archive entry ${entry.path} has neither content nor absPath`);
313
197
  }
314
- await new Promise((resolve3, reject) => {
198
+ await new Promise((resolve4, reject) => {
315
199
  const out = createWriteStream(outputPath);
316
200
  out.on("error", reject);
317
- out.on("close", () => resolve3());
201
+ out.on("close", () => resolve4());
318
202
  zip.outputStream.on("error", reject);
319
203
  zip.outputStream.pipe(out);
320
204
  zip.end();
@@ -322,27 +206,7 @@ async function writeGezkArchive(outputPath, entries) {
322
206
  }
323
207
 
324
208
  // src/format/file-hash.ts
325
- import { createHash as createHash2 } from "crypto";
326
- import { createReadStream } from "fs";
327
- async function hashFileStreaming(path, maxBytes = Number.MAX_SAFE_INTEGER) {
328
- const hasher = createHash2("sha256");
329
- let sizeBytes = 0;
330
- await new Promise((resolve3, reject) => {
331
- const stream = createReadStream(path, { highWaterMark: 4 * 1024 * 1024 });
332
- stream.on("data", (chunk) => {
333
- const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
334
- sizeBytes += bytes.byteLength;
335
- if (sizeBytes > maxBytes) {
336
- stream.destroy(new Error(`file exceeds ${maxBytes} byte limit`));
337
- return;
338
- }
339
- hasher.update(bytes);
340
- });
341
- stream.on("error", reject);
342
- stream.on("end", resolve3);
343
- });
344
- return { sha256: hasher.digest("hex"), sizeBytes };
345
- }
209
+ import { hashFileStreaming } from "@bendyline/gezk/node";
346
210
 
347
211
  // src/format/node-sqlite.ts
348
212
  import { createRequire as createRequire2 } from "module";
@@ -350,25 +214,34 @@ var nodeRequire2 = createRequire2(import.meta.url);
350
214
  var sqlite = nodeRequire2("node:sqlite");
351
215
  var DatabaseSync = sqlite.DatabaseSync;
352
216
 
353
- // src/compiler/vec-load.ts
217
+ // src/reader/fts-query.ts
218
+ function sanitizeFtsQuery(query) {
219
+ const tokens = [
220
+ ...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
221
+ ];
222
+ if (tokens.length === 0) return null;
223
+ return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
224
+ }
225
+ function documentFtsTopIds(db, match, limit) {
226
+ return db.prepare(
227
+ "SELECT document_id FROM fts_documents WHERE fts_documents MATCH ? ORDER BY rank LIMIT ?"
228
+ ).all(match, limit).map((row) => row.document_id);
229
+ }
230
+ var SMOKE_QUERY_TOP_N = 10;
231
+ function documentSmokeQueryMisses(db, smoke) {
232
+ const match = sanitizeFtsQuery(smoke.query);
233
+ if (!match) return [...smoke.expectedDocumentIds];
234
+ const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N);
235
+ return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
236
+ }
237
+
238
+ // src/toolchain.ts
354
239
  import { createRequire as createRequire3 } from "module";
355
240
  var nodeRequire3 = createRequire3(import.meta.url);
356
- function loadVecExtension(db) {
357
- try {
358
- db.enableLoadExtension(true);
359
- const sqliteVec = nodeRequire3("sqlite-vec");
360
- db.loadExtension(sqliteVec.getLoadablePath());
361
- } catch (err) {
362
- throw new Error(
363
- `sqlite-vec extension failed to load \u2014 the compiler cannot build vector tables: ${err instanceof Error ? err.message : String(err)}`
364
- );
365
- } finally {
366
- try {
367
- db.enableLoadExtension(false);
368
- } catch {
369
- }
370
- }
371
- }
241
+ var KNOWLEDGE_TOOLCHAIN = {
242
+ name: "@bendyline/gezel-knowledge",
243
+ version: nodeRequire3("../package.json").version
244
+ };
372
245
 
373
246
  // src/compiler/compile.ts
374
247
  async function compileKnowledgeCatalog(opts) {
@@ -380,9 +253,9 @@ async function compileKnowledgeCatalog(opts) {
380
253
  const topicIds = new Set(opts.topics.map((t) => t.id));
381
254
  const profile = opts.embeddingProfile;
382
255
  const chunkerOpts = {
383
- unit: "tokens",
384
- target: opts.chunkingProfile.targetTokens,
385
- overlap: opts.chunkingProfile.overlapTokens,
256
+ unit: opts.chunkingProfile.unit,
257
+ target: opts.chunkingProfile.target,
258
+ overlap: opts.chunkingProfile.overlap,
386
259
  countTokens: opts.countTokens
387
260
  };
388
261
  const prepared = [];
@@ -404,7 +277,8 @@ async function compileKnowledgeCatalog(opts) {
404
277
  (a, b) => a.topicPathKey === b.topicPathKey ? a.doc.id < b.doc.id ? -1 : a.doc.id > b.doc.id ? 1 : 0 : a.topicPathKey < b.topicPathKey ? -1 : 1
405
278
  );
406
279
  const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
407
- const embedded = totalChunks <= SHARD_TARGET_CHUNKS;
280
+ const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
281
+ const embedded = totalChunks <= shardTarget;
408
282
  const shardOf = /* @__PURE__ */ new Map();
409
283
  let shardCount = 1;
410
284
  if (embedded) {
@@ -413,7 +287,7 @@ async function compileKnowledgeCatalog(opts) {
413
287
  let current = 0;
414
288
  let filled = 0;
415
289
  for (const p of prepared) {
416
- if (filled > 0 && filled + p.chunks.length > SHARD_TARGET_CHUNKS) {
290
+ if (filled > 0 && filled + p.chunks.length > shardTarget) {
417
291
  current++;
418
292
  filled = 0;
419
293
  }
@@ -441,7 +315,6 @@ async function compileKnowledgeCatalog(opts) {
441
315
  router.exec("BEGIN");
442
316
  if (embedded) {
443
317
  router.exec(SHARD_DDL.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
444
- router.exec(vecChunksDdl(profile.dimensions));
445
318
  }
446
319
  const commonMeta = {
447
320
  format_version: String(GEZK_FORMAT_VERSION),
@@ -538,7 +411,6 @@ async function compileKnowledgeCatalog(opts) {
538
411
  if (!db) {
539
412
  db = createCatalogDb(join(opts.workDir, shardPath(id)));
540
413
  db.exec(SHARD_DDL);
541
- db.exec(vecChunksDdl(profile.dimensions));
542
414
  db.exec("BEGIN");
543
415
  shardDbs.set(id, db);
544
416
  }
@@ -549,7 +421,7 @@ async function compileKnowledgeCatalog(opts) {
549
421
  let stmts = vecStmts.get(db);
550
422
  if (!stmts) {
551
423
  stmts = {
552
- vec: db.prepare("INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, vec_bit(?))"),
424
+ vec: db.prepare("INSERT INTO chunk_vectors_bit (chunk_id, v) VALUES (?, ?)"),
553
425
  int8: db.prepare("INSERT INTO chunk_vectors_int8 (chunk_id, v) VALUES (?, ?)")
554
426
  };
555
427
  vecStmts.set(db, stmts);
@@ -628,7 +500,7 @@ async function compileKnowledgeCatalog(opts) {
628
500
  const header = buildContextHeader(
629
501
  p.doc.title,
630
502
  chunk.headingPath,
631
- opts.chunkingProfile.contextHeader.maxTokens,
503
+ opts.chunkingProfile.contextHeader.max,
632
504
  opts.countTokens
633
505
  );
634
506
  pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
@@ -637,6 +509,30 @@ async function compileKnowledgeCatalog(opts) {
637
509
  }
638
510
  }
639
511
  await flushEmbeds();
512
+ for (let id = 0; id < shardCount; id++) {
513
+ if (!embedded)
514
+ putMeta(shardDb(id), {
515
+ shard_id: String(id),
516
+ chunk_count: String(shardChunkCounts.get(id) ?? 0)
517
+ });
518
+ }
519
+ if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
520
+ const finalDir = join(opts.workDir, "final");
521
+ mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
522
+ const vacuumed = [];
523
+ const vacuumInto = (db, archivePath) => {
524
+ db.exec("COMMIT");
525
+ const dest = join(finalDir, archivePath);
526
+ db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
527
+ vacuumed.push({ archivePath, absPath: dest });
528
+ return dest;
529
+ };
530
+ const shardBytes = /* @__PURE__ */ new Map();
531
+ if (!embedded) {
532
+ for (let id = 0; id < shardCount; id++) {
533
+ shardBytes.set(id, statSync(vacuumInto(shardDb(id), shardPath(id))).size);
534
+ }
535
+ }
640
536
  {
641
537
  const insertShard = router.prepare(
642
538
  "INSERT INTO shards (id, path, chunk_count, document_count, topic_ids_json, centroid_count, bytes) VALUES (?, ?, ?, ?, ?, ?, ?)"
@@ -659,37 +555,45 @@ async function compileKnowledgeCatalog(opts) {
659
555
  BigInt(shardDocCounts.get(id) ?? 0),
660
556
  JSON.stringify([...shardTopicIds.get(id) ?? []].sort()),
661
557
  BigInt(centroids.length),
662
- BigInt(0)
663
- // authoritative sizes live in the manifest (post-VACUUM)
558
+ BigInt(shardBytes.get(id) ?? 0)
664
559
  );
665
560
  for (const c of centroids) {
666
561
  insertCentroid.run(BigInt(id), Buffer.from(c.centroid.buffer.slice(0)), BigInt(c.weight));
667
562
  }
668
563
  }
669
564
  }
670
- for (let id = 0; id < shardCount; id++) {
671
- if (!embedded)
672
- putMeta(shardDb(id), {
673
- shard_id: String(id),
674
- chunk_count: String(shardChunkCounts.get(id) ?? 0)
675
- });
676
- }
677
- if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
678
- const finalDir = join(opts.workDir, "final");
679
- mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
680
- const vacuumed = [];
681
- const vacuumInto = (db, archivePath) => {
682
- db.exec("COMMIT");
683
- const dest = join(finalDir, archivePath);
684
- db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
685
- vacuumed.push({ archivePath, absPath: dest });
686
- };
687
565
  vacuumInto(router, ROUTER_DB_PATH);
688
- if (!embedded) {
689
- for (let id = 0; id < shardCount; id++) vacuumInto(shardDb(id), shardPath(id));
690
- }
691
566
  router.close();
692
567
  for (const db of shardDbs.values()) db.close();
568
+ let smokeQueries = opts.smokeQueries;
569
+ if (smokeQueries && smokeQueries.length > 0) {
570
+ const routerFinal = new DatabaseSync(join(finalDir, ROUTER_DB_PATH), { readOnly: true });
571
+ try {
572
+ const verdicts = smokeQueries.map((smoke) => ({
573
+ smoke,
574
+ missing: documentSmokeQueryMisses(routerFinal, smoke)
575
+ }));
576
+ if ((opts.smokeQueryPolicy ?? "require") === "select") {
577
+ smokeQueries = verdicts.filter((v) => v.missing.length === 0).slice(0, 3).map((v) => v.smoke);
578
+ if (smokeQueries.length === 0) {
579
+ throw new Error(
580
+ `no smoke-query candidate passes against the built index (tried ${verdicts.length}: ${verdicts.map((v) => `"${v.smoke.query}"`).join(", ")}) \u2014 the fts_documents index cannot surface any candidate's expected document in the top ${SMOKE_QUERY_TOP_N}`
581
+ );
582
+ }
583
+ } else {
584
+ const failed = verdicts.filter((v) => v.missing.length > 0);
585
+ if (failed.length > 0) {
586
+ throw new Error(
587
+ `smoke queries failed against the built index: ${failed.map((v) => `"${v.smoke.query}" (missing ${v.missing.join(", ")})`).join(
588
+ "; "
589
+ )} \u2014 every recorded sanity query must return its expected documents in the top ${SMOKE_QUERY_TOP_N} fts_documents results; pick more specific queries (distinctive titles beat common words in the rank race) or pass smokeQueryPolicy: 'select' with candidates`
590
+ );
591
+ }
592
+ }
593
+ } finally {
594
+ routerFinal.close();
595
+ }
596
+ }
693
597
  const files = [];
694
598
  const routerShardStats = [];
695
599
  for (const f of vacuumed) {
@@ -714,17 +618,21 @@ async function compileKnowledgeCatalog(opts) {
714
618
  });
715
619
  }
716
620
  }
717
- const extraFiles = opts.extraFiles ?? {};
621
+ const extraFiles = {
622
+ [README_PATH]: defaultReadme(opts),
623
+ [LICENSE_NOTICE_PATH]: defaultLicenseNotice(opts),
624
+ ...opts.extraFiles ?? {}
625
+ };
718
626
  for (const [path, content] of Object.entries(extraFiles)) {
719
627
  const bytes = Buffer.from(content, "utf8");
720
628
  files.push({
721
629
  path,
722
630
  sizeBytes: bytes.length,
723
- sha256: createHash3("sha256").update(bytes).digest("hex")
631
+ sha256: createHash("sha256").update(bytes).digest("hex")
724
632
  });
725
633
  }
726
634
  const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
727
- kind: "gezel-knowledge-catalog",
635
+ kind: GEZK_MANIFEST_KIND,
728
636
  formatVersion: GEZK_FORMAT_VERSION,
729
637
  indexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION,
730
638
  id: opts.catalog.id,
@@ -735,20 +643,23 @@ async function compileKnowledgeCatalog(opts) {
735
643
  publisher: opts.catalog.publisher,
736
644
  createdAt: opts.catalog.createdAt,
737
645
  ...opts.catalog.sourceSnapshot ? { sourceSnapshot: opts.catalog.sourceSnapshot } : {},
738
- license: opts.catalog.license,
646
+ license: {
647
+ ...opts.catalog.license,
648
+ noticePath: opts.catalog.license.noticePath ?? LICENSE_NOTICE_PATH
649
+ },
739
650
  embedding: profile,
740
651
  chunking: opts.chunkingProfile,
741
652
  topics: opts.topics.map((t) => ({ id: t.id, name: t.name })),
742
653
  router: {
743
- shardTargetChunks: SHARD_TARGET_CHUNKS,
654
+ shardTargetChunks: shardTarget,
744
655
  shards: routerShardStats.sort((a, b) => a.id - b.id),
745
656
  totalCentroids: routerShardStats.reduce((sum, s) => sum + s.centroids, 0)
746
657
  },
747
658
  counts: { documents: prepared.length, chunks: totalChunks, shards: shardCount },
748
659
  files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
749
- compatibility: { maximumIndexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION },
750
- ...opts.smokeQueries ? { smokeQueries: opts.smokeQueries } : {},
751
- ...opts.toolchain ? { toolchain: opts.toolchain } : {}
660
+ requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
661
+ ...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
662
+ toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
752
663
  });
753
664
  const manifest = opts.finalizeManifest ? KnowledgeCatalogManifestSchema.parse(opts.finalizeManifest(unsignedManifest)) : unsignedManifest;
754
665
  const archiveEntries = [
@@ -778,14 +689,37 @@ async function compileKnowledgeCatalog(opts) {
778
689
  }
779
690
  }
780
691
  function createCatalogDb(absPath) {
781
- const db = new DatabaseSync(absPath, { allowExtension: true });
782
- loadVecExtension(db);
692
+ const db = new DatabaseSync(absPath);
783
693
  db.exec("PRAGMA page_size=8192");
784
694
  db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
785
695
  db.exec(`PRAGMA user_version=${GEZK_INDEX_SCHEMA_VERSION}`);
786
696
  db.exec("PRAGMA journal_mode=DELETE");
787
697
  return db;
788
698
  }
699
+ function defaultReadme(opts) {
700
+ const c = opts.catalog;
701
+ return [
702
+ `# ${c.name}`,
703
+ "",
704
+ ...c.description ? [c.description, ""] : [],
705
+ `A gezk knowledge catalog (${c.id}@${c.version}) published by ${c.publisher.name}.`,
706
+ `Language: ${c.language}. License: ${c.license.name} (see ${c.license.noticePath ?? LICENSE_NOTICE_PATH}).`,
707
+ ...c.sourceSnapshot ? [`Source snapshot: ${c.sourceSnapshot.name} (${c.sourceSnapshot.date}).`] : [],
708
+ ""
709
+ ].join("\n");
710
+ }
711
+ function defaultLicenseNotice(opts) {
712
+ const c = opts.catalog;
713
+ const publisher = c.publisher.url ? `${c.publisher.name} <${c.publisher.url}>` : c.publisher.name;
714
+ const license = c.license.spdx ? `${c.license.name} (SPDX: ${c.license.spdx})` : c.license.name;
715
+ return [
716
+ `${c.name} (${c.id}@${c.version})`,
717
+ `Publisher: ${publisher}`,
718
+ `License: ${license}`,
719
+ ...c.license.attributionRequired ? ["Attribution is required when redistributing this content."] : [],
720
+ ""
721
+ ].join("\n");
722
+ }
789
723
  function normalizeMarkdown(md) {
790
724
  return md.replace(/\r\n/g, "\n").normalize("NFC");
791
725
  }
@@ -803,7 +737,7 @@ ${path.join(" > ")}
803
737
  return header;
804
738
  }
805
739
  function seedFor(catalogId, shardId) {
806
- const digest = createHash3("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
740
+ const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
807
741
  return digest.readBigUInt64LE(24);
808
742
  }
809
743
  function makePrng(seed) {
@@ -888,14 +822,14 @@ function kMeans(samples, k, seed) {
888
822
  }
889
823
 
890
824
  // src/archive/read.ts
891
- import { createHash as createHash4 } from "crypto";
825
+ import { createHash as createHash2 } from "crypto";
892
826
  import { createWriteStream as createWriteStream2 } from "fs";
893
827
  import { mkdir as mkdir2 } from "fs/promises";
894
828
  import { createRequire as createRequire4 } from "module";
895
829
  import { dirname as dirname2, isAbsolute, normalize, relative, resolve } from "path";
896
830
  import { Transform } from "stream";
897
831
  import { pipeline } from "stream/promises";
898
- import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/gezel";
832
+ import { GEZK_MANIFEST_KIND as GEZK_MANIFEST_KIND2, KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/gezk";
899
833
  var nodeRequire4 = createRequire4(import.meta.url);
900
834
  var GEZK_ARCHIVE_LIMITS = {
901
835
  maxEntries: 16384,
@@ -985,7 +919,7 @@ async function inspectEntries(archivePath) {
985
919
  function reconcileEntries(manifest, entries) {
986
920
  const declared = new Map(manifest.files.map((file) => [file.path, file]));
987
921
  for (const path of entries.keys()) {
988
- if (path === MANIFEST_PATH) continue;
922
+ if (path === MANIFEST_PATH || path === MIMETYPE_PATH) continue;
989
923
  if (!declared.has(path)) {
990
924
  throw new GezkArchiveError(`archive contains undeclared file: ${path}`, "undeclared-file");
991
925
  }
@@ -1015,14 +949,88 @@ async function inspectGezkArchive(archivePath) {
1015
949
  totalUncompressedBytes: inspected.totalUncompressedBytes
1016
950
  };
1017
951
  }
952
+ function assertMimetypeEntry(entry, position) {
953
+ if (position !== 0) {
954
+ throw new GezkArchiveError("the mimetype entry must be the first archive entry", "mimetype");
955
+ }
956
+ if (entry.compressionMethod !== 0 || entry.uncompressedSize !== GEZK_MIME_TYPE.length) {
957
+ throw new GezkArchiveError("the mimetype entry must be stored, not compressed", "mimetype");
958
+ }
959
+ }
960
+ function assertSupportedFormat(raw) {
961
+ const kind = raw?.kind;
962
+ const version = raw?.formatVersion;
963
+ if (kind !== GEZK_MANIFEST_KIND2 || version !== GEZK_FORMAT_VERSION) {
964
+ throw new GezkArchiveError(
965
+ `unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${GEZK_FORMAT_VERSION} \u2014 catalogs built for an earlier version must be rebuilt`,
966
+ "format-version"
967
+ );
968
+ }
969
+ }
970
+ function readEntryText(zip, entry) {
971
+ return new Promise((resolveText, rejectText) => {
972
+ zip.openReadStream(entry, (err, stream) => {
973
+ if (err || !stream) {
974
+ rejectText(new GezkArchiveError(`${entry.fileName} read failed: ${err?.message}`, "io"));
975
+ return;
976
+ }
977
+ const parts = [];
978
+ stream.on("data", (d) => parts.push(d));
979
+ stream.on("end", () => resolveText(Buffer.concat(parts).toString("utf8")));
980
+ stream.on(
981
+ "error",
982
+ (streamErr) => rejectText(
983
+ new GezkArchiveError(`${entry.fileName} read failed: ${streamErr.message}`, "io")
984
+ )
985
+ );
986
+ });
987
+ });
988
+ }
1018
989
  async function readGezkManifest(archivePath) {
1019
990
  const zip = await openZip(archivePath);
1020
991
  return new Promise((resolvePromise, reject) => {
992
+ let position = 0;
993
+ let mimetypeSeen = false;
994
+ const fail = (err) => {
995
+ zip.close();
996
+ reject(
997
+ err instanceof GezkArchiveError ? err : new GezkArchiveError(err instanceof Error ? err.message : String(err), "io")
998
+ );
999
+ };
1021
1000
  zip.on("entry", (entry) => {
1001
+ const index = position++;
1002
+ if (entry.fileName === MIMETYPE_PATH) {
1003
+ try {
1004
+ assertMimetypeEntry(entry, index);
1005
+ } catch (err) {
1006
+ fail(err);
1007
+ return;
1008
+ }
1009
+ readEntryText(zip, entry).then((text) => {
1010
+ if (text !== GEZK_MIME_TYPE) {
1011
+ fail(
1012
+ new GezkArchiveError(
1013
+ `not a gezk archive (mimetype ${JSON.stringify(text)})`,
1014
+ "mimetype"
1015
+ )
1016
+ );
1017
+ return;
1018
+ }
1019
+ mimetypeSeen = true;
1020
+ zip.readEntry();
1021
+ }, fail);
1022
+ return;
1023
+ }
1022
1024
  if (entry.fileName !== MANIFEST_PATH) {
1023
1025
  zip.readEntry();
1024
1026
  return;
1025
1027
  }
1028
+ if (!mimetypeSeen) {
1029
+ fail(
1030
+ new GezkArchiveError("archive does not start with the gezk mimetype entry", "mimetype")
1031
+ );
1032
+ return;
1033
+ }
1026
1034
  if (entry.uncompressedSize > GEZK_ARCHIVE_LIMITS.maxManifestBytes) {
1027
1035
  zip.close();
1028
1036
  reject(new GezkArchiveError("manifest exceeds size limit", "limits"));
@@ -1039,12 +1047,14 @@ async function readGezkManifest(archivePath) {
1039
1047
  stream.on("end", () => {
1040
1048
  zip.close();
1041
1049
  try {
1042
- resolvePromise(
1043
- KnowledgeCatalogManifestSchema2.parse(
1044
- JSON.parse(Buffer.concat(parts).toString("utf8"))
1045
- )
1046
- );
1050
+ const raw = JSON.parse(Buffer.concat(parts).toString("utf8"));
1051
+ assertSupportedFormat(raw);
1052
+ resolvePromise(KnowledgeCatalogManifestSchema2.parse(raw));
1047
1053
  } catch (parseErr) {
1054
+ if (parseErr instanceof GezkArchiveError) {
1055
+ reject(parseErr);
1056
+ return;
1057
+ }
1048
1058
  reject(
1049
1059
  new GezkArchiveError(
1050
1060
  `invalid manifest: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,
@@ -1102,7 +1112,7 @@ async function extractGezkVerified(archivePath, destDir) {
1102
1112
  pipeline(stream, createWriteStream2(target)).then(res, rej);
1103
1113
  return;
1104
1114
  }
1105
- const hasher = createHash4("sha256");
1115
+ const hasher = createHash2("sha256");
1106
1116
  let sizeBytes = 0;
1107
1117
  const verify = new Transform({
1108
1118
  transform(chunk, _encoding, callback) {
@@ -1163,9 +1173,7 @@ async function extractGezkVerified(archivePath, destDir) {
1163
1173
  }
1164
1174
 
1165
1175
  // src/reader/open.ts
1166
- import { createRequire as createRequire5 } from "module";
1167
1176
  import { pathToFileURL } from "url";
1168
- var nodeRequire5 = createRequire5(import.meta.url);
1169
1177
  var CatalogOpenError = class extends Error {
1170
1178
  constructor(message, reason) {
1171
1179
  super(message);
@@ -1178,7 +1186,7 @@ function openCatalogDatabase(absPath) {
1178
1186
  const uri = `${pathToFileURL(absPath).href}?immutable=1`;
1179
1187
  let db;
1180
1188
  try {
1181
- db = new DatabaseSync(uri, { readOnly: true, allowExtension: true });
1189
+ db = new DatabaseSync(uri, { readOnly: true });
1182
1190
  } catch (err) {
1183
1191
  throw new CatalogOpenError(
1184
1192
  `cannot open catalog database ${absPath}: ${err instanceof Error ? err.message : String(err)}`,
@@ -1186,20 +1194,6 @@ function openCatalogDatabase(absPath) {
1186
1194
  );
1187
1195
  }
1188
1196
  try {
1189
- let vecLoaded = false;
1190
- try {
1191
- db.enableLoadExtension(true);
1192
- const sqliteVec = nodeRequire5("sqlite-vec");
1193
- db.loadExtension(sqliteVec.getLoadablePath());
1194
- vecLoaded = true;
1195
- } catch {
1196
- vecLoaded = false;
1197
- } finally {
1198
- try {
1199
- db.enableLoadExtension(false);
1200
- } catch {
1201
- }
1202
- }
1203
1197
  const appId = readPragmaNumber(db, "application_id");
1204
1198
  if (appId !== GEZK_APPLICATION_ID) {
1205
1199
  throw new CatalogOpenError(
@@ -1210,13 +1204,10 @@ function openCatalogDatabase(absPath) {
1210
1204
  const schemaVersion = readPragmaNumber(db, "user_version");
1211
1205
  if (schemaVersion !== GEZK_INDEX_SCHEMA_VERSION) {
1212
1206
  throw new CatalogOpenError(
1213
- `unsupported index schema version ${schemaVersion} (reader supports ${GEZK_INDEX_SCHEMA_VERSION})`,
1207
+ `unsupported index schema version ${schemaVersion} (this reader supports ${GEZK_INDEX_SCHEMA_VERSION}; catalogs built for gezk 0.4 and earlier must be rebuilt)`,
1214
1208
  "schema-version"
1215
1209
  );
1216
1210
  }
1217
- if (!vecLoaded) {
1218
- throw new CatalogOpenError("sqlite-vec extension failed to load", "vec-unavailable");
1219
- }
1220
1211
  db.exec("PRAGMA mmap_size=536870912");
1221
1212
  db.exec("PRAGMA cache_size=-8192");
1222
1213
  return {
@@ -1245,6 +1236,80 @@ function readPragmaNumber(db, pragma) {
1245
1236
  // src/reader/catalog-handle.ts
1246
1237
  import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1247
1238
  import { brotliDecompressSync } from "zlib";
1239
+
1240
+ // src/reader/bit-scan.ts
1241
+ var POPCOUNT8 = new Uint8Array(256);
1242
+ for (let i = 1; i < 256; i++) POPCOUNT8[i] = (i & 1) + POPCOUNT8[i >> 1];
1243
+ function popcount32(x) {
1244
+ let v = x - (x >>> 1 & 1431655765);
1245
+ v = (v & 858993459) + (v >>> 2 & 858993459);
1246
+ return Math.imul(v + (v >>> 4) & 252645135, 16843009) >>> 24;
1247
+ }
1248
+ function hammingAll(index, query, out) {
1249
+ const { bits, bytesPerRow, rows } = index;
1250
+ if (bytesPerRow % 4 === 0 && bits.byteOffset % 4 === 0) {
1251
+ const perRow = bytesPerRow / 4;
1252
+ const words = new Uint32Array(bits.buffer, bits.byteOffset, rows * perRow);
1253
+ const aligned = new Uint8Array(bytesPerRow);
1254
+ aligned.set(query);
1255
+ const qwords = new Uint32Array(aligned.buffer, 0, perRow);
1256
+ let offset2 = 0;
1257
+ for (let r = 0; r < rows; r++) {
1258
+ let d = 0;
1259
+ for (let w = 0; w < perRow; w++) {
1260
+ d += popcount32((words[offset2 + w] ^ qwords[w]) >>> 0);
1261
+ }
1262
+ out[r] = d;
1263
+ offset2 += perRow;
1264
+ }
1265
+ return;
1266
+ }
1267
+ let offset = 0;
1268
+ for (let r = 0; r < rows; r++) {
1269
+ let d = 0;
1270
+ for (let b = 0; b < bytesPerRow; b++) {
1271
+ d += POPCOUNT8[bits[offset + b] ^ query[b]];
1272
+ }
1273
+ out[r] = d;
1274
+ offset += bytesPerRow;
1275
+ }
1276
+ }
1277
+ function hammingTopK(index, query, k) {
1278
+ if (query.length !== index.bytesPerRow) {
1279
+ throw new Error(`query has ${query.length} bytes, shard rows have ${index.bytesPerRow}`);
1280
+ }
1281
+ const limit = Math.min(k, index.rows);
1282
+ if (limit <= 0) return [];
1283
+ const distances = new Uint16Array(index.rows);
1284
+ hammingAll(index, query, distances);
1285
+ const maxDistance = index.bytesPerRow * 8;
1286
+ const histogram = new Uint32Array(maxDistance + 1);
1287
+ for (let r = 0; r < index.rows; r++) {
1288
+ const d = distances[r];
1289
+ histogram[d] = histogram[d] + 1;
1290
+ }
1291
+ let threshold = 0;
1292
+ let seen = 0;
1293
+ for (; threshold <= maxDistance; threshold++) {
1294
+ seen += histogram[threshold];
1295
+ if (seen >= limit) break;
1296
+ }
1297
+ let atThreshold = limit - (seen - histogram[threshold]);
1298
+ const hits = [];
1299
+ for (let r = 0; r < index.rows && hits.length < limit; r++) {
1300
+ const d = distances[r];
1301
+ if (d < threshold) hits.push({ chunkId: r + 1, distance: d });
1302
+ else if (d === threshold && atThreshold > 0) {
1303
+ hits.push({ chunkId: r + 1, distance: d });
1304
+ atThreshold--;
1305
+ }
1306
+ }
1307
+ hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
1308
+ return hits;
1309
+ }
1310
+
1311
+ // src/reader/catalog-handle.ts
1312
+ var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
1248
1313
  var CatalogHandle = class _CatalogHandle {
1249
1314
  constructor(rootDir, router) {
1250
1315
  this.rootDir = rootDir;
@@ -1259,6 +1324,7 @@ var CatalogHandle = class _CatalogHandle {
1259
1324
  rootDir;
1260
1325
  router;
1261
1326
  connections = /* @__PURE__ */ new Map();
1327
+ bitIndexes = /* @__PURE__ */ new Map();
1262
1328
  meta;
1263
1329
  shards;
1264
1330
  /** Open a catalog's extracted version directory (the dir holding manifest.json). */
@@ -1269,6 +1335,7 @@ var CatalogHandle = class _CatalogHandle {
1269
1335
  close() {
1270
1336
  for (const conn of this.connections.values()) conn.close();
1271
1337
  this.connections.clear();
1338
+ this.bitIndexes.clear();
1272
1339
  }
1273
1340
  /** Integrity check for the router connection, used by deep validation. */
1274
1341
  routerQuickCheck() {
@@ -1299,6 +1366,57 @@ var CatalogHandle = class _CatalogHandle {
1299
1366
  }
1300
1367
  return conn.db;
1301
1368
  }
1369
+ /**
1370
+ * A shard's sign-bit rows as one contiguous array, loaded on first use and
1371
+ * kept while the resident budget allows. Chunk ids must be dense from 1 —
1372
+ * the row index IS the id — so a gap is a corrupt shard, not a sparse one.
1373
+ */
1374
+ shardBits(shard) {
1375
+ const cached = this.bitIndexes.get(shard.path);
1376
+ if (cached) return cached;
1377
+ const db = this.shardDb(shard);
1378
+ const span = db.prepare(
1379
+ "SELECT COUNT(*) AS n, MIN(chunk_id) AS lo, MAX(chunk_id) AS hi FROM chunk_vectors_bit"
1380
+ ).get();
1381
+ const rows = Number(span.n);
1382
+ if (rows > 0 && (Number(span.lo) !== 1 || Number(span.hi) !== rows)) {
1383
+ throw new CatalogOpenError(`chunk ids are not dense in ${shard.path}`, "corrupt");
1384
+ }
1385
+ const bytesPerRow = Math.ceil(this.dimensions() / 8);
1386
+ const bits = new Uint8Array(rows * bytesPerRow);
1387
+ const rowsIter = db.prepare("SELECT chunk_id, v FROM chunk_vectors_bit ORDER BY chunk_id").iterate();
1388
+ for (const row of rowsIter) {
1389
+ if (row.v.byteLength !== bytesPerRow) {
1390
+ throw new CatalogOpenError(
1391
+ `bit vector width ${row.v.byteLength} != ${bytesPerRow} in ${shard.path}`,
1392
+ "corrupt"
1393
+ );
1394
+ }
1395
+ bits.set(row.v, (Number(row.chunk_id) - 1) * bytesPerRow);
1396
+ }
1397
+ const index = { bits, bytesPerRow, rows };
1398
+ let resident = bits.byteLength;
1399
+ for (const other of this.bitIndexes.values()) resident += other.bits.byteLength;
1400
+ for (const [key, other] of this.bitIndexes) {
1401
+ if (resident <= BIT_INDEX_BUDGET_BYTES) break;
1402
+ this.bitIndexes.delete(key);
1403
+ resident -= other.bits.byteLength;
1404
+ }
1405
+ this.bitIndexes.set(shard.path, index);
1406
+ return index;
1407
+ }
1408
+ /** The embedding dimension from the router's profile echo. */
1409
+ dimensions() {
1410
+ const raw = this.meta.embedding_profile_json;
1411
+ if (raw) {
1412
+ try {
1413
+ const dims = JSON.parse(raw).dimensions;
1414
+ if (typeof dims === "number" && dims > 0) return dims;
1415
+ } catch {
1416
+ }
1417
+ }
1418
+ throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
1419
+ }
1302
1420
  /** Resolve a manifest/router path without allowing it to escape the catalog. */
1303
1421
  resolveCatalogPath(path) {
1304
1422
  const root = resolve2(this.rootDir);
@@ -1375,13 +1493,26 @@ var CatalogHandle = class _CatalogHandle {
1375
1493
  const match = ftsQuery(query);
1376
1494
  if (!match) return [];
1377
1495
  try {
1378
- return this.router.db.prepare(
1379
- "SELECT document_id FROM fts_documents WHERE fts_documents MATCH ? ORDER BY rank LIMIT ?"
1380
- ).all(match, limit).map((r, i) => ({ documentId: r.document_id, rank: i }));
1496
+ return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
1497
+ documentId,
1498
+ rank: i
1499
+ }));
1381
1500
  } catch {
1382
1501
  return [];
1383
1502
  }
1384
1503
  }
1504
+ /**
1505
+ * Router-level shape stats: topic-tree size and total route centroids.
1506
+ * Read-only counts for inspectors and recall/routing diagnostics — a
1507
+ * single-topic catalog fills shards in documentId order, which is the
1508
+ * first thing to check when centroid routing underperforms.
1509
+ */
1510
+ routerStats() {
1511
+ const count = (table) => Number(
1512
+ this.router.db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get().c
1513
+ );
1514
+ return { topics: count("topics"), routeCentroids: count("route_centroids") };
1515
+ }
1385
1516
  /**
1386
1517
  * Score every shard against the (unit float32) query: shard score = max
1387
1518
  * cosine over its centroids (§3.4). This is the cross-catalog routing
@@ -1423,16 +1554,16 @@ var CatalogHandle = class _CatalogHandle {
1423
1554
  * after global cross-catalog routing has already spent the S budget.
1424
1555
  */
1425
1556
  searchShards(queryVector, shardIds, finalK) {
1426
- const queryBits = Buffer.from(quantizeBinary(queryVector));
1557
+ const queryBits = quantizeBinary(queryVector);
1427
1558
  const hits = [];
1428
1559
  for (const shardId of shardIds) {
1429
1560
  const shard = this.shards.find((s) => s.id === shardId);
1430
1561
  if (!shard) continue;
1431
1562
  const db = this.shardDb(shard);
1432
1563
  const k = rerankK(finalK, shard.chunkCount);
1433
- const candidates = db.prepare(
1434
- "SELECT chunk_id FROM vec_chunks WHERE embedding MATCH vec_bit(?) AND k = ? ORDER BY distance"
1435
- ).all(queryBits, BigInt(k));
1564
+ const candidates = hammingTopK(this.shardBits(shard), queryBits, k).map((hit) => ({
1565
+ chunk_id: hit.chunkId
1566
+ }));
1436
1567
  if (candidates.length === 0) continue;
1437
1568
  const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
1438
1569
  const reranked = candidates.map((c) => {
@@ -1499,21 +1630,19 @@ var CatalogHandle = class _CatalogHandle {
1499
1630
  return hits;
1500
1631
  }
1501
1632
  /**
1502
- * Embedder-free integrity smoke (§6.3): chunk 1's own bit vector must be
1503
- * its own nearest neighbor — proves the extension parses this file's vec
1504
- * blobs. Returns false instead of throwing (callers report a reason).
1633
+ * Embedder-free integrity smoke: chunk 1's own bit vector must be its own
1634
+ * nearest neighbor — proves the shard's vectors load and scan. Returns
1635
+ * false instead of throwing (callers report a reason).
1505
1636
  */
1506
1637
  selfKnnSmoke(shardId) {
1507
1638
  const shard = this.shards.find((s) => s.id === shardId);
1508
1639
  if (!shard) return false;
1509
1640
  try {
1510
- const db = this.shardDb(shard);
1511
- const first = db.prepare("SELECT embedding FROM vec_chunks WHERE chunk_id = 1").get();
1512
- if (!first) return false;
1513
- const nearest = db.prepare(
1514
- "SELECT chunk_id FROM vec_chunks WHERE embedding MATCH vec_bit(?) AND k = 1 ORDER BY distance"
1515
- ).get(Buffer.from(first.embedding));
1516
- return nearest !== void 0 && Number(nearest.chunk_id) === 1;
1641
+ const index = this.shardBits(shard);
1642
+ if (index.rows === 0) return false;
1643
+ const first = index.bits.subarray(0, index.bytesPerRow);
1644
+ const nearest = hammingTopK(index, first, 1)[0];
1645
+ return nearest !== void 0 && nearest.chunkId === 1 && nearest.distance === 0;
1517
1646
  } catch {
1518
1647
  return false;
1519
1648
  }
@@ -1549,18 +1678,12 @@ function parseHeadingPath(raw) {
1549
1678
  return [];
1550
1679
  }
1551
1680
  }
1552
- function ftsQuery(query) {
1553
- const tokens = [
1554
- ...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
1555
- ];
1556
- if (tokens.length === 0) return null;
1557
- return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
1558
- }
1681
+ var ftsQuery = sanitizeFtsQuery;
1559
1682
 
1560
1683
  // src/reader/validate.ts
1561
1684
  import { readFile } from "fs/promises";
1562
1685
  import { join as join3 } from "path";
1563
- import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/gezel";
1686
+ import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/gezk";
1564
1687
  async function validateExtractedCatalog(rootDir, opts = {}) {
1565
1688
  const checks = [];
1566
1689
  const check = (name, ok, detail) => {
@@ -1629,6 +1752,11 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1629
1752
  );
1630
1753
  const topics = handle.topics();
1631
1754
  check("toc-present", topics.length >= 1, `${topics.length} topics`);
1755
+ check(
1756
+ "license-notice",
1757
+ manifest.files.some((f) => f.path === manifest.license.noticePath),
1758
+ `manifest.files lacks the declared notice ${manifest.license.noticePath}`
1759
+ );
1632
1760
  const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
1633
1761
  check(
1634
1762
  "counts-documents",
@@ -1679,7 +1807,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1679
1807
  db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n
1680
1808
  );
1681
1809
  const vecCount = Number(
1682
- db.prepare("SELECT COUNT(*) AS n FROM vec_chunks").get().n
1810
+ db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit").get().n
1683
1811
  );
1684
1812
  const int8Count = Number(
1685
1813
  db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8").get().n
@@ -1689,18 +1817,36 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1689
1817
  chunkCount === shard.chunkCount && vecCount === chunkCount && int8Count === chunkCount,
1690
1818
  `chunks ${chunkCount}, vec ${vecCount}, int8 ${int8Count}, shards row ${shard.chunkCount}`
1691
1819
  );
1820
+ const dims = manifest.embedding.dimensions;
1821
+ const badBit = Number(
1822
+ db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit WHERE length(v) != ?").get(Math.ceil(dims / 8)).n
1823
+ );
1824
+ const badInt8 = Number(
1825
+ db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8 WHERE length(v) != ?").get(dims).n
1826
+ );
1827
+ check(
1828
+ `vector-widths:${shard.path}`,
1829
+ badBit === 0 && badInt8 === 0,
1830
+ `${badBit} bit rows and ${badInt8} int8 rows have the wrong width for ${dims} dimensions`
1831
+ );
1832
+ const span = db.prepare("SELECT MIN(id) AS lo, MAX(id) AS hi FROM chunks").get();
1833
+ check(
1834
+ `chunk-ids-dense:${shard.path}`,
1835
+ chunkCount === 0 || Number(span.lo) === 1 && Number(span.hi) === chunkCount,
1836
+ `ids span ${String(span.lo)}..${String(span.hi)} for ${chunkCount} chunks`
1837
+ );
1692
1838
  } finally {
1693
1839
  conn.close();
1694
1840
  }
1695
1841
  check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
1696
1842
  }
1697
1843
  for (const smoke of manifest.smokeQueries ?? []) {
1698
- const hits = handle.searchDocumentsFts(smoke.query, 10).map((h) => h.documentId);
1844
+ const hits = handle.searchDocumentsFts(smoke.query, SMOKE_QUERY_TOP_N).map((h) => h.documentId);
1699
1845
  const missing = smoke.expectedDocumentIds.filter((id) => !hits.includes(id));
1700
1846
  check(
1701
1847
  `smoke:${smoke.query}`,
1702
1848
  missing.length === 0,
1703
- missing.length > 0 ? `missing ${missing.join(", ")}` : void 0
1849
+ missing.length > 0 ? `the catalog's built-in sanity query "${smoke.query}" did not return its expected document${missing.length > 1 ? "s" : ""} (${missing.join(", ")}) in the top ${SMOKE_QUERY_TOP_N} results \u2014 the search index cannot answer queries the publisher recorded as guaranteed, so the archive is corrupt or was built by a broken toolchain; re-download it or report it to the publisher` : void 0
1704
1850
  );
1705
1851
  }
1706
1852
  }
@@ -1710,56 +1856,642 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1710
1856
  return report(manifest);
1711
1857
  }
1712
1858
 
1859
+ // src/export/duckdb.ts
1860
+ import { spawn } from "child_process";
1861
+ import { access } from "fs/promises";
1862
+ var DuckdbError = class extends Error {
1863
+ constructor(message, stderr) {
1864
+ super(message);
1865
+ this.stderr = stderr;
1866
+ this.name = "DuckdbError";
1867
+ }
1868
+ stderr;
1869
+ };
1870
+ async function duckdbVersion(binaryPath) {
1871
+ const { stdout } = await runDuckdbProcess(binaryPath, ["--version"], "");
1872
+ const match = /v?(\d+\.\d+\.\d+)/.exec(stdout);
1873
+ if (!match?.[1])
1874
+ throw new DuckdbError(`could not read the DuckDB version from: ${stdout.trim()}`, "");
1875
+ return match[1];
1876
+ }
1877
+ async function assertDuckdbCli(cli) {
1878
+ await access(cli.binaryPath).catch(() => {
1879
+ throw new DuckdbError(`DuckDB CLI not found at ${cli.binaryPath}`, "");
1880
+ });
1881
+ const version = await duckdbVersion(cli.binaryPath);
1882
+ if (cli.expectedVersion && version !== cli.expectedVersion) {
1883
+ throw new DuckdbError(
1884
+ `DuckDB ${cli.expectedVersion} expected, but ${cli.binaryPath} is ${version}; Parquet output is only reproducible against the pinned release`,
1885
+ ""
1886
+ );
1887
+ }
1888
+ return version;
1889
+ }
1890
+ async function runDuckdbScript(cli, sql) {
1891
+ const { stdout } = await runDuckdbProcess(cli.binaryPath, ["-batch", ":memory:"], sql);
1892
+ return stdout;
1893
+ }
1894
+ function runDuckdbProcess(binaryPath, args, stdin) {
1895
+ return new Promise((resolve4, reject) => {
1896
+ const child = spawn(binaryPath, args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
1897
+ let stdout = "";
1898
+ let stderr = "";
1899
+ child.stdout.setEncoding("utf8");
1900
+ child.stderr.setEncoding("utf8");
1901
+ child.stdout.on("data", (chunk) => {
1902
+ stdout += chunk;
1903
+ });
1904
+ child.stderr.on("data", (chunk) => {
1905
+ stderr += chunk;
1906
+ });
1907
+ child.on(
1908
+ "error",
1909
+ (err) => reject(new DuckdbError(`could not start DuckDB: ${err.message}`, stderr))
1910
+ );
1911
+ child.on("close", (code) => {
1912
+ if (code === 0) resolve4({ stdout });
1913
+ else
1914
+ reject(
1915
+ new DuckdbError(
1916
+ `DuckDB exited with code ${code}: ${stderr.trim() || stdout.trim()}`,
1917
+ stderr
1918
+ )
1919
+ );
1920
+ });
1921
+ child.stdin.on("error", () => {
1922
+ });
1923
+ child.stdin.end(stdin);
1924
+ });
1925
+ }
1926
+
1927
+ // src/export/parquet.ts
1928
+ import { createHash as createHash3 } from "crypto";
1929
+ import { createWriteStream as createWriteStream3 } from "fs";
1930
+ import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
1931
+ import { join as join4, resolve as resolve3 } from "path";
1932
+ import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
1933
+ import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
1934
+ var PARQUET_EXPORT_VERSION = 1;
1935
+ var DEFAULT_ROW_GROUP_SIZE = 32768;
1936
+ var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
1937
+ var PARQUET_REPORT_PATH = "parquet-manifest.json";
1938
+ async function exportCatalogParquet(opts) {
1939
+ const duckdbVersion2 = await assertDuckdbCli(opts.duckdb);
1940
+ const outDir = resolve3(opts.outDir);
1941
+ await mkdir3(outDir, { recursive: true });
1942
+ const work = join4(outDir, `.export-${process.pid}-${Date.now().toString(36)}`);
1943
+ await mkdir3(work, { recursive: true });
1944
+ try {
1945
+ let rootDir;
1946
+ if ("archivePath" in opts.source) {
1947
+ opts.onProgress?.({ phase: "extract" });
1948
+ rootDir = join4(work, "catalog");
1949
+ await extractGezkVerified(opts.source.archivePath, rootDir);
1950
+ } else {
1951
+ rootDir = resolve3(opts.source.rootDir);
1952
+ }
1953
+ const manifest = KnowledgeCatalogManifestSchema4.parse(
1954
+ JSON.parse(await readFile2(join4(rootDir, MANIFEST_PATH), "utf8"))
1955
+ );
1956
+ const staging = join4(work, "ndjson");
1957
+ await mkdir3(staging, { recursive: true });
1958
+ const files = [];
1959
+ const metadata = kvMetadata(manifest);
1960
+ const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
1961
+ const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
1962
+ try {
1963
+ const topicPaths = topicPathsOf(router);
1964
+ const topicRows = await stageTopics(router, join4(staging, "topics.ndjson"));
1965
+ opts.onProgress?.({ phase: "stage", table: "topics", rows: topicRows });
1966
+ files.push(
1967
+ await writeParquet(opts.duckdb, {
1968
+ outDir,
1969
+ file: "topics.parquet",
1970
+ table: "topics",
1971
+ rows: topicRows,
1972
+ sql: topicsCopy(
1973
+ join4(staging, "topics.ndjson"),
1974
+ join4(outDir, "topics.parquet"),
1975
+ rowGroupSize,
1976
+ metadata
1977
+ ),
1978
+ onProgress: opts.onProgress
1979
+ })
1980
+ );
1981
+ for (const shard of manifest.router.shards) {
1982
+ const tag = String(shard.id).padStart(3, "0");
1983
+ const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
1984
+ const documentRows = await stageDocuments(router, shard.id, topicPaths, documentsNdjson);
1985
+ opts.onProgress?.({
1986
+ phase: "stage",
1987
+ table: "documents",
1988
+ shardId: shard.id,
1989
+ rows: documentRows
1990
+ });
1991
+ files.push(
1992
+ await writeParquet(opts.duckdb, {
1993
+ outDir,
1994
+ file: `documents-${tag}.parquet`,
1995
+ table: "documents",
1996
+ shardId: shard.id,
1997
+ rows: documentRows,
1998
+ sql: documentsCopy(
1999
+ documentsNdjson,
2000
+ join4(outDir, `documents-${tag}.parquet`),
2001
+ rowGroupSize,
2002
+ metadata
2003
+ ),
2004
+ onProgress: opts.onProgress
2005
+ })
2006
+ );
2007
+ const shardDb = openCatalogDatabase(join4(rootDir, shard.path));
2008
+ const chunksNdjson = join4(staging, `chunks-${tag}.ndjson`);
2009
+ let chunkRows;
2010
+ try {
2011
+ chunkRows = await stageChunks(
2012
+ shardDb,
2013
+ shard.id,
2014
+ manifest.embedding.dimensions,
2015
+ chunksNdjson
2016
+ );
2017
+ } finally {
2018
+ shardDb.close();
2019
+ }
2020
+ opts.onProgress?.({ phase: "stage", table: "chunks", shardId: shard.id, rows: chunkRows });
2021
+ files.push(
2022
+ await writeParquet(opts.duckdb, {
2023
+ outDir,
2024
+ file: `chunks-${tag}.parquet`,
2025
+ table: "chunks",
2026
+ shardId: shard.id,
2027
+ rows: chunkRows,
2028
+ sql: chunksCopy(
2029
+ chunksNdjson,
2030
+ join4(outDir, `chunks-${tag}.parquet`),
2031
+ manifest.embedding.dimensions,
2032
+ rowGroupSize,
2033
+ metadata
2034
+ ),
2035
+ onProgress: opts.onProgress
2036
+ })
2037
+ );
2038
+ }
2039
+ } finally {
2040
+ router.close();
2041
+ }
2042
+ const report = {
2043
+ exportVersion: PARQUET_EXPORT_VERSION,
2044
+ catalogId: manifest.id,
2045
+ version: manifest.version,
2046
+ publisherId: manifest.publisher.id,
2047
+ formatVersion: manifest.formatVersion,
2048
+ embeddingProfileId: manifest.embedding.id,
2049
+ dimensions: manifest.embedding.dimensions,
2050
+ files,
2051
+ duckdbVersion: duckdbVersion2,
2052
+ exporter: { name: KNOWLEDGE_TOOLCHAIN.name, version: KNOWLEDGE_TOOLCHAIN.version }
2053
+ };
2054
+ const { writeFile } = await import("fs/promises");
2055
+ await writeFile(
2056
+ join4(outDir, PARQUET_REPORT_PATH),
2057
+ `${JSON.stringify(report, null, 2)}
2058
+ `,
2059
+ "utf8"
2060
+ );
2061
+ return report;
2062
+ } finally {
2063
+ await rm(work, { recursive: true, force: true }).catch(() => {
2064
+ });
2065
+ }
2066
+ }
2067
+ function ndjsonWriter(path) {
2068
+ const stream = createWriteStream3(path, { encoding: "utf8" });
2069
+ return {
2070
+ write(row) {
2071
+ const line = `${JSON.stringify(row)}
2072
+ `;
2073
+ if (stream.write(line)) return Promise.resolve();
2074
+ return new Promise((resolveDrain) => stream.once("drain", () => resolveDrain()));
2075
+ },
2076
+ close() {
2077
+ return new Promise((resolveClose, reject) => {
2078
+ stream.on("error", reject);
2079
+ stream.end(() => resolveClose());
2080
+ });
2081
+ }
2082
+ };
2083
+ }
2084
+ function topicPathsOf(router) {
2085
+ const parents = /* @__PURE__ */ new Map();
2086
+ for (const row of router.db.prepare("SELECT id, parent_id FROM topics").all()) {
2087
+ parents.set(row.id, row.parent_id);
2088
+ }
2089
+ const paths = /* @__PURE__ */ new Map();
2090
+ for (const id of parents.keys()) {
2091
+ const path = [];
2092
+ let cursor = id;
2093
+ const seen = /* @__PURE__ */ new Set();
2094
+ while (cursor && !seen.has(cursor)) {
2095
+ seen.add(cursor);
2096
+ path.unshift(cursor);
2097
+ cursor = parents.get(cursor) ?? null;
2098
+ }
2099
+ paths.set(id, path);
2100
+ }
2101
+ return paths;
2102
+ }
2103
+ async function stageTopics(router, path) {
2104
+ const out = ndjsonWriter(path);
2105
+ let rows = 0;
2106
+ const statement = router.db.prepare(
2107
+ "SELECT id, parent_id, name, description, sort_key, document_count FROM topics ORDER BY sort_key, id"
2108
+ );
2109
+ for (const row of statement.iterate()) {
2110
+ await out.write({
2111
+ id: row.id,
2112
+ parent_id: row.parent_id ?? null,
2113
+ name: row.name,
2114
+ description: row.description ?? null,
2115
+ sort_key: row.sort_key,
2116
+ document_count: Number(row.document_count)
2117
+ });
2118
+ rows++;
2119
+ }
2120
+ await out.close();
2121
+ return rows;
2122
+ }
2123
+ async function stageDocuments(router, shardId, topicPaths, path) {
2124
+ const out = ndjsonWriter(path);
2125
+ let rows = 0;
2126
+ const aliasesOf = router.db.prepare(
2127
+ "SELECT alias FROM aliases WHERE document_id = ? ORDER BY alias"
2128
+ );
2129
+ const statement = router.db.prepare(
2130
+ `SELECT id, title, slug, summary, language, topic_id, shard_id, chunk_count,
2131
+ source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob
2132
+ FROM documents WHERE shard_id = ? ORDER BY id`
2133
+ );
2134
+ for (const row of statement.iterate(shardId)) {
2135
+ const aliases = aliasesOf.all(row.id).map(
2136
+ (a) => a.alias
2137
+ );
2138
+ await out.write({
2139
+ id: row.id,
2140
+ title: row.title,
2141
+ slug: row.slug,
2142
+ summary: row.summary ?? null,
2143
+ language: row.language,
2144
+ topic_id: row.topic_id,
2145
+ topic_path: topicPaths.get(row.topic_id) ?? [row.topic_id],
2146
+ shard_id: Number(row.shard_id),
2147
+ chunk_count: Number(row.chunk_count),
2148
+ markdown: decodeBody(row.body_codec, row.body_blob),
2149
+ source_url: row.source_url ?? null,
2150
+ source_revision: row.source_revision ?? null,
2151
+ source_updated_at: row.source_updated_at ?? null,
2152
+ attribution: row.attribution_json ?? null,
2153
+ aliases
2154
+ });
2155
+ rows++;
2156
+ }
2157
+ await out.close();
2158
+ return rows;
2159
+ }
2160
+ function decodeBody(codec, blob) {
2161
+ const bytes = Buffer.from(blob);
2162
+ if (codec === "br") {
2163
+ return brotliDecompressSync2(bytes, { maxOutputLength: MAX_KNOWLEDGE_DOCUMENT_BYTES }).toString(
2164
+ "utf8"
2165
+ );
2166
+ }
2167
+ if (codec === "none") return bytes.toString("utf8");
2168
+ throw new Error(`unknown body codec ${codec}`);
2169
+ }
2170
+ async function stageChunks(shard, shardId, dimensions, path) {
2171
+ const out = ndjsonWriter(path);
2172
+ const bitBytes = Math.ceil(dimensions / 8);
2173
+ let rows = 0;
2174
+ const statement = shard.db.prepare(
2175
+ `SELECT c.id, c.chunk_uid, c.document_id, c.ordinal, c.title, c.heading_path, c.heading_text,
2176
+ c.text, c.token_count, c.content_hash, c.line_start, c.line_end,
2177
+ i.v AS int8, b.v AS bits
2178
+ FROM chunks c
2179
+ JOIN chunk_vectors_int8 i ON i.chunk_id = c.id
2180
+ JOIN chunk_vectors_bit b ON b.chunk_id = c.id
2181
+ ORDER BY c.id`
2182
+ );
2183
+ for (const row of statement.iterate()) {
2184
+ const int8 = Buffer.from(row.int8);
2185
+ const bits = Buffer.from(row.bits);
2186
+ if (int8.length !== dimensions || bits.length !== bitBytes) {
2187
+ throw new Error(
2188
+ `chunk ${row.chunk_uid}: vector widths ${int8.length}/${bits.length} do not match the ${dimensions}-dimension profile`
2189
+ );
2190
+ }
2191
+ await out.write({
2192
+ id: Number(row.id),
2193
+ chunk_uid: row.chunk_uid,
2194
+ document_id: row.document_id,
2195
+ ordinal: Number(row.ordinal),
2196
+ title: row.title,
2197
+ heading_path: parseJsonStrings(row.heading_path),
2198
+ heading_text: row.heading_text,
2199
+ text: row.text,
2200
+ token_count: Number(row.token_count),
2201
+ content_hash: row.content_hash,
2202
+ line_start: Number(row.line_start),
2203
+ line_end: Number(row.line_end),
2204
+ shard_id: shardId,
2205
+ embedding: Array.from(new Int8Array(int8.buffer, int8.byteOffset, int8.length)),
2206
+ embedding_bit: bits.toString("base64")
2207
+ });
2208
+ rows++;
2209
+ }
2210
+ await out.close();
2211
+ return rows;
2212
+ }
2213
+ function parseJsonStrings(raw) {
2214
+ try {
2215
+ const parsed = JSON.parse(raw);
2216
+ return Array.isArray(parsed) ? parsed.filter((p) => typeof p === "string") : [];
2217
+ } catch {
2218
+ return [];
2219
+ }
2220
+ }
2221
+ function sqlString(value) {
2222
+ return `'${value.replace(/'/g, "''")}'`;
2223
+ }
2224
+ function kvMetadata(manifest) {
2225
+ return {
2226
+ gezk_catalog_id: manifest.id,
2227
+ gezk_catalog_version: manifest.version,
2228
+ gezk_publisher_id: manifest.publisher.id,
2229
+ gezk_format_version: manifest.formatVersion,
2230
+ gezk_embedding_profile: manifest.embedding.id,
2231
+ gezk_embedding_model: manifest.embedding.model.repo,
2232
+ gezk_embedding_dimensions: String(manifest.embedding.dimensions),
2233
+ gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
2234
+ gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
2235
+ gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
2236
+ gezk_license: manifest.license.name
2237
+ };
2238
+ }
2239
+ function copyOptions(rowGroupSize, metadata) {
2240
+ const kv = Object.entries(metadata).map(([key, value]) => `${key}: ${sqlString(value)}`).join(", ");
2241
+ return `(FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 3, ROW_GROUP_SIZE ${rowGroupSize}, KV_METADATA {${kv}})`;
2242
+ }
2243
+ var PRELUDE = "SET threads = 1;\nSET preserve_insertion_order = true;\n";
2244
+ function readNdjson(path, columns) {
2245
+ const spec = Object.entries(columns).map(([name, type]) => `${name}: ${sqlString(type)}`).join(", ");
2246
+ return `read_ndjson(${sqlString(path)}, columns = {${spec}}, maximum_object_size = ${NDJSON_MAX_OBJECT_BYTES})`;
2247
+ }
2248
+ function topicsCopy(ndjson, out, rowGroupSize, metadata) {
2249
+ const source = readNdjson(ndjson, {
2250
+ id: "VARCHAR",
2251
+ parent_id: "VARCHAR",
2252
+ name: "VARCHAR",
2253
+ description: "VARCHAR",
2254
+ sort_key: "VARCHAR",
2255
+ document_count: "BIGINT"
2256
+ });
2257
+ return `${PRELUDE}COPY (SELECT id, parent_id, name, description, sort_key, document_count FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2258
+ `;
2259
+ }
2260
+ function documentsCopy(ndjson, out, rowGroupSize, metadata) {
2261
+ const source = readNdjson(ndjson, {
2262
+ id: "VARCHAR",
2263
+ title: "VARCHAR",
2264
+ slug: "VARCHAR",
2265
+ summary: "VARCHAR",
2266
+ language: "VARCHAR",
2267
+ topic_id: "VARCHAR",
2268
+ topic_path: "VARCHAR[]",
2269
+ shard_id: "INTEGER",
2270
+ chunk_count: "INTEGER",
2271
+ markdown: "VARCHAR",
2272
+ source_url: "VARCHAR",
2273
+ source_revision: "VARCHAR",
2274
+ source_updated_at: "VARCHAR",
2275
+ attribution: "VARCHAR",
2276
+ aliases: "VARCHAR[]"
2277
+ });
2278
+ return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2279
+ `;
2280
+ }
2281
+ function chunksCopy(ndjson, out, dimensions, rowGroupSize, metadata) {
2282
+ const source = readNdjson(ndjson, {
2283
+ id: "BIGINT",
2284
+ chunk_uid: "VARCHAR",
2285
+ document_id: "VARCHAR",
2286
+ ordinal: "INTEGER",
2287
+ title: "VARCHAR",
2288
+ heading_path: "VARCHAR[]",
2289
+ heading_text: "VARCHAR",
2290
+ text: "VARCHAR",
2291
+ token_count: "INTEGER",
2292
+ content_hash: "VARCHAR",
2293
+ line_start: "INTEGER",
2294
+ line_end: "INTEGER",
2295
+ shard_id: "INTEGER",
2296
+ embedding: "TINYINT[]",
2297
+ embedding_bit: "VARCHAR"
2298
+ });
2299
+ return `${PRELUDE}COPY (SELECT id, chunk_uid, document_id, ordinal, title, heading_path, heading_text, text, token_count, content_hash, line_start, line_end, shard_id, embedding::TINYINT[${dimensions}] AS embedding, from_base64(embedding_bit) AS embedding_bit FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2300
+ `;
2301
+ }
2302
+ async function writeParquet(cli, spec) {
2303
+ spec.onProgress?.({ phase: "write", file: spec.file });
2304
+ const target = join4(spec.outDir, spec.file);
2305
+ await rm(target, { force: true });
2306
+ await runDuckdbScript(cli, spec.sql);
2307
+ const bytes = await readFile2(target);
2308
+ return {
2309
+ path: spec.file,
2310
+ table: spec.table,
2311
+ ...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
2312
+ rows: spec.rows,
2313
+ sizeBytes: (await stat2(target)).size,
2314
+ sha256: createHash3("sha256").update(bytes).digest("hex")
2315
+ };
2316
+ }
2317
+ async function findDuckdbBinary(env = process.env) {
2318
+ const configured = env.GEZEL_DUCKDB_BIN?.trim();
2319
+ if (configured) return configured;
2320
+ const { readdir: readdir2 } = await import("fs/promises");
2321
+ const { homedir } = await import("os");
2322
+ const name = process.platform === "win32" ? "duckdb.exe" : "duckdb";
2323
+ const homes = [env.GEZEL_HOME, join4(homedir(), ".gezel-dev"), join4(homedir(), ".gezel")].filter(
2324
+ (h) => Boolean(h)
2325
+ );
2326
+ for (const home of homes) {
2327
+ const engines = join4(home, "engines", "duckdb");
2328
+ const versions = (await readdir2(engines).catch(() => [])).sort().reverse();
2329
+ for (const version of versions) {
2330
+ const candidate = join4(engines, version, name);
2331
+ if (await stat2(candidate).catch(() => null)) return candidate;
2332
+ }
2333
+ }
2334
+ const pathDirs = (env.PATH ?? "").split(process.platform === "win32" ? ";" : ":");
2335
+ for (const dir of pathDirs) {
2336
+ if (!dir) continue;
2337
+ const candidate = join4(dir, name);
2338
+ if (await stat2(candidate).catch(() => null)) return candidate;
2339
+ }
2340
+ return null;
2341
+ }
2342
+
1713
2343
  // src/profiles/registry.ts
1714
2344
  var SHARED_ENCODING = {
1715
2345
  pooling: "mean",
1716
2346
  normalized: true,
1717
2347
  dimensions: 384,
1718
2348
  maxTokens: 512,
1719
- vectorEncoding: "bit384+int8",
2349
+ vectorEncoding: "bit+int8",
1720
2350
  distance: { stage1: "hamming", stage2: "cosine" },
1721
2351
  quantization: {
1722
2352
  int8: { method: "symmetric-linear", scale: 127 },
1723
2353
  binary: { method: "sign", threshold: 0, packing: "lsb-first" }
1724
2354
  }
1725
2355
  };
1726
- var GEZEL_MULTILINGUAL_E5_SMALL_1 = {
1727
- id: "gezel-multilingual-e5-small@1",
2356
+ var MULTILINGUAL_E5_SMALL_1 = {
2357
+ id: "multilingual-e5-small@1",
1728
2358
  model: {
1729
2359
  repo: "Xenova/multilingual-e5-small",
1730
- revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78"
2360
+ revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
2361
+ onnxFile: "onnx/model.onnx",
2362
+ onnxDigest: "sha256:4aa845c27760e06e9a686b9d8b5d440eae4b6612cd09e5b522b716d3941f77ff"
2363
+ },
2364
+ tokenizer: {
2365
+ kind: "sentencepiece-xlmr",
2366
+ file: "tokenizer.json",
2367
+ digest: "sha256:0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39"
1731
2368
  },
1732
- tokenizer: { kind: "sentencepiece-xlmr" },
1733
2369
  queryInstruction: "query: ",
1734
2370
  passageInstruction: "passage: ",
1735
2371
  ...SHARED_ENCODING
1736
2372
  };
1737
- var GEZEL_BGE_SMALL_EN_V15_1 = {
1738
- id: "gezel-bge-small-en-v1.5@1",
2373
+ var BGE_SMALL_EN_V15_1 = {
2374
+ id: "bge-small-en-v1.5@1",
1739
2375
  model: {
1740
2376
  repo: "Xenova/bge-small-en-v1.5",
1741
- revision: "ea104dacec62c0de699686887e3f920caeb4f3e3"
2377
+ revision: "ea104dacec62c0de699686887e3f920caeb4f3e3",
2378
+ onnxFile: "onnx/model.onnx",
2379
+ onnxDigest: "sha256:828e1496d7fabb79cfa4dcd84fa38625c0d3d21da474a00f08db0f559940cf35"
2380
+ },
2381
+ tokenizer: {
2382
+ kind: "wordpiece-bert",
2383
+ file: "tokenizer.json",
2384
+ digest: "sha256:d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66"
1742
2385
  },
1743
- tokenizer: { kind: "wordpiece-bert" },
1744
2386
  queryInstruction: "Represent this sentence for searching relevant passages: ",
1745
2387
  passageInstruction: "",
1746
2388
  ...SHARED_ENCODING
1747
2389
  };
1748
2390
  var KNOWLEDGE_EMBEDDING_PROFILES = [
1749
- GEZEL_MULTILINGUAL_E5_SMALL_1,
1750
- GEZEL_BGE_SMALL_EN_V15_1
2391
+ MULTILINGUAL_E5_SMALL_1,
2392
+ BGE_SMALL_EN_V15_1
1751
2393
  ];
1752
2394
  function knowledgeEmbeddingProfile(id) {
1753
2395
  return KNOWLEDGE_EMBEDDING_PROFILES.find((p) => p.id === id) ?? null;
1754
2396
  }
1755
- var GEZEL_MARKDOWN_CHUNKS_2 = {
1756
- id: "gezel-markdown-chunks@2",
2397
+ var MARKDOWN_CHUNKS_2 = {
2398
+ id: "markdown-chunks@2",
1757
2399
  unit: "tokens",
1758
2400
  tokenizer: "profile",
1759
- targetTokens: 420,
1760
- overlapTokens: 64,
1761
- contextHeader: { maxTokens: 64 }
2401
+ target: 420,
2402
+ overlap: 64,
2403
+ contextHeader: { max: 64 }
1762
2404
  };
2405
+ var GEZEL_MULTILINGUAL_E5_SMALL_1 = MULTILINGUAL_E5_SMALL_1;
2406
+ var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
2407
+ var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
2408
+
2409
+ // src/embedding/profile-embedder.ts
2410
+ import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
2411
+
2412
+ // src/embedding/artifact-verify.ts
2413
+ import { createHash as createHash4 } from "crypto";
2414
+ import { createReadStream } from "fs";
2415
+ import { stat as stat3 } from "fs/promises";
2416
+ import { join as join5 } from "path";
2417
+ import { embeddingProfileArtifacts } from "@bendyline/gezk";
2418
+ var EmbedderArtifactError = class extends Error {
2419
+ constructor(reason, profileId, checks) {
2420
+ super(describeFailure(reason, profileId, checks));
2421
+ this.reason = reason;
2422
+ this.profileId = profileId;
2423
+ this.checks = checks;
2424
+ this.name = "EmbedderArtifactError";
2425
+ }
2426
+ reason;
2427
+ profileId;
2428
+ checks;
2429
+ isActionable = true;
2430
+ };
2431
+ function describeFailure(reason, profileId, checks) {
2432
+ const detail = checks.filter((c) => c.actual !== c.expected).map(
2433
+ (c) => c.actual === null ? `${c.file} not found at ${c.path}` : `${c.file} is ${c.actual}, profile pins ${c.expected}`
2434
+ ).join("; ");
2435
+ return reason === "missing" ? `cannot verify the model files for profile ${profileId}: ${detail}` : `the model files loaded for profile ${profileId} differ from the ones it pins: ${detail}`;
2436
+ }
2437
+ function transformersCachePath(cacheDir, repo, revision, file) {
2438
+ return revision === "main" ? join5(cacheDir, repo, file) : join5(cacheDir, repo, revision, file);
2439
+ }
2440
+ var digestMemo = /* @__PURE__ */ new Map();
2441
+ var DIGEST_MEMO_MAX = 32;
2442
+ async function fileDigest(path) {
2443
+ let size;
2444
+ let mtimeMs;
2445
+ try {
2446
+ const info = await stat3(path);
2447
+ if (!info.isFile()) return null;
2448
+ size = info.size;
2449
+ mtimeMs = info.mtimeMs;
2450
+ } catch {
2451
+ return null;
2452
+ }
2453
+ const key = `${path} ${size} ${mtimeMs}`;
2454
+ const memo = digestMemo.get(key);
2455
+ if (memo) return memo;
2456
+ const digest = await new Promise((resolve4, reject) => {
2457
+ const hash = createHash4("sha256");
2458
+ createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve4(`sha256:${hash.digest("hex")}`));
2459
+ });
2460
+ if (digestMemo.size >= DIGEST_MEMO_MAX) {
2461
+ const oldest = digestMemo.keys().next().value;
2462
+ if (oldest !== void 0) digestMemo.delete(oldest);
2463
+ }
2464
+ digestMemo.set(key, digest);
2465
+ return digest;
2466
+ }
2467
+ async function verifyProfileArtifacts(profile, opts) {
2468
+ const artifacts = embeddingProfileArtifacts(profile);
2469
+ const revision = opts.revision ?? profile.model.revision;
2470
+ const pinned = [];
2471
+ if (artifacts.onnxDigest) {
2472
+ pinned.push({ role: "onnx", file: artifacts.onnxFile, expected: artifacts.onnxDigest });
2473
+ }
2474
+ if (artifacts.tokenizerDigest) {
2475
+ pinned.push({
2476
+ role: "tokenizer",
2477
+ file: artifacts.tokenizerFile,
2478
+ expected: artifacts.tokenizerDigest
2479
+ });
2480
+ }
2481
+ if (pinned.length === 0) return { status: "unpinned", checks: [] };
2482
+ const checks = [];
2483
+ for (const { role, file, expected } of pinned) {
2484
+ const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file);
2485
+ checks.push({ role, file, path, expected, actual: await fileDigest(path) });
2486
+ }
2487
+ if (checks.some((c) => c.actual === null)) {
2488
+ throw new EmbedderArtifactError("missing", profile.id, checks);
2489
+ }
2490
+ if (checks.some((c) => c.actual !== c.expected)) {
2491
+ throw new EmbedderArtifactError("mismatch", profile.id, checks);
2492
+ }
2493
+ return { status: "verified", checks };
2494
+ }
1763
2495
 
1764
2496
  // src/embedding/profile-embedder.ts
1765
2497
  var MAX_BATCH = 8;
@@ -1771,6 +2503,42 @@ var EmbedderUnavailableError = class extends Error {
1771
2503
  this.name = "EmbedderUnavailableError";
1772
2504
  }
1773
2505
  };
2506
+ var ONNX_DTYPE_BY_SUFFIX = [
2507
+ ["_q4f16", "q4f16"],
2508
+ ["_quantized", "q8"],
2509
+ ["_uint8", "uint8"],
2510
+ ["_int8", "int8"],
2511
+ ["_bnb4", "bnb4"],
2512
+ ["_fp16", "fp16"],
2513
+ ["_q4", "q4"]
2514
+ ];
2515
+ var POOLING_BY_PROFILE = {
2516
+ mean: "mean",
2517
+ cls: "cls"
2518
+ };
2519
+ function resolveTransformersModelOptions(profile) {
2520
+ const { onnxFile, tokenizerFile } = embeddingProfileArtifacts2(profile);
2521
+ if (tokenizerFile !== "tokenizer.json") {
2522
+ throw new EmbedderUnavailableError(
2523
+ `profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
2524
+ );
2525
+ }
2526
+ if (!onnxFile.endsWith(".onnx")) {
2527
+ throw new EmbedderUnavailableError(
2528
+ `profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
2529
+ );
2530
+ }
2531
+ const slash = onnxFile.lastIndexOf("/");
2532
+ const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
2533
+ const base = onnxFile.slice(slash + 1, -".onnx".length);
2534
+ const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
2535
+ const [suffix, dtype] = variant ?? ["", "fp32"];
2536
+ const modelFileName = base.slice(0, base.length - suffix.length);
2537
+ if (modelFileName === "") {
2538
+ throw new EmbedderUnavailableError(`profile ${profile.id} pins an unnamed graph ${onnxFile}`);
2539
+ }
2540
+ return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
2541
+ }
1774
2542
  async function loadTransformers() {
1775
2543
  try {
1776
2544
  const specifier = "@huggingface/transformers";
@@ -1786,30 +2554,52 @@ async function loadTransformers() {
1786
2554
  }
1787
2555
  }
1788
2556
  async function createProfileEmbedder(profile, opts = {}) {
1789
- const transformers = await loadTransformers();
2557
+ const pooling = POOLING_BY_PROFILE[profile.pooling];
2558
+ if (!pooling) {
2559
+ throw new EmbedderUnavailableError(
2560
+ `profile ${profile.id} uses ${profile.pooling} pooling, which this runtime does not implement`
2561
+ );
2562
+ }
2563
+ const modelOptions = resolveTransformersModelOptions(profile);
2564
+ const transformers = opts.transformers ?? await loadTransformers();
1790
2565
  const cacheDir = opts.cacheDir ?? process.env.GEZEL_HF_CACHE_DIR;
1791
2566
  if (cacheDir) {
1792
2567
  transformers.env.cacheDir = cacheDir;
1793
2568
  transformers.env.useFSCache = true;
1794
2569
  transformers.env.allowRemoteModels = true;
1795
2570
  }
1796
- const modelOpts = { revision: profile.model.revision };
1797
2571
  const [pipe, tokenizer] = await Promise.all([
1798
- transformers.pipeline("feature-extraction", profile.model.repo, modelOpts),
1799
- transformers.AutoTokenizer.from_pretrained(profile.model.repo, modelOpts)
2572
+ transformers.pipeline("feature-extraction", profile.model.repo, {
2573
+ ...modelOptions,
2574
+ ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
2575
+ }),
2576
+ transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
2577
+ revision: modelOptions.revision
2578
+ })
1800
2579
  ]);
2580
+ let verification;
2581
+ try {
2582
+ verification = await verifyProfileArtifacts(profile, {
2583
+ cacheDir: cacheDir ?? transformers.env.cacheDir ?? ""
2584
+ });
2585
+ } catch (err) {
2586
+ await pipe.dispose?.().catch(() => {
2587
+ });
2588
+ throw err;
2589
+ }
1801
2590
  const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
1802
2591
  const embed = async (texts) => {
1803
2592
  const out = [];
1804
2593
  for (let i = 0; i < texts.length; i += MAX_BATCH) {
1805
2594
  const slice = texts.slice(i, i + MAX_BATCH).map(cap);
1806
- const result = await pipe(slice, { pooling: "mean", normalize: true });
2595
+ const result = await pipe(slice, { pooling, normalize: true });
1807
2596
  out.push(...result.tolist());
1808
2597
  }
1809
2598
  return out;
1810
2599
  };
1811
2600
  return {
1812
2601
  profile,
2602
+ verification,
1813
2603
  embed,
1814
2604
  embedQuery: async (text) => {
1815
2605
  const [vector] = await embed([`${profile.queryInstruction}${text}`]);
@@ -1823,9 +2613,9 @@ async function createProfileEmbedder(profile, opts = {}) {
1823
2613
  }
1824
2614
 
1825
2615
  // src/markdown-adapter/load.ts
1826
- import { readFile as readFile2, readdir } from "fs/promises";
1827
- import { join as join4, relative as relative3, sep } from "path";
1828
- import { CatalogDocumentSchema } from "@bendyline/gezel";
2616
+ import { readFile as readFile3, readdir } from "fs/promises";
2617
+ import { join as join6, relative as relative3, sep } from "path";
2618
+ import { CatalogDocumentSchema } from "@bendyline/gezk";
1829
2619
  var ROOT_TOPIC_ID = "general";
1830
2620
  var SUMMARY_MAX_CHARS = 280;
1831
2621
  function parseFrontMatter(markdown) {
@@ -1879,7 +2669,7 @@ async function walkMarkdownFiles(root) {
1879
2669
  );
1880
2670
  for (const entry of entries) {
1881
2671
  if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
1882
- const abs = join4(dir, entry.name);
2672
+ const abs = join6(dir, entry.name);
1883
2673
  if (entry.isDirectory()) await visit(abs);
1884
2674
  else if (entry.isFile() && /\.(md|markdown)$/i.test(entry.name)) out.push(abs);
1885
2675
  }
@@ -1930,7 +2720,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
1930
2720
  const id = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
1931
2721
  if (seenIds.has(id)) throw new Error(`duplicate document id from path: ${rel}`);
1932
2722
  seenIds.add(id);
1933
- const raw = (await readFile2(abs, "utf8")).replace(/^\uFEFF/, "");
2723
+ const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
1934
2724
  const fm = parseFrontMatter(raw);
1935
2725
  const stem = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
1936
2726
  const title = fm.title ?? firstHeading(fm.body) ?? stem;
@@ -1959,110 +2749,131 @@ async function loadMarkdownCatalog(rootDir, opts) {
1959
2749
  return { topics, documents };
1960
2750
  }
1961
2751
 
1962
- // src/signatures/jcs.ts
1963
- function canonicalizeJson(value) {
1964
- return serialize(value, "$");
1965
- }
1966
- function serialize(value, path) {
1967
- if (value === null) return "null";
1968
- switch (typeof value) {
1969
- case "boolean":
1970
- return value ? "true" : "false";
1971
- case "number":
1972
- if (!Number.isFinite(value)) throw new Error(`non-finite number at ${path}`);
1973
- return JSON.stringify(value);
1974
- case "string":
1975
- return JSON.stringify(value);
1976
- case "object":
1977
- break;
1978
- default:
1979
- throw new Error(`value of type ${typeof value} at ${path} has no JSON identity`);
1980
- }
1981
- if (Array.isArray(value)) {
1982
- return `[${value.map((item, i) => serialize(item ?? null, `${path}[${i}]`)).join(",")}]`;
1983
- }
1984
- const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => compareUtf16(a, b));
1985
- return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${serialize(v, `${path}.${k}`)}`).join(",")}}`;
1986
- }
1987
- function compareUtf16(a, b) {
1988
- const n = Math.min(a.length, b.length);
1989
- for (let i = 0; i < n; i++) {
1990
- const d = a.charCodeAt(i) - b.charCodeAt(i);
1991
- if (d !== 0) return d;
1992
- }
1993
- return a.length - b.length;
1994
- }
2752
+ // src/registry-client/fetch.ts
2753
+ import { KnowledgeRegistryIndexSchema } from "@bendyline/gezk";
1995
2754
 
1996
2755
  // src/signatures/signing.ts
1997
2756
  import {
1998
- createHash as createHash5,
1999
- createPrivateKey,
2000
- createPublicKey,
2001
- sign as cryptoSign,
2002
- verify as cryptoVerify,
2003
- generateKeyPairSync
2004
- } from "crypto";
2005
- function knowledgeKeyId(publicKeyPem) {
2006
- const der = createPublicKey(publicKeyPem).export({ type: "spki", format: "der" });
2007
- return createHash5("sha256").update(der).digest("hex").slice(0, 16);
2008
- }
2009
- function generateKnowledgeSigningKeyPair() {
2010
- const { publicKey, privateKey } = generateKeyPairSync("ed25519");
2011
- const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }).toString();
2012
- return {
2013
- publicKeyPem,
2014
- privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
2015
- keyId: knowledgeKeyId(publicKeyPem)
2016
- };
2017
- }
2018
- function manifestSigningPayload(manifest) {
2019
- const { signature: _omitted, ...unsigned } = manifest;
2020
- return Buffer.from(canonicalizeJson(unsigned), "utf8");
2021
- }
2022
- function signManifest(manifest, privateKeyPem) {
2023
- const key = createPrivateKey(privateKeyPem);
2024
- const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
2025
- const value = cryptoSign(null, manifestSigningPayload(manifest), key).toString("base64");
2026
- return {
2027
- ...manifest,
2028
- signature: {
2029
- algorithm: "ed25519",
2030
- keyId: knowledgeKeyId(publicKeyPem),
2031
- canonicalization: "rfc8785",
2032
- value
2033
- }
2034
- };
2035
- }
2036
- function verifyManifestSignature(manifest, anchors) {
2037
- const signature = manifest.signature;
2038
- if (!signature) return { ok: false, reason: "unsigned" };
2039
- const anchor = anchors.find((a) => a.keyId === signature.keyId);
2040
- if (!anchor) {
2041
- return { ok: false, reason: "unknown-key", detail: `no trust anchor for ${signature.keyId}` };
2757
+ generateKnowledgeSigningKeyPair,
2758
+ knowledgeKeyId,
2759
+ manifestSigningPayload,
2760
+ signManifest,
2761
+ signRegistryIndex,
2762
+ verifyManifestSignature,
2763
+ verifyRegistryIndex
2764
+ } from "@bendyline/gezk/node";
2765
+
2766
+ // src/registry-client/fetch.ts
2767
+ var DEFAULT_MAX_REGISTRY_BYTES = 4 * 1024 * 1024;
2768
+ var DEFAULT_TIMEOUT_MS = 3e4;
2769
+ var KnowledgeRegistryFetchError = class extends Error {
2770
+ code;
2771
+ constructor(code, message) {
2772
+ super(message);
2773
+ this.name = "KnowledgeRegistryFetchError";
2774
+ this.code = code;
2042
2775
  }
2776
+ };
2777
+ async function fetchKnowledgeRegistry(url, opts) {
2778
+ const fetchImpl = opts.fetchImpl ?? fetch;
2779
+ const maxBytes = opts.maxBytes ?? DEFAULT_MAX_REGISTRY_BYTES;
2780
+ let response;
2043
2781
  try {
2044
- if (knowledgeKeyId(anchor.publicKeyPem) !== signature.keyId) {
2045
- return { ok: false, reason: "unknown-key", detail: "anchor keyId does not match its key" };
2046
- }
2047
- const ok = cryptoVerify(
2048
- null,
2049
- manifestSigningPayload(manifest),
2050
- createPublicKey(anchor.publicKeyPem),
2051
- Buffer.from(signature.value, "base64")
2782
+ response = await fetchImpl(url, {
2783
+ signal: opts.signal ?? AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
2784
+ redirect: "follow",
2785
+ headers: { accept: "application/json" }
2786
+ });
2787
+ } catch (err) {
2788
+ throw new KnowledgeRegistryFetchError(
2789
+ "http",
2790
+ `registry fetch failed: ${err instanceof Error ? err.message : String(err)}`
2791
+ );
2792
+ }
2793
+ if (!response.ok) {
2794
+ throw new KnowledgeRegistryFetchError("http", `registry fetch failed: HTTP ${response.status}`);
2795
+ }
2796
+ const declared = Number(response.headers.get("content-length") ?? "0");
2797
+ if (declared > maxBytes) {
2798
+ throw new KnowledgeRegistryFetchError(
2799
+ "too-large",
2800
+ `registry declares ${declared} bytes (cap ${maxBytes})`
2801
+ );
2802
+ }
2803
+ const body = Buffer.from(await response.arrayBuffer());
2804
+ if (body.byteLength > maxBytes) {
2805
+ throw new KnowledgeRegistryFetchError(
2806
+ "too-large",
2807
+ `registry body is ${body.byteLength} bytes (cap ${maxBytes})`
2052
2808
  );
2053
- return ok ? { ok: true, keyId: signature.keyId } : { ok: false, reason: "bad-signature" };
2809
+ }
2810
+ let registry;
2811
+ try {
2812
+ registry = KnowledgeRegistryIndexSchema.parse(JSON.parse(body.toString("utf8")));
2054
2813
  } catch (err) {
2055
- return { ok: false, reason: "error", detail: err instanceof Error ? err.message : String(err) };
2814
+ throw new KnowledgeRegistryFetchError(
2815
+ "invalid",
2816
+ `registry is not a valid gezk-registry document: ${err instanceof Error ? err.message : String(err)}`
2817
+ );
2818
+ }
2819
+ const verdict = verifyRegistryIndex(registry, opts.anchors);
2820
+ if (!verdict.ok) {
2821
+ throw new KnowledgeRegistryFetchError(
2822
+ "unsigned-or-untrusted",
2823
+ `registry signature rejected (${verdict.reason}${verdict.detail ? `: ${verdict.detail}` : ""})`
2824
+ );
2825
+ }
2826
+ return { registry, keyId: verdict.keyId };
2827
+ }
2828
+ function findRegistryEntry(registry, ref) {
2829
+ if (registry.publisher.id !== ref.publisherId) return null;
2830
+ const entry = registry.catalogs.find(
2831
+ (c) => c.catalogId === ref.catalogId && c.version === ref.version
2832
+ );
2833
+ if (!entry) return null;
2834
+ if (ref.contentDigest && entry.contentDigest !== ref.contentDigest.toLowerCase()) return null;
2835
+ return entry;
2836
+ }
2837
+ function compareCatalogVersions(a, b) {
2838
+ const as = a.split(".");
2839
+ const bs = b.split(".");
2840
+ const len = Math.max(as.length, bs.length);
2841
+ for (let i = 0; i < len; i++) {
2842
+ const av = as[i] ?? "";
2843
+ const bv = bs[i] ?? "";
2844
+ const an = /^\d+$/.test(av) ? Number(av) : null;
2845
+ const bn = /^\d+$/.test(bv) ? Number(bv) : null;
2846
+ if (an !== null && bn !== null) {
2847
+ if (an !== bn) return an < bn ? -1 : 1;
2848
+ } else if (av !== bv) {
2849
+ return av < bv ? -1 : 1;
2850
+ }
2056
2851
  }
2852
+ return 0;
2853
+ }
2854
+ function newerRegistryEntries(registry, installed) {
2855
+ return registry.catalogs.filter(
2856
+ (c) => c.catalogId === installed.catalogId && compareCatalogVersions(c.version, installed.version) > 0
2857
+ ).sort((x, y) => compareCatalogVersions(y.version, x.version));
2057
2858
  }
2859
+
2860
+ // src/signatures/anchors.ts
2861
+ var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
2862
+
2863
+ // src/signatures/jcs.ts
2864
+ import { canonicalizeJson } from "@bendyline/gezk";
2058
2865
  export {
2866
+ BGE_SMALL_EN_V15_1,
2059
2867
  BODY_CODEC_MIN_BYTES,
2868
+ BUILTIN_KNOWLEDGE_TRUST_ANCHORS,
2060
2869
  CENTROID_CHUNKS_PER,
2061
2870
  CENTROID_KMEANS_MAX_ITER,
2062
2871
  CENTROID_MAX_PER_SHARD,
2063
2872
  CENTROID_SAMPLE_MAX,
2064
2873
  CatalogHandle,
2065
2874
  CatalogOpenError,
2875
+ DuckdbError,
2876
+ EmbedderArtifactError,
2066
2877
  EmbedderUnavailableError,
2067
2878
  GEZEL_BGE_SMALL_EN_V15_1,
2068
2879
  GEZEL_MARKDOWN_CHUNKS_2,
@@ -2071,10 +2882,20 @@ export {
2071
2882
  GEZK_ARCHIVE_LIMITS,
2072
2883
  GEZK_FORMAT_VERSION,
2073
2884
  GEZK_INDEX_SCHEMA_VERSION,
2885
+ GEZK_MIME_TYPE,
2074
2886
  GezkArchiveError,
2075
2887
  KNOWLEDGE_EMBEDDING_PROFILES,
2888
+ KNOWLEDGE_TOOLCHAIN,
2889
+ KnowledgeRegistryFetchError,
2890
+ LICENSE_NOTICE_PATH,
2076
2891
  MANIFEST_PATH,
2892
+ MARKDOWN_CHUNKS_2,
2077
2893
  MAX_KNOWLEDGE_DOCUMENT_BYTES,
2894
+ MIMETYPE_PATH,
2895
+ MULTILINGUAL_E5_SMALL_1,
2896
+ PARQUET_EXPORT_VERSION,
2897
+ PARQUET_REPORT_PATH,
2898
+ README_PATH,
2078
2899
  RERANK_FINAL_K,
2079
2900
  ROUTER_DB_PATH,
2080
2901
  ROUTER_DDL,
@@ -2083,31 +2904,46 @@ export {
2083
2904
  SHARD_DDL,
2084
2905
  SHARD_MAX_CHUNKS,
2085
2906
  SHARD_TARGET_CHUNKS,
2907
+ SOURCE_NOTICES_PATH,
2086
2908
  ZIP_FIXED_MTIME,
2909
+ assertDuckdbCli,
2087
2910
  canonicalizeJson,
2088
2911
  chunkContentHash,
2089
2912
  chunkMarkdownProfile,
2090
2913
  chunkUid,
2914
+ compareCatalogVersions,
2091
2915
  compileKnowledgeCatalog,
2092
2916
  createProfileEmbedder,
2093
2917
  documentSlug,
2918
+ duckdbVersion,
2919
+ exportCatalogParquet,
2094
2920
  extractGezkVerified,
2921
+ fetchKnowledgeRegistry,
2922
+ findDuckdbBinary,
2923
+ findRegistryEntry,
2095
2924
  generateKnowledgeSigningKeyPair,
2925
+ hammingTopK,
2096
2926
  inspectGezkArchive,
2097
2927
  knowledgeEmbeddingProfile,
2098
2928
  knowledgeKeyId,
2099
2929
  l2Normalize,
2100
2930
  loadMarkdownCatalog,
2101
2931
  manifestSigningPayload,
2932
+ newerRegistryEntries,
2102
2933
  openCatalogDatabase,
2103
2934
  quantizeBinary,
2104
2935
  quantizeInt8,
2105
2936
  readGezkManifest,
2106
2937
  rerankK,
2107
2938
  rerankScore,
2939
+ resolveTransformersModelOptions,
2940
+ runDuckdbScript,
2108
2941
  signManifest,
2942
+ signRegistryIndex,
2943
+ transformersCachePath,
2109
2944
  validateExtractedCatalog,
2110
- vecChunksDdl,
2111
2945
  verifyManifestSignature,
2946
+ verifyProfileArtifacts,
2947
+ verifyRegistryIndex,
2112
2948
  writeGezkArchive
2113
2949
  };