@bendyline/gezel-knowledge 1.0.7 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +28 -17
  2. package/dist/index.d.ts +376 -157
  3. package/dist/index.js +1061 -421
  4. package/package.json +5 -4
package/dist/index.js CHANGED
@@ -1,13 +1,21 @@
1
1
  // src/format/constants.ts
2
- var GEZK_FORMAT_VERSION = 1;
3
- var GEZK_INDEX_SCHEMA_VERSION = 1;
4
- var GEZK_APPLICATION_ID = 1195727435;
5
- var ROUTER_DB_PATH = "index/router.db";
6
- var MANIFEST_PATH = "manifest.json";
2
+ import {
3
+ BODY_CODEC_MIN_BYTES,
4
+ GEZK_APPLICATION_ID,
5
+ GEZK_FORMAT_VERSION,
6
+ GEZK_INDEX_SCHEMA_VERSION,
7
+ GEZK_MIME_TYPE,
8
+ LICENSE_NOTICE_PATH,
9
+ MANIFEST_PATH,
10
+ MAX_KNOWLEDGE_DOCUMENT_BYTES,
11
+ MIMETYPE_PATH,
12
+ README_PATH,
13
+ ROUTER_DB_PATH,
14
+ SOURCE_NOTICES_PATH,
15
+ ZIP_FIXED_MTIME
16
+ } from "@bendyline/gezk";
7
17
  var SHARD_TARGET_CHUNKS = 2e5;
8
18
  var SHARD_MAX_CHUNKS = 25e4;
9
- var BODY_CODEC_MIN_BYTES = 512;
10
- var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
11
19
  var CENTROID_CHUNKS_PER = 12500;
12
20
  var CENTROID_MAX_PER_SHARD = 32;
13
21
  var CENTROID_KMEANS_MAX_ITER = 25;
@@ -18,144 +26,16 @@ var RERANK_FINAL_K = 24;
18
26
  function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
19
27
  return Math.min(512, Math.max(128, 8 * finalK), chunkCount);
20
28
  }
21
- var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
22
29
 
23
30
  // src/format/ddl.ts
24
- var ROUTER_DDL = `
25
- CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
26
-
27
- CREATE TABLE topics(
28
- id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,
29
- description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL
30
- );
31
-
32
- CREATE TABLE documents(
33
- id TEXT PRIMARY KEY,
34
- title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
35
- language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
36
- shard_id INTEGER NOT NULL,
37
- chunk_count INTEGER NOT NULL,
38
- source_url TEXT, source_revision TEXT, source_updated_at TEXT,
39
- attribution_json TEXT,
40
- body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
41
- body_blob BLOB NOT NULL
42
- );
43
- CREATE INDEX documents_topic ON documents(topic_id, slug);
44
- CREATE INDEX documents_shard ON documents(shard_id);
45
-
46
- CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
47
- PRIMARY KEY (alias, document_id)) WITHOUT ROWID;
48
-
49
- CREATE TABLE shards(
50
- id INTEGER PRIMARY KEY,
51
- path TEXT NOT NULL,
52
- chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,
53
- topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,
54
- bytes INTEGER NOT NULL
55
- );
56
-
57
- CREATE TABLE route_centroids(
58
- id INTEGER PRIMARY KEY,
59
- shard_id INTEGER NOT NULL REFERENCES shards(id),
60
- embedding BLOB NOT NULL,
61
- weight INTEGER NOT NULL
62
- );
63
- CREATE INDEX route_centroids_shard ON route_centroids(shard_id);
64
-
65
- CREATE VIRTUAL TABLE fts_documents USING fts5(
66
- title, summary, aliases, document_id UNINDEXED,
67
- tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
68
- );
69
- `;
70
- var SHARD_DDL = `
71
- CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
72
-
73
- CREATE TABLE chunks(
74
- id INTEGER PRIMARY KEY,
75
- chunk_uid TEXT NOT NULL,
76
- document_id TEXT NOT NULL,
77
- ordinal INTEGER NOT NULL,
78
- title TEXT NOT NULL,
79
- heading_path TEXT NOT NULL,
80
- heading_text TEXT NOT NULL,
81
- line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,
82
- token_count INTEGER NOT NULL,
83
- content_hash TEXT NOT NULL,
84
- text TEXT NOT NULL
85
- );
86
- CREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);
87
- CREATE INDEX chunks_document ON chunks(document_id, ordinal);
88
-
89
- CREATE VIRTUAL TABLE fts_chunks USING fts5(
90
- title, heading_text, text,
91
- content = 'chunks', content_rowid = 'id',
92
- tokenize = 'unicode61 remove_diacritics 2'
93
- );
94
-
95
- CREATE TABLE chunk_vectors_int8(
96
- chunk_id INTEGER PRIMARY KEY,
97
- v BLOB NOT NULL
98
- );
99
- `;
100
- function vecChunksDdl(dim) {
101
- return `CREATE VIRTUAL TABLE vec_chunks USING vec0(
102
- chunk_id INTEGER PRIMARY KEY,
103
- embedding bit[${dim}],
104
- chunk_size=1024
105
- );`;
106
- }
31
+ import { ROUTER_DDL, SHARD_DDL } from "@bendyline/gezk";
107
32
 
108
33
  // src/format/ids.ts
109
- import { createHash } from "crypto";
110
- function chunkUid(documentId, ordinal, chunkText) {
111
- const textHash = createHash("sha256").update(chunkText, "utf8").digest();
112
- return createHash("sha256").update(documentId, "utf8").update(Buffer.from([0])).update(String(ordinal), "utf8").update(Buffer.from([0])).update(textHash).digest("hex").slice(0, 32);
113
- }
114
- function chunkContentHash(chunkText) {
115
- return createHash("sha256").update(chunkText, "utf8").digest("hex");
116
- }
117
- function documentSlug(title) {
118
- const slug = title.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
119
- return slug || "untitled";
120
- }
34
+ import { documentSlug } from "@bendyline/gezk";
35
+ import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
121
36
 
122
37
  // src/format/quantize.ts
123
- function quantizeInt8(unitVector) {
124
- const out = new Int8Array(unitVector.length);
125
- for (let i = 0; i < unitVector.length; i++) {
126
- const q = Math.round(unitVector[i] * 127);
127
- out[i] = q > 127 ? 127 : q < -127 ? -127 : q;
128
- }
129
- return out;
130
- }
131
- function quantizeBinary(unitVector) {
132
- const bytes = Math.ceil(unitVector.length / 8);
133
- const out = new Uint8Array(bytes);
134
- for (let i = 0; i < unitVector.length; i++) {
135
- if (unitVector[i] > 0) {
136
- out[i >> 3] = out[i >> 3] | 1 << (i & 7);
137
- }
138
- }
139
- return out;
140
- }
141
- function rerankScore(query, passageInt8) {
142
- let dot = 0;
143
- const n = Math.min(query.length, passageInt8.length);
144
- for (let i = 0; i < n; i++) dot += query[i] * (passageInt8[i] / 127);
145
- return dot;
146
- }
147
- function l2Normalize(vector) {
148
- let sum = 0;
149
- for (let i = 0; i < vector.length; i++) {
150
- const v = vector[i];
151
- sum += v * v;
152
- }
153
- const norm = Math.sqrt(sum);
154
- if (!Number.isFinite(norm) || norm === 0) throw new Error("degenerate vector (zero norm)");
155
- const out = new Float32Array(vector.length);
156
- for (let i = 0; i < vector.length; i++) out[i] = vector[i] / norm;
157
- return out;
158
- }
38
+ import { l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from "@bendyline/gezk";
159
39
 
160
40
  // src/chunking/markdown-chunker.ts
161
41
  var HEADING = /^(#{1,6})\s+(.*)$/;
@@ -284,12 +164,12 @@ function splitOversizeLine(line, opts) {
284
164
  }
285
165
 
286
166
  // src/compiler/compile.ts
287
- import { createHash as createHash3 } from "crypto";
288
- import { mkdirSync, rmSync } from "fs";
167
+ import { createHash } from "crypto";
168
+ import { mkdirSync, rmSync, statSync } from "fs";
289
169
  import { stat } from "fs/promises";
290
170
  import { join } from "path";
291
171
  import { brotliCompressSync, constants as zlibConstants } from "zlib";
292
- import { KnowledgeCatalogManifestSchema } from "@bendyline/gezel";
172
+ import { GEZK_MANIFEST_KIND, KnowledgeCatalogManifestSchema } from "@bendyline/gezk";
293
173
 
294
174
  // src/archive/write.ts
295
175
  import { createWriteStream } from "fs";
@@ -301,9 +181,13 @@ async function writeGezkArchive(outputPath, entries) {
301
181
  const yazl = nodeRequire("yazl");
302
182
  await mkdir(dirname(outputPath), { recursive: true });
303
183
  const zip = new yazl.ZipFile();
304
- const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress: true };
184
+ const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress: false };
185
+ zip.addBuffer(Buffer.from(GEZK_MIME_TYPE, "utf8"), MIMETYPE_PATH, options);
305
186
  const sorted = [...entries].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
306
187
  for (const entry of sorted) {
188
+ if (entry.path === MIMETYPE_PATH) {
189
+ throw new Error(`the writer owns the ${MIMETYPE_PATH} entry`);
190
+ }
307
191
  if (entry.path.includes("\\") || entry.path.startsWith("/") || entry.path.includes("..")) {
308
192
  throw new Error(`unsafe archive path: ${entry.path}`);
309
193
  }
@@ -311,10 +195,10 @@ async function writeGezkArchive(outputPath, entries) {
311
195
  else if (entry.absPath) zip.addFile(entry.absPath, entry.path, options);
312
196
  else throw new Error(`archive entry ${entry.path} has neither content nor absPath`);
313
197
  }
314
- await new Promise((resolve3, reject) => {
198
+ await new Promise((resolve4, reject) => {
315
199
  const out = createWriteStream(outputPath);
316
200
  out.on("error", reject);
317
- out.on("close", () => resolve3());
201
+ out.on("close", () => resolve4());
318
202
  zip.outputStream.on("error", reject);
319
203
  zip.outputStream.pipe(out);
320
204
  zip.end();
@@ -322,27 +206,7 @@ async function writeGezkArchive(outputPath, entries) {
322
206
  }
323
207
 
324
208
  // src/format/file-hash.ts
325
- import { createHash as createHash2 } from "crypto";
326
- import { createReadStream } from "fs";
327
- async function hashFileStreaming(path, maxBytes = Number.MAX_SAFE_INTEGER) {
328
- const hasher = createHash2("sha256");
329
- let sizeBytes = 0;
330
- await new Promise((resolve3, reject) => {
331
- const stream = createReadStream(path, { highWaterMark: 4 * 1024 * 1024 });
332
- stream.on("data", (chunk) => {
333
- const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
334
- sizeBytes += bytes.byteLength;
335
- if (sizeBytes > maxBytes) {
336
- stream.destroy(new Error(`file exceeds ${maxBytes} byte limit`));
337
- return;
338
- }
339
- hasher.update(bytes);
340
- });
341
- stream.on("error", reject);
342
- stream.on("end", resolve3);
343
- });
344
- return { sha256: hasher.digest("hex"), sizeBytes };
345
- }
209
+ import { hashFileStreaming } from "@bendyline/gezk/node";
346
210
 
347
211
  // src/format/node-sqlite.ts
348
212
  import { createRequire as createRequire2 } from "module";
@@ -371,25 +235,13 @@ function documentSmokeQueryMisses(db, smoke) {
371
235
  return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
372
236
  }
373
237
 
374
- // src/compiler/vec-load.ts
238
+ // src/toolchain.ts
375
239
  import { createRequire as createRequire3 } from "module";
376
240
  var nodeRequire3 = createRequire3(import.meta.url);
377
- function loadVecExtension(db) {
378
- try {
379
- db.enableLoadExtension(true);
380
- const sqliteVec = nodeRequire3("sqlite-vec");
381
- db.loadExtension(sqliteVec.getLoadablePath());
382
- } catch (err) {
383
- throw new Error(
384
- `sqlite-vec extension failed to load \u2014 the compiler cannot build vector tables: ${err instanceof Error ? err.message : String(err)}`
385
- );
386
- } finally {
387
- try {
388
- db.enableLoadExtension(false);
389
- } catch {
390
- }
391
- }
392
- }
241
+ var KNOWLEDGE_TOOLCHAIN = {
242
+ name: "@bendyline/gezel-knowledge",
243
+ version: nodeRequire3("../package.json").version
244
+ };
393
245
 
394
246
  // src/compiler/compile.ts
395
247
  async function compileKnowledgeCatalog(opts) {
@@ -401,9 +253,9 @@ async function compileKnowledgeCatalog(opts) {
401
253
  const topicIds = new Set(opts.topics.map((t) => t.id));
402
254
  const profile = opts.embeddingProfile;
403
255
  const chunkerOpts = {
404
- unit: "tokens",
405
- target: opts.chunkingProfile.targetTokens,
406
- overlap: opts.chunkingProfile.overlapTokens,
256
+ unit: opts.chunkingProfile.unit,
257
+ target: opts.chunkingProfile.target,
258
+ overlap: opts.chunkingProfile.overlap,
407
259
  countTokens: opts.countTokens
408
260
  };
409
261
  const prepared = [];
@@ -463,7 +315,6 @@ async function compileKnowledgeCatalog(opts) {
463
315
  router.exec("BEGIN");
464
316
  if (embedded) {
465
317
  router.exec(SHARD_DDL.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
466
- router.exec(vecChunksDdl(profile.dimensions));
467
318
  }
468
319
  const commonMeta = {
469
320
  format_version: String(GEZK_FORMAT_VERSION),
@@ -560,7 +411,6 @@ async function compileKnowledgeCatalog(opts) {
560
411
  if (!db) {
561
412
  db = createCatalogDb(join(opts.workDir, shardPath(id)));
562
413
  db.exec(SHARD_DDL);
563
- db.exec(vecChunksDdl(profile.dimensions));
564
414
  db.exec("BEGIN");
565
415
  shardDbs.set(id, db);
566
416
  }
@@ -571,7 +421,7 @@ async function compileKnowledgeCatalog(opts) {
571
421
  let stmts = vecStmts.get(db);
572
422
  if (!stmts) {
573
423
  stmts = {
574
- vec: db.prepare("INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, vec_bit(?))"),
424
+ vec: db.prepare("INSERT INTO chunk_vectors_bit (chunk_id, v) VALUES (?, ?)"),
575
425
  int8: db.prepare("INSERT INTO chunk_vectors_int8 (chunk_id, v) VALUES (?, ?)")
576
426
  };
577
427
  vecStmts.set(db, stmts);
@@ -650,7 +500,7 @@ async function compileKnowledgeCatalog(opts) {
650
500
  const header = buildContextHeader(
651
501
  p.doc.title,
652
502
  chunk.headingPath,
653
- opts.chunkingProfile.contextHeader.maxTokens,
503
+ opts.chunkingProfile.contextHeader.max,
654
504
  opts.countTokens
655
505
  );
656
506
  pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
@@ -659,6 +509,30 @@ async function compileKnowledgeCatalog(opts) {
659
509
  }
660
510
  }
661
511
  await flushEmbeds();
512
+ for (let id = 0; id < shardCount; id++) {
513
+ if (!embedded)
514
+ putMeta(shardDb(id), {
515
+ shard_id: String(id),
516
+ chunk_count: String(shardChunkCounts.get(id) ?? 0)
517
+ });
518
+ }
519
+ if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
520
+ const finalDir = join(opts.workDir, "final");
521
+ mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
522
+ const vacuumed = [];
523
+ const vacuumInto = (db, archivePath) => {
524
+ db.exec("COMMIT");
525
+ const dest = join(finalDir, archivePath);
526
+ db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
527
+ vacuumed.push({ archivePath, absPath: dest });
528
+ return dest;
529
+ };
530
+ const shardBytes = /* @__PURE__ */ new Map();
531
+ if (!embedded) {
532
+ for (let id = 0; id < shardCount; id++) {
533
+ shardBytes.set(id, statSync(vacuumInto(shardDb(id), shardPath(id))).size);
534
+ }
535
+ }
662
536
  {
663
537
  const insertShard = router.prepare(
664
538
  "INSERT INTO shards (id, path, chunk_count, document_count, topic_ids_json, centroid_count, bytes) VALUES (?, ?, ?, ?, ?, ?, ?)"
@@ -681,35 +555,14 @@ async function compileKnowledgeCatalog(opts) {
681
555
  BigInt(shardDocCounts.get(id) ?? 0),
682
556
  JSON.stringify([...shardTopicIds.get(id) ?? []].sort()),
683
557
  BigInt(centroids.length),
684
- BigInt(0)
685
- // authoritative sizes live in the manifest (post-VACUUM)
558
+ BigInt(shardBytes.get(id) ?? 0)
686
559
  );
687
560
  for (const c of centroids) {
688
561
  insertCentroid.run(BigInt(id), Buffer.from(c.centroid.buffer.slice(0)), BigInt(c.weight));
689
562
  }
690
563
  }
691
564
  }
692
- for (let id = 0; id < shardCount; id++) {
693
- if (!embedded)
694
- putMeta(shardDb(id), {
695
- shard_id: String(id),
696
- chunk_count: String(shardChunkCounts.get(id) ?? 0)
697
- });
698
- }
699
- if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
700
- const finalDir = join(opts.workDir, "final");
701
- mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
702
- const vacuumed = [];
703
- const vacuumInto = (db, archivePath) => {
704
- db.exec("COMMIT");
705
- const dest = join(finalDir, archivePath);
706
- db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
707
- vacuumed.push({ archivePath, absPath: dest });
708
- };
709
565
  vacuumInto(router, ROUTER_DB_PATH);
710
- if (!embedded) {
711
- for (let id = 0; id < shardCount; id++) vacuumInto(shardDb(id), shardPath(id));
712
- }
713
566
  router.close();
714
567
  for (const db of shardDbs.values()) db.close();
715
568
  let smokeQueries = opts.smokeQueries;
@@ -765,17 +618,21 @@ async function compileKnowledgeCatalog(opts) {
765
618
  });
766
619
  }
767
620
  }
768
- const extraFiles = opts.extraFiles ?? {};
621
+ const extraFiles = {
622
+ [README_PATH]: defaultReadme(opts),
623
+ [LICENSE_NOTICE_PATH]: defaultLicenseNotice(opts),
624
+ ...opts.extraFiles ?? {}
625
+ };
769
626
  for (const [path, content] of Object.entries(extraFiles)) {
770
627
  const bytes = Buffer.from(content, "utf8");
771
628
  files.push({
772
629
  path,
773
630
  sizeBytes: bytes.length,
774
- sha256: createHash3("sha256").update(bytes).digest("hex")
631
+ sha256: createHash("sha256").update(bytes).digest("hex")
775
632
  });
776
633
  }
777
634
  const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
778
- kind: "gezel-knowledge-catalog",
635
+ kind: GEZK_MANIFEST_KIND,
779
636
  formatVersion: GEZK_FORMAT_VERSION,
780
637
  indexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION,
781
638
  id: opts.catalog.id,
@@ -786,7 +643,10 @@ async function compileKnowledgeCatalog(opts) {
786
643
  publisher: opts.catalog.publisher,
787
644
  createdAt: opts.catalog.createdAt,
788
645
  ...opts.catalog.sourceSnapshot ? { sourceSnapshot: opts.catalog.sourceSnapshot } : {},
789
- license: opts.catalog.license,
646
+ license: {
647
+ ...opts.catalog.license,
648
+ noticePath: opts.catalog.license.noticePath ?? LICENSE_NOTICE_PATH
649
+ },
790
650
  embedding: profile,
791
651
  chunking: opts.chunkingProfile,
792
652
  topics: opts.topics.map((t) => ({ id: t.id, name: t.name })),
@@ -797,9 +657,9 @@ async function compileKnowledgeCatalog(opts) {
797
657
  },
798
658
  counts: { documents: prepared.length, chunks: totalChunks, shards: shardCount },
799
659
  files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
800
- compatibility: { maximumIndexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION },
660
+ requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
801
661
  ...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
802
- ...opts.toolchain ? { toolchain: opts.toolchain } : {}
662
+ toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
803
663
  });
804
664
  const manifest = opts.finalizeManifest ? KnowledgeCatalogManifestSchema.parse(opts.finalizeManifest(unsignedManifest)) : unsignedManifest;
805
665
  const archiveEntries = [
@@ -829,14 +689,37 @@ async function compileKnowledgeCatalog(opts) {
829
689
  }
830
690
  }
831
691
  function createCatalogDb(absPath) {
832
- const db = new DatabaseSync(absPath, { allowExtension: true });
833
- loadVecExtension(db);
692
+ const db = new DatabaseSync(absPath);
834
693
  db.exec("PRAGMA page_size=8192");
835
694
  db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
836
695
  db.exec(`PRAGMA user_version=${GEZK_INDEX_SCHEMA_VERSION}`);
837
696
  db.exec("PRAGMA journal_mode=DELETE");
838
697
  return db;
839
698
  }
699
+ function defaultReadme(opts) {
700
+ const c = opts.catalog;
701
+ return [
702
+ `# ${c.name}`,
703
+ "",
704
+ ...c.description ? [c.description, ""] : [],
705
+ `A gezk knowledge catalog (${c.id}@${c.version}) published by ${c.publisher.name}.`,
706
+ `Language: ${c.language}. License: ${c.license.name} (see ${c.license.noticePath ?? LICENSE_NOTICE_PATH}).`,
707
+ ...c.sourceSnapshot ? [`Source snapshot: ${c.sourceSnapshot.name} (${c.sourceSnapshot.date}).`] : [],
708
+ ""
709
+ ].join("\n");
710
+ }
711
+ function defaultLicenseNotice(opts) {
712
+ const c = opts.catalog;
713
+ const publisher = c.publisher.url ? `${c.publisher.name} <${c.publisher.url}>` : c.publisher.name;
714
+ const license = c.license.spdx ? `${c.license.name} (SPDX: ${c.license.spdx})` : c.license.name;
715
+ return [
716
+ `${c.name} (${c.id}@${c.version})`,
717
+ `Publisher: ${publisher}`,
718
+ `License: ${license}`,
719
+ ...c.license.attributionRequired ? ["Attribution is required when redistributing this content."] : [],
720
+ ""
721
+ ].join("\n");
722
+ }
840
723
  function normalizeMarkdown(md) {
841
724
  return md.replace(/\r\n/g, "\n").normalize("NFC");
842
725
  }
@@ -854,7 +737,7 @@ ${path.join(" > ")}
854
737
  return header;
855
738
  }
856
739
  function seedFor(catalogId, shardId) {
857
- const digest = createHash3("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
740
+ const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
858
741
  return digest.readBigUInt64LE(24);
859
742
  }
860
743
  function makePrng(seed) {
@@ -939,14 +822,14 @@ function kMeans(samples, k, seed) {
939
822
  }
940
823
 
941
824
  // src/archive/read.ts
942
- import { createHash as createHash4 } from "crypto";
825
+ import { createHash as createHash2 } from "crypto";
943
826
  import { createWriteStream as createWriteStream2 } from "fs";
944
827
  import { mkdir as mkdir2 } from "fs/promises";
945
828
  import { createRequire as createRequire4 } from "module";
946
829
  import { dirname as dirname2, isAbsolute, normalize, relative, resolve } from "path";
947
830
  import { Transform } from "stream";
948
831
  import { pipeline } from "stream/promises";
949
- import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/gezel";
832
+ import { GEZK_MANIFEST_KIND as GEZK_MANIFEST_KIND2, KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/gezk";
950
833
  var nodeRequire4 = createRequire4(import.meta.url);
951
834
  var GEZK_ARCHIVE_LIMITS = {
952
835
  maxEntries: 16384,
@@ -1036,7 +919,7 @@ async function inspectEntries(archivePath) {
1036
919
  function reconcileEntries(manifest, entries) {
1037
920
  const declared = new Map(manifest.files.map((file) => [file.path, file]));
1038
921
  for (const path of entries.keys()) {
1039
- if (path === MANIFEST_PATH) continue;
922
+ if (path === MANIFEST_PATH || path === MIMETYPE_PATH) continue;
1040
923
  if (!declared.has(path)) {
1041
924
  throw new GezkArchiveError(`archive contains undeclared file: ${path}`, "undeclared-file");
1042
925
  }
@@ -1066,14 +949,88 @@ async function inspectGezkArchive(archivePath) {
1066
949
  totalUncompressedBytes: inspected.totalUncompressedBytes
1067
950
  };
1068
951
  }
952
+ function assertMimetypeEntry(entry, position) {
953
+ if (position !== 0) {
954
+ throw new GezkArchiveError("the mimetype entry must be the first archive entry", "mimetype");
955
+ }
956
+ if (entry.compressionMethod !== 0 || entry.uncompressedSize !== GEZK_MIME_TYPE.length) {
957
+ throw new GezkArchiveError("the mimetype entry must be stored, not compressed", "mimetype");
958
+ }
959
+ }
960
+ function assertSupportedFormat(raw) {
961
+ const kind = raw?.kind;
962
+ const version = raw?.formatVersion;
963
+ if (kind !== GEZK_MANIFEST_KIND2 || version !== GEZK_FORMAT_VERSION) {
964
+ throw new GezkArchiveError(
965
+ `unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${GEZK_FORMAT_VERSION} \u2014 catalogs built for an earlier version must be rebuilt`,
966
+ "format-version"
967
+ );
968
+ }
969
+ }
970
+ function readEntryText(zip, entry) {
971
+ return new Promise((resolveText, rejectText) => {
972
+ zip.openReadStream(entry, (err, stream) => {
973
+ if (err || !stream) {
974
+ rejectText(new GezkArchiveError(`${entry.fileName} read failed: ${err?.message}`, "io"));
975
+ return;
976
+ }
977
+ const parts = [];
978
+ stream.on("data", (d) => parts.push(d));
979
+ stream.on("end", () => resolveText(Buffer.concat(parts).toString("utf8")));
980
+ stream.on(
981
+ "error",
982
+ (streamErr) => rejectText(
983
+ new GezkArchiveError(`${entry.fileName} read failed: ${streamErr.message}`, "io")
984
+ )
985
+ );
986
+ });
987
+ });
988
+ }
1069
989
  async function readGezkManifest(archivePath) {
1070
990
  const zip = await openZip(archivePath);
1071
991
  return new Promise((resolvePromise, reject) => {
992
+ let position = 0;
993
+ let mimetypeSeen = false;
994
+ const fail = (err) => {
995
+ zip.close();
996
+ reject(
997
+ err instanceof GezkArchiveError ? err : new GezkArchiveError(err instanceof Error ? err.message : String(err), "io")
998
+ );
999
+ };
1072
1000
  zip.on("entry", (entry) => {
1001
+ const index = position++;
1002
+ if (entry.fileName === MIMETYPE_PATH) {
1003
+ try {
1004
+ assertMimetypeEntry(entry, index);
1005
+ } catch (err) {
1006
+ fail(err);
1007
+ return;
1008
+ }
1009
+ readEntryText(zip, entry).then((text) => {
1010
+ if (text !== GEZK_MIME_TYPE) {
1011
+ fail(
1012
+ new GezkArchiveError(
1013
+ `not a gezk archive (mimetype ${JSON.stringify(text)})`,
1014
+ "mimetype"
1015
+ )
1016
+ );
1017
+ return;
1018
+ }
1019
+ mimetypeSeen = true;
1020
+ zip.readEntry();
1021
+ }, fail);
1022
+ return;
1023
+ }
1073
1024
  if (entry.fileName !== MANIFEST_PATH) {
1074
1025
  zip.readEntry();
1075
1026
  return;
1076
1027
  }
1028
+ if (!mimetypeSeen) {
1029
+ fail(
1030
+ new GezkArchiveError("archive does not start with the gezk mimetype entry", "mimetype")
1031
+ );
1032
+ return;
1033
+ }
1077
1034
  if (entry.uncompressedSize > GEZK_ARCHIVE_LIMITS.maxManifestBytes) {
1078
1035
  zip.close();
1079
1036
  reject(new GezkArchiveError("manifest exceeds size limit", "limits"));
@@ -1090,12 +1047,14 @@ async function readGezkManifest(archivePath) {
1090
1047
  stream.on("end", () => {
1091
1048
  zip.close();
1092
1049
  try {
1093
- resolvePromise(
1094
- KnowledgeCatalogManifestSchema2.parse(
1095
- JSON.parse(Buffer.concat(parts).toString("utf8"))
1096
- )
1097
- );
1050
+ const raw = JSON.parse(Buffer.concat(parts).toString("utf8"));
1051
+ assertSupportedFormat(raw);
1052
+ resolvePromise(KnowledgeCatalogManifestSchema2.parse(raw));
1098
1053
  } catch (parseErr) {
1054
+ if (parseErr instanceof GezkArchiveError) {
1055
+ reject(parseErr);
1056
+ return;
1057
+ }
1099
1058
  reject(
1100
1059
  new GezkArchiveError(
1101
1060
  `invalid manifest: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,
@@ -1153,7 +1112,7 @@ async function extractGezkVerified(archivePath, destDir) {
1153
1112
  pipeline(stream, createWriteStream2(target)).then(res, rej);
1154
1113
  return;
1155
1114
  }
1156
- const hasher = createHash4("sha256");
1115
+ const hasher = createHash2("sha256");
1157
1116
  let sizeBytes = 0;
1158
1117
  const verify = new Transform({
1159
1118
  transform(chunk, _encoding, callback) {
@@ -1214,9 +1173,7 @@ async function extractGezkVerified(archivePath, destDir) {
1214
1173
  }
1215
1174
 
1216
1175
  // src/reader/open.ts
1217
- import { createRequire as createRequire5 } from "module";
1218
1176
  import { pathToFileURL } from "url";
1219
- var nodeRequire5 = createRequire5(import.meta.url);
1220
1177
  var CatalogOpenError = class extends Error {
1221
1178
  constructor(message, reason) {
1222
1179
  super(message);
@@ -1229,7 +1186,7 @@ function openCatalogDatabase(absPath) {
1229
1186
  const uri = `${pathToFileURL(absPath).href}?immutable=1`;
1230
1187
  let db;
1231
1188
  try {
1232
- db = new DatabaseSync(uri, { readOnly: true, allowExtension: true });
1189
+ db = new DatabaseSync(uri, { readOnly: true });
1233
1190
  } catch (err) {
1234
1191
  throw new CatalogOpenError(
1235
1192
  `cannot open catalog database ${absPath}: ${err instanceof Error ? err.message : String(err)}`,
@@ -1237,20 +1194,6 @@ function openCatalogDatabase(absPath) {
1237
1194
  );
1238
1195
  }
1239
1196
  try {
1240
- let vecLoaded = false;
1241
- try {
1242
- db.enableLoadExtension(true);
1243
- const sqliteVec = nodeRequire5("sqlite-vec");
1244
- db.loadExtension(sqliteVec.getLoadablePath());
1245
- vecLoaded = true;
1246
- } catch {
1247
- vecLoaded = false;
1248
- } finally {
1249
- try {
1250
- db.enableLoadExtension(false);
1251
- } catch {
1252
- }
1253
- }
1254
1197
  const appId = readPragmaNumber(db, "application_id");
1255
1198
  if (appId !== GEZK_APPLICATION_ID) {
1256
1199
  throw new CatalogOpenError(
@@ -1261,13 +1204,10 @@ function openCatalogDatabase(absPath) {
1261
1204
  const schemaVersion = readPragmaNumber(db, "user_version");
1262
1205
  if (schemaVersion !== GEZK_INDEX_SCHEMA_VERSION) {
1263
1206
  throw new CatalogOpenError(
1264
- `unsupported index schema version ${schemaVersion} (reader supports ${GEZK_INDEX_SCHEMA_VERSION})`,
1207
+ `unsupported index schema version ${schemaVersion} (this reader supports ${GEZK_INDEX_SCHEMA_VERSION}; catalogs built for gezk 0.4 and earlier must be rebuilt)`,
1265
1208
  "schema-version"
1266
1209
  );
1267
1210
  }
1268
- if (!vecLoaded) {
1269
- throw new CatalogOpenError("sqlite-vec extension failed to load", "vec-unavailable");
1270
- }
1271
1211
  db.exec("PRAGMA mmap_size=536870912");
1272
1212
  db.exec("PRAGMA cache_size=-8192");
1273
1213
  return {
@@ -1296,6 +1236,80 @@ function readPragmaNumber(db, pragma) {
1296
1236
  // src/reader/catalog-handle.ts
1297
1237
  import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1298
1238
  import { brotliDecompressSync } from "zlib";
1239
+
1240
+ // src/reader/bit-scan.ts
1241
+ var POPCOUNT8 = new Uint8Array(256);
1242
+ for (let i = 1; i < 256; i++) POPCOUNT8[i] = (i & 1) + POPCOUNT8[i >> 1];
1243
+ function popcount32(x) {
1244
+ let v = x - (x >>> 1 & 1431655765);
1245
+ v = (v & 858993459) + (v >>> 2 & 858993459);
1246
+ return Math.imul(v + (v >>> 4) & 252645135, 16843009) >>> 24;
1247
+ }
1248
+ function hammingAll(index, query, out) {
1249
+ const { bits, bytesPerRow, rows } = index;
1250
+ if (bytesPerRow % 4 === 0 && bits.byteOffset % 4 === 0) {
1251
+ const perRow = bytesPerRow / 4;
1252
+ const words = new Uint32Array(bits.buffer, bits.byteOffset, rows * perRow);
1253
+ const aligned = new Uint8Array(bytesPerRow);
1254
+ aligned.set(query);
1255
+ const qwords = new Uint32Array(aligned.buffer, 0, perRow);
1256
+ let offset2 = 0;
1257
+ for (let r = 0; r < rows; r++) {
1258
+ let d = 0;
1259
+ for (let w = 0; w < perRow; w++) {
1260
+ d += popcount32((words[offset2 + w] ^ qwords[w]) >>> 0);
1261
+ }
1262
+ out[r] = d;
1263
+ offset2 += perRow;
1264
+ }
1265
+ return;
1266
+ }
1267
+ let offset = 0;
1268
+ for (let r = 0; r < rows; r++) {
1269
+ let d = 0;
1270
+ for (let b = 0; b < bytesPerRow; b++) {
1271
+ d += POPCOUNT8[bits[offset + b] ^ query[b]];
1272
+ }
1273
+ out[r] = d;
1274
+ offset += bytesPerRow;
1275
+ }
1276
+ }
1277
+ function hammingTopK(index, query, k) {
1278
+ if (query.length !== index.bytesPerRow) {
1279
+ throw new Error(`query has ${query.length} bytes, shard rows have ${index.bytesPerRow}`);
1280
+ }
1281
+ const limit = Math.min(k, index.rows);
1282
+ if (limit <= 0) return [];
1283
+ const distances = new Uint16Array(index.rows);
1284
+ hammingAll(index, query, distances);
1285
+ const maxDistance = index.bytesPerRow * 8;
1286
+ const histogram = new Uint32Array(maxDistance + 1);
1287
+ for (let r = 0; r < index.rows; r++) {
1288
+ const d = distances[r];
1289
+ histogram[d] = histogram[d] + 1;
1290
+ }
1291
+ let threshold = 0;
1292
+ let seen = 0;
1293
+ for (; threshold <= maxDistance; threshold++) {
1294
+ seen += histogram[threshold];
1295
+ if (seen >= limit) break;
1296
+ }
1297
+ let atThreshold = limit - (seen - histogram[threshold]);
1298
+ const hits = [];
1299
+ for (let r = 0; r < index.rows && hits.length < limit; r++) {
1300
+ const d = distances[r];
1301
+ if (d < threshold) hits.push({ chunkId: r + 1, distance: d });
1302
+ else if (d === threshold && atThreshold > 0) {
1303
+ hits.push({ chunkId: r + 1, distance: d });
1304
+ atThreshold--;
1305
+ }
1306
+ }
1307
+ hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
1308
+ return hits;
1309
+ }
1310
+
1311
+ // src/reader/catalog-handle.ts
1312
+ var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
1299
1313
  var CatalogHandle = class _CatalogHandle {
1300
1314
  constructor(rootDir, router) {
1301
1315
  this.rootDir = rootDir;
@@ -1310,6 +1324,7 @@ var CatalogHandle = class _CatalogHandle {
1310
1324
  rootDir;
1311
1325
  router;
1312
1326
  connections = /* @__PURE__ */ new Map();
1327
+ bitIndexes = /* @__PURE__ */ new Map();
1313
1328
  meta;
1314
1329
  shards;
1315
1330
  /** Open a catalog's extracted version directory (the dir holding manifest.json). */
@@ -1320,6 +1335,7 @@ var CatalogHandle = class _CatalogHandle {
1320
1335
  close() {
1321
1336
  for (const conn of this.connections.values()) conn.close();
1322
1337
  this.connections.clear();
1338
+ this.bitIndexes.clear();
1323
1339
  }
1324
1340
  /** Integrity check for the router connection, used by deep validation. */
1325
1341
  routerQuickCheck() {
@@ -1350,6 +1366,57 @@ var CatalogHandle = class _CatalogHandle {
1350
1366
  }
1351
1367
  return conn.db;
1352
1368
  }
1369
+ /**
1370
+ * A shard's sign-bit rows as one contiguous array, loaded on first use and
1371
+ * kept while the resident budget allows. Chunk ids must be dense from 1 —
1372
+ * the row index IS the id — so a gap is a corrupt shard, not a sparse one.
1373
+ */
1374
+ shardBits(shard) {
1375
+ const cached = this.bitIndexes.get(shard.path);
1376
+ if (cached) return cached;
1377
+ const db = this.shardDb(shard);
1378
+ const span = db.prepare(
1379
+ "SELECT COUNT(*) AS n, MIN(chunk_id) AS lo, MAX(chunk_id) AS hi FROM chunk_vectors_bit"
1380
+ ).get();
1381
+ const rows = Number(span.n);
1382
+ if (rows > 0 && (Number(span.lo) !== 1 || Number(span.hi) !== rows)) {
1383
+ throw new CatalogOpenError(`chunk ids are not dense in ${shard.path}`, "corrupt");
1384
+ }
1385
+ const bytesPerRow = Math.ceil(this.dimensions() / 8);
1386
+ const bits = new Uint8Array(rows * bytesPerRow);
1387
+ const rowsIter = db.prepare("SELECT chunk_id, v FROM chunk_vectors_bit ORDER BY chunk_id").iterate();
1388
+ for (const row of rowsIter) {
1389
+ if (row.v.byteLength !== bytesPerRow) {
1390
+ throw new CatalogOpenError(
1391
+ `bit vector width ${row.v.byteLength} != ${bytesPerRow} in ${shard.path}`,
1392
+ "corrupt"
1393
+ );
1394
+ }
1395
+ bits.set(row.v, (Number(row.chunk_id) - 1) * bytesPerRow);
1396
+ }
1397
+ const index = { bits, bytesPerRow, rows };
1398
+ let resident = bits.byteLength;
1399
+ for (const other of this.bitIndexes.values()) resident += other.bits.byteLength;
1400
+ for (const [key, other] of this.bitIndexes) {
1401
+ if (resident <= BIT_INDEX_BUDGET_BYTES) break;
1402
+ this.bitIndexes.delete(key);
1403
+ resident -= other.bits.byteLength;
1404
+ }
1405
+ this.bitIndexes.set(shard.path, index);
1406
+ return index;
1407
+ }
1408
+ /** The embedding dimension from the router's profile echo. */
1409
+ dimensions() {
1410
+ const raw = this.meta.embedding_profile_json;
1411
+ if (raw) {
1412
+ try {
1413
+ const dims = JSON.parse(raw).dimensions;
1414
+ if (typeof dims === "number" && dims > 0) return dims;
1415
+ } catch {
1416
+ }
1417
+ }
1418
+ throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
1419
+ }
1353
1420
  /** Resolve a manifest/router path without allowing it to escape the catalog. */
1354
1421
  resolveCatalogPath(path) {
1355
1422
  const root = resolve2(this.rootDir);
@@ -1487,16 +1554,16 @@ var CatalogHandle = class _CatalogHandle {
1487
1554
  * after global cross-catalog routing has already spent the S budget.
1488
1555
  */
1489
1556
  searchShards(queryVector, shardIds, finalK) {
1490
- const queryBits = Buffer.from(quantizeBinary(queryVector));
1557
+ const queryBits = quantizeBinary(queryVector);
1491
1558
  const hits = [];
1492
1559
  for (const shardId of shardIds) {
1493
1560
  const shard = this.shards.find((s) => s.id === shardId);
1494
1561
  if (!shard) continue;
1495
1562
  const db = this.shardDb(shard);
1496
1563
  const k = rerankK(finalK, shard.chunkCount);
1497
- const candidates = db.prepare(
1498
- "SELECT chunk_id FROM vec_chunks WHERE embedding MATCH vec_bit(?) AND k = ? ORDER BY distance"
1499
- ).all(queryBits, BigInt(k));
1564
+ const candidates = hammingTopK(this.shardBits(shard), queryBits, k).map((hit) => ({
1565
+ chunk_id: hit.chunkId
1566
+ }));
1500
1567
  if (candidates.length === 0) continue;
1501
1568
  const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
1502
1569
  const reranked = candidates.map((c) => {
@@ -1563,21 +1630,19 @@ var CatalogHandle = class _CatalogHandle {
1563
1630
  return hits;
1564
1631
  }
1565
1632
  /**
1566
- * Embedder-free integrity smoke (§6.3): chunk 1's own bit vector must be
1567
- * its own nearest neighbor — proves the extension parses this file's vec
1568
- * blobs. Returns false instead of throwing (callers report a reason).
1633
+ * Embedder-free integrity smoke: chunk 1's own bit vector must be its own
1634
+ * nearest neighbor — proves the shard's vectors load and scan. Returns
1635
+ * false instead of throwing (callers report a reason).
1569
1636
  */
1570
1637
  selfKnnSmoke(shardId) {
1571
1638
  const shard = this.shards.find((s) => s.id === shardId);
1572
1639
  if (!shard) return false;
1573
1640
  try {
1574
- const db = this.shardDb(shard);
1575
- const first = db.prepare("SELECT embedding FROM vec_chunks WHERE chunk_id = 1").get();
1576
- if (!first) return false;
1577
- const nearest = db.prepare(
1578
- "SELECT chunk_id FROM vec_chunks WHERE embedding MATCH vec_bit(?) AND k = 1 ORDER BY distance"
1579
- ).get(Buffer.from(first.embedding));
1580
- return nearest !== void 0 && Number(nearest.chunk_id) === 1;
1641
+ const index = this.shardBits(shard);
1642
+ if (index.rows === 0) return false;
1643
+ const first = index.bits.subarray(0, index.bytesPerRow);
1644
+ const nearest = hammingTopK(index, first, 1)[0];
1645
+ return nearest !== void 0 && nearest.chunkId === 1 && nearest.distance === 0;
1581
1646
  } catch {
1582
1647
  return false;
1583
1648
  }
@@ -1618,7 +1683,7 @@ var ftsQuery = sanitizeFtsQuery;
1618
1683
  // src/reader/validate.ts
1619
1684
  import { readFile } from "fs/promises";
1620
1685
  import { join as join3 } from "path";
1621
- import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/gezel";
1686
+ import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/gezk";
1622
1687
  async function validateExtractedCatalog(rootDir, opts = {}) {
1623
1688
  const checks = [];
1624
1689
  const check = (name, ok, detail) => {
@@ -1687,6 +1752,11 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1687
1752
  );
1688
1753
  const topics = handle.topics();
1689
1754
  check("toc-present", topics.length >= 1, `${topics.length} topics`);
1755
+ check(
1756
+ "license-notice",
1757
+ manifest.files.some((f) => f.path === manifest.license.noticePath),
1758
+ `manifest.files lacks the declared notice ${manifest.license.noticePath}`
1759
+ );
1690
1760
  const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
1691
1761
  check(
1692
1762
  "counts-documents",
@@ -1737,7 +1807,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1737
1807
  db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n
1738
1808
  );
1739
1809
  const vecCount = Number(
1740
- db.prepare("SELECT COUNT(*) AS n FROM vec_chunks").get().n
1810
+ db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit").get().n
1741
1811
  );
1742
1812
  const int8Count = Number(
1743
1813
  db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8").get().n
@@ -1747,6 +1817,24 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1747
1817
  chunkCount === shard.chunkCount && vecCount === chunkCount && int8Count === chunkCount,
1748
1818
  `chunks ${chunkCount}, vec ${vecCount}, int8 ${int8Count}, shards row ${shard.chunkCount}`
1749
1819
  );
1820
+ const dims = manifest.embedding.dimensions;
1821
+ const badBit = Number(
1822
+ db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit WHERE length(v) != ?").get(Math.ceil(dims / 8)).n
1823
+ );
1824
+ const badInt8 = Number(
1825
+ db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8 WHERE length(v) != ?").get(dims).n
1826
+ );
1827
+ check(
1828
+ `vector-widths:${shard.path}`,
1829
+ badBit === 0 && badInt8 === 0,
1830
+ `${badBit} bit rows and ${badInt8} int8 rows have the wrong width for ${dims} dimensions`
1831
+ );
1832
+ const span = db.prepare("SELECT MIN(id) AS lo, MAX(id) AS hi FROM chunks").get();
1833
+ check(
1834
+ `chunk-ids-dense:${shard.path}`,
1835
+ chunkCount === 0 || Number(span.lo) === 1 && Number(span.hi) === chunkCount,
1836
+ `ids span ${String(span.lo)}..${String(span.hi)} for ${chunkCount} chunks`
1837
+ );
1750
1838
  } finally {
1751
1839
  conn.close();
1752
1840
  }
@@ -1768,56 +1856,642 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1768
1856
  return report(manifest);
1769
1857
  }
1770
1858
 
1859
+ // src/export/duckdb.ts
1860
+ import { spawn } from "child_process";
1861
+ import { access } from "fs/promises";
1862
+ var DuckdbError = class extends Error {
1863
+ constructor(message, stderr) {
1864
+ super(message);
1865
+ this.stderr = stderr;
1866
+ this.name = "DuckdbError";
1867
+ }
1868
+ stderr;
1869
+ };
1870
+ async function duckdbVersion(binaryPath) {
1871
+ const { stdout } = await runDuckdbProcess(binaryPath, ["--version"], "");
1872
+ const match = /v?(\d+\.\d+\.\d+)/.exec(stdout);
1873
+ if (!match?.[1])
1874
+ throw new DuckdbError(`could not read the DuckDB version from: ${stdout.trim()}`, "");
1875
+ return match[1];
1876
+ }
1877
+ async function assertDuckdbCli(cli) {
1878
+ await access(cli.binaryPath).catch(() => {
1879
+ throw new DuckdbError(`DuckDB CLI not found at ${cli.binaryPath}`, "");
1880
+ });
1881
+ const version = await duckdbVersion(cli.binaryPath);
1882
+ if (cli.expectedVersion && version !== cli.expectedVersion) {
1883
+ throw new DuckdbError(
1884
+ `DuckDB ${cli.expectedVersion} expected, but ${cli.binaryPath} is ${version}; Parquet output is only reproducible against the pinned release`,
1885
+ ""
1886
+ );
1887
+ }
1888
+ return version;
1889
+ }
1890
+ async function runDuckdbScript(cli, sql) {
1891
+ const { stdout } = await runDuckdbProcess(cli.binaryPath, ["-batch", ":memory:"], sql);
1892
+ return stdout;
1893
+ }
1894
+ function runDuckdbProcess(binaryPath, args, stdin) {
1895
+ return new Promise((resolve4, reject) => {
1896
+ const child = spawn(binaryPath, args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
1897
+ let stdout = "";
1898
+ let stderr = "";
1899
+ child.stdout.setEncoding("utf8");
1900
+ child.stderr.setEncoding("utf8");
1901
+ child.stdout.on("data", (chunk) => {
1902
+ stdout += chunk;
1903
+ });
1904
+ child.stderr.on("data", (chunk) => {
1905
+ stderr += chunk;
1906
+ });
1907
+ child.on(
1908
+ "error",
1909
+ (err) => reject(new DuckdbError(`could not start DuckDB: ${err.message}`, stderr))
1910
+ );
1911
+ child.on("close", (code) => {
1912
+ if (code === 0) resolve4({ stdout });
1913
+ else
1914
+ reject(
1915
+ new DuckdbError(
1916
+ `DuckDB exited with code ${code}: ${stderr.trim() || stdout.trim()}`,
1917
+ stderr
1918
+ )
1919
+ );
1920
+ });
1921
+ child.stdin.on("error", () => {
1922
+ });
1923
+ child.stdin.end(stdin);
1924
+ });
1925
+ }
1926
+
1927
+ // src/export/parquet.ts
1928
+ import { createHash as createHash3 } from "crypto";
1929
+ import { createWriteStream as createWriteStream3 } from "fs";
1930
+ import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
1931
+ import { join as join4, resolve as resolve3 } from "path";
1932
+ import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
1933
+ import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
1934
+ var PARQUET_EXPORT_VERSION = 1;
1935
+ var DEFAULT_ROW_GROUP_SIZE = 32768;
1936
+ var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
1937
+ var PARQUET_REPORT_PATH = "parquet-manifest.json";
1938
+ async function exportCatalogParquet(opts) {
1939
+ const duckdbVersion2 = await assertDuckdbCli(opts.duckdb);
1940
+ const outDir = resolve3(opts.outDir);
1941
+ await mkdir3(outDir, { recursive: true });
1942
+ const work = join4(outDir, `.export-${process.pid}-${Date.now().toString(36)}`);
1943
+ await mkdir3(work, { recursive: true });
1944
+ try {
1945
+ let rootDir;
1946
+ if ("archivePath" in opts.source) {
1947
+ opts.onProgress?.({ phase: "extract" });
1948
+ rootDir = join4(work, "catalog");
1949
+ await extractGezkVerified(opts.source.archivePath, rootDir);
1950
+ } else {
1951
+ rootDir = resolve3(opts.source.rootDir);
1952
+ }
1953
+ const manifest = KnowledgeCatalogManifestSchema4.parse(
1954
+ JSON.parse(await readFile2(join4(rootDir, MANIFEST_PATH), "utf8"))
1955
+ );
1956
+ const staging = join4(work, "ndjson");
1957
+ await mkdir3(staging, { recursive: true });
1958
+ const files = [];
1959
+ const metadata = kvMetadata(manifest);
1960
+ const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
1961
+ const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
1962
+ try {
1963
+ const topicPaths = topicPathsOf(router);
1964
+ const topicRows = await stageTopics(router, join4(staging, "topics.ndjson"));
1965
+ opts.onProgress?.({ phase: "stage", table: "topics", rows: topicRows });
1966
+ files.push(
1967
+ await writeParquet(opts.duckdb, {
1968
+ outDir,
1969
+ file: "topics.parquet",
1970
+ table: "topics",
1971
+ rows: topicRows,
1972
+ sql: topicsCopy(
1973
+ join4(staging, "topics.ndjson"),
1974
+ join4(outDir, "topics.parquet"),
1975
+ rowGroupSize,
1976
+ metadata
1977
+ ),
1978
+ onProgress: opts.onProgress
1979
+ })
1980
+ );
1981
+ for (const shard of manifest.router.shards) {
1982
+ const tag = String(shard.id).padStart(3, "0");
1983
+ const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
1984
+ const documentRows = await stageDocuments(router, shard.id, topicPaths, documentsNdjson);
1985
+ opts.onProgress?.({
1986
+ phase: "stage",
1987
+ table: "documents",
1988
+ shardId: shard.id,
1989
+ rows: documentRows
1990
+ });
1991
+ files.push(
1992
+ await writeParquet(opts.duckdb, {
1993
+ outDir,
1994
+ file: `documents-${tag}.parquet`,
1995
+ table: "documents",
1996
+ shardId: shard.id,
1997
+ rows: documentRows,
1998
+ sql: documentsCopy(
1999
+ documentsNdjson,
2000
+ join4(outDir, `documents-${tag}.parquet`),
2001
+ rowGroupSize,
2002
+ metadata
2003
+ ),
2004
+ onProgress: opts.onProgress
2005
+ })
2006
+ );
2007
+ const shardDb = openCatalogDatabase(join4(rootDir, shard.path));
2008
+ const chunksNdjson = join4(staging, `chunks-${tag}.ndjson`);
2009
+ let chunkRows;
2010
+ try {
2011
+ chunkRows = await stageChunks(
2012
+ shardDb,
2013
+ shard.id,
2014
+ manifest.embedding.dimensions,
2015
+ chunksNdjson
2016
+ );
2017
+ } finally {
2018
+ shardDb.close();
2019
+ }
2020
+ opts.onProgress?.({ phase: "stage", table: "chunks", shardId: shard.id, rows: chunkRows });
2021
+ files.push(
2022
+ await writeParquet(opts.duckdb, {
2023
+ outDir,
2024
+ file: `chunks-${tag}.parquet`,
2025
+ table: "chunks",
2026
+ shardId: shard.id,
2027
+ rows: chunkRows,
2028
+ sql: chunksCopy(
2029
+ chunksNdjson,
2030
+ join4(outDir, `chunks-${tag}.parquet`),
2031
+ manifest.embedding.dimensions,
2032
+ rowGroupSize,
2033
+ metadata
2034
+ ),
2035
+ onProgress: opts.onProgress
2036
+ })
2037
+ );
2038
+ }
2039
+ } finally {
2040
+ router.close();
2041
+ }
2042
+ const report = {
2043
+ exportVersion: PARQUET_EXPORT_VERSION,
2044
+ catalogId: manifest.id,
2045
+ version: manifest.version,
2046
+ publisherId: manifest.publisher.id,
2047
+ formatVersion: manifest.formatVersion,
2048
+ embeddingProfileId: manifest.embedding.id,
2049
+ dimensions: manifest.embedding.dimensions,
2050
+ files,
2051
+ duckdbVersion: duckdbVersion2,
2052
+ exporter: { name: KNOWLEDGE_TOOLCHAIN.name, version: KNOWLEDGE_TOOLCHAIN.version }
2053
+ };
2054
+ const { writeFile } = await import("fs/promises");
2055
+ await writeFile(
2056
+ join4(outDir, PARQUET_REPORT_PATH),
2057
+ `${JSON.stringify(report, null, 2)}
2058
+ `,
2059
+ "utf8"
2060
+ );
2061
+ return report;
2062
+ } finally {
2063
+ await rm(work, { recursive: true, force: true }).catch(() => {
2064
+ });
2065
+ }
2066
+ }
2067
+ function ndjsonWriter(path) {
2068
+ const stream = createWriteStream3(path, { encoding: "utf8" });
2069
+ return {
2070
+ write(row) {
2071
+ const line = `${JSON.stringify(row)}
2072
+ `;
2073
+ if (stream.write(line)) return Promise.resolve();
2074
+ return new Promise((resolveDrain) => stream.once("drain", () => resolveDrain()));
2075
+ },
2076
+ close() {
2077
+ return new Promise((resolveClose, reject) => {
2078
+ stream.on("error", reject);
2079
+ stream.end(() => resolveClose());
2080
+ });
2081
+ }
2082
+ };
2083
+ }
2084
+ function topicPathsOf(router) {
2085
+ const parents = /* @__PURE__ */ new Map();
2086
+ for (const row of router.db.prepare("SELECT id, parent_id FROM topics").all()) {
2087
+ parents.set(row.id, row.parent_id);
2088
+ }
2089
+ const paths = /* @__PURE__ */ new Map();
2090
+ for (const id of parents.keys()) {
2091
+ const path = [];
2092
+ let cursor = id;
2093
+ const seen = /* @__PURE__ */ new Set();
2094
+ while (cursor && !seen.has(cursor)) {
2095
+ seen.add(cursor);
2096
+ path.unshift(cursor);
2097
+ cursor = parents.get(cursor) ?? null;
2098
+ }
2099
+ paths.set(id, path);
2100
+ }
2101
+ return paths;
2102
+ }
2103
+ async function stageTopics(router, path) {
2104
+ const out = ndjsonWriter(path);
2105
+ let rows = 0;
2106
+ const statement = router.db.prepare(
2107
+ "SELECT id, parent_id, name, description, sort_key, document_count FROM topics ORDER BY sort_key, id"
2108
+ );
2109
+ for (const row of statement.iterate()) {
2110
+ await out.write({
2111
+ id: row.id,
2112
+ parent_id: row.parent_id ?? null,
2113
+ name: row.name,
2114
+ description: row.description ?? null,
2115
+ sort_key: row.sort_key,
2116
+ document_count: Number(row.document_count)
2117
+ });
2118
+ rows++;
2119
+ }
2120
+ await out.close();
2121
+ return rows;
2122
+ }
2123
+ async function stageDocuments(router, shardId, topicPaths, path) {
2124
+ const out = ndjsonWriter(path);
2125
+ let rows = 0;
2126
+ const aliasesOf = router.db.prepare(
2127
+ "SELECT alias FROM aliases WHERE document_id = ? ORDER BY alias"
2128
+ );
2129
+ const statement = router.db.prepare(
2130
+ `SELECT id, title, slug, summary, language, topic_id, shard_id, chunk_count,
2131
+ source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob
2132
+ FROM documents WHERE shard_id = ? ORDER BY id`
2133
+ );
2134
+ for (const row of statement.iterate(shardId)) {
2135
+ const aliases = aliasesOf.all(row.id).map(
2136
+ (a) => a.alias
2137
+ );
2138
+ await out.write({
2139
+ id: row.id,
2140
+ title: row.title,
2141
+ slug: row.slug,
2142
+ summary: row.summary ?? null,
2143
+ language: row.language,
2144
+ topic_id: row.topic_id,
2145
+ topic_path: topicPaths.get(row.topic_id) ?? [row.topic_id],
2146
+ shard_id: Number(row.shard_id),
2147
+ chunk_count: Number(row.chunk_count),
2148
+ markdown: decodeBody(row.body_codec, row.body_blob),
2149
+ source_url: row.source_url ?? null,
2150
+ source_revision: row.source_revision ?? null,
2151
+ source_updated_at: row.source_updated_at ?? null,
2152
+ attribution: row.attribution_json ?? null,
2153
+ aliases
2154
+ });
2155
+ rows++;
2156
+ }
2157
+ await out.close();
2158
+ return rows;
2159
+ }
2160
+ function decodeBody(codec, blob) {
2161
+ const bytes = Buffer.from(blob);
2162
+ if (codec === "br") {
2163
+ return brotliDecompressSync2(bytes, { maxOutputLength: MAX_KNOWLEDGE_DOCUMENT_BYTES }).toString(
2164
+ "utf8"
2165
+ );
2166
+ }
2167
+ if (codec === "none") return bytes.toString("utf8");
2168
+ throw new Error(`unknown body codec ${codec}`);
2169
+ }
2170
+ async function stageChunks(shard, shardId, dimensions, path) {
2171
+ const out = ndjsonWriter(path);
2172
+ const bitBytes = Math.ceil(dimensions / 8);
2173
+ let rows = 0;
2174
+ const statement = shard.db.prepare(
2175
+ `SELECT c.id, c.chunk_uid, c.document_id, c.ordinal, c.title, c.heading_path, c.heading_text,
2176
+ c.text, c.token_count, c.content_hash, c.line_start, c.line_end,
2177
+ i.v AS int8, b.v AS bits
2178
+ FROM chunks c
2179
+ JOIN chunk_vectors_int8 i ON i.chunk_id = c.id
2180
+ JOIN chunk_vectors_bit b ON b.chunk_id = c.id
2181
+ ORDER BY c.id`
2182
+ );
2183
+ for (const row of statement.iterate()) {
2184
+ const int8 = Buffer.from(row.int8);
2185
+ const bits = Buffer.from(row.bits);
2186
+ if (int8.length !== dimensions || bits.length !== bitBytes) {
2187
+ throw new Error(
2188
+ `chunk ${row.chunk_uid}: vector widths ${int8.length}/${bits.length} do not match the ${dimensions}-dimension profile`
2189
+ );
2190
+ }
2191
+ await out.write({
2192
+ id: Number(row.id),
2193
+ chunk_uid: row.chunk_uid,
2194
+ document_id: row.document_id,
2195
+ ordinal: Number(row.ordinal),
2196
+ title: row.title,
2197
+ heading_path: parseJsonStrings(row.heading_path),
2198
+ heading_text: row.heading_text,
2199
+ text: row.text,
2200
+ token_count: Number(row.token_count),
2201
+ content_hash: row.content_hash,
2202
+ line_start: Number(row.line_start),
2203
+ line_end: Number(row.line_end),
2204
+ shard_id: shardId,
2205
+ embedding: Array.from(new Int8Array(int8.buffer, int8.byteOffset, int8.length)),
2206
+ embedding_bit: bits.toString("base64")
2207
+ });
2208
+ rows++;
2209
+ }
2210
+ await out.close();
2211
+ return rows;
2212
+ }
2213
+ function parseJsonStrings(raw) {
2214
+ try {
2215
+ const parsed = JSON.parse(raw);
2216
+ return Array.isArray(parsed) ? parsed.filter((p) => typeof p === "string") : [];
2217
+ } catch {
2218
+ return [];
2219
+ }
2220
+ }
2221
+ function sqlString(value) {
2222
+ return `'${value.replace(/'/g, "''")}'`;
2223
+ }
2224
+ function kvMetadata(manifest) {
2225
+ return {
2226
+ gezk_catalog_id: manifest.id,
2227
+ gezk_catalog_version: manifest.version,
2228
+ gezk_publisher_id: manifest.publisher.id,
2229
+ gezk_format_version: manifest.formatVersion,
2230
+ gezk_embedding_profile: manifest.embedding.id,
2231
+ gezk_embedding_model: manifest.embedding.model.repo,
2232
+ gezk_embedding_dimensions: String(manifest.embedding.dimensions),
2233
+ gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
2234
+ gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
2235
+ gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
2236
+ gezk_license: manifest.license.name
2237
+ };
2238
+ }
2239
+ function copyOptions(rowGroupSize, metadata) {
2240
+ const kv = Object.entries(metadata).map(([key, value]) => `${key}: ${sqlString(value)}`).join(", ");
2241
+ return `(FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 3, ROW_GROUP_SIZE ${rowGroupSize}, KV_METADATA {${kv}})`;
2242
+ }
2243
+ var PRELUDE = "SET threads = 1;\nSET preserve_insertion_order = true;\n";
2244
+ function readNdjson(path, columns) {
2245
+ const spec = Object.entries(columns).map(([name, type]) => `${name}: ${sqlString(type)}`).join(", ");
2246
+ return `read_ndjson(${sqlString(path)}, columns = {${spec}}, maximum_object_size = ${NDJSON_MAX_OBJECT_BYTES})`;
2247
+ }
2248
+ function topicsCopy(ndjson, out, rowGroupSize, metadata) {
2249
+ const source = readNdjson(ndjson, {
2250
+ id: "VARCHAR",
2251
+ parent_id: "VARCHAR",
2252
+ name: "VARCHAR",
2253
+ description: "VARCHAR",
2254
+ sort_key: "VARCHAR",
2255
+ document_count: "BIGINT"
2256
+ });
2257
+ return `${PRELUDE}COPY (SELECT id, parent_id, name, description, sort_key, document_count FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2258
+ `;
2259
+ }
2260
+ function documentsCopy(ndjson, out, rowGroupSize, metadata) {
2261
+ const source = readNdjson(ndjson, {
2262
+ id: "VARCHAR",
2263
+ title: "VARCHAR",
2264
+ slug: "VARCHAR",
2265
+ summary: "VARCHAR",
2266
+ language: "VARCHAR",
2267
+ topic_id: "VARCHAR",
2268
+ topic_path: "VARCHAR[]",
2269
+ shard_id: "INTEGER",
2270
+ chunk_count: "INTEGER",
2271
+ markdown: "VARCHAR",
2272
+ source_url: "VARCHAR",
2273
+ source_revision: "VARCHAR",
2274
+ source_updated_at: "VARCHAR",
2275
+ attribution: "VARCHAR",
2276
+ aliases: "VARCHAR[]"
2277
+ });
2278
+ return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2279
+ `;
2280
+ }
2281
+ function chunksCopy(ndjson, out, dimensions, rowGroupSize, metadata) {
2282
+ const source = readNdjson(ndjson, {
2283
+ id: "BIGINT",
2284
+ chunk_uid: "VARCHAR",
2285
+ document_id: "VARCHAR",
2286
+ ordinal: "INTEGER",
2287
+ title: "VARCHAR",
2288
+ heading_path: "VARCHAR[]",
2289
+ heading_text: "VARCHAR",
2290
+ text: "VARCHAR",
2291
+ token_count: "INTEGER",
2292
+ content_hash: "VARCHAR",
2293
+ line_start: "INTEGER",
2294
+ line_end: "INTEGER",
2295
+ shard_id: "INTEGER",
2296
+ embedding: "TINYINT[]",
2297
+ embedding_bit: "VARCHAR"
2298
+ });
2299
+ return `${PRELUDE}COPY (SELECT id, chunk_uid, document_id, ordinal, title, heading_path, heading_text, text, token_count, content_hash, line_start, line_end, shard_id, embedding::TINYINT[${dimensions}] AS embedding, from_base64(embedding_bit) AS embedding_bit FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2300
+ `;
2301
+ }
2302
+ async function writeParquet(cli, spec) {
2303
+ spec.onProgress?.({ phase: "write", file: spec.file });
2304
+ const target = join4(spec.outDir, spec.file);
2305
+ await rm(target, { force: true });
2306
+ await runDuckdbScript(cli, spec.sql);
2307
+ const bytes = await readFile2(target);
2308
+ return {
2309
+ path: spec.file,
2310
+ table: spec.table,
2311
+ ...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
2312
+ rows: spec.rows,
2313
+ sizeBytes: (await stat2(target)).size,
2314
+ sha256: createHash3("sha256").update(bytes).digest("hex")
2315
+ };
2316
+ }
2317
+ async function findDuckdbBinary(env = process.env) {
2318
+ const configured = env.GEZEL_DUCKDB_BIN?.trim();
2319
+ if (configured) return configured;
2320
+ const { readdir: readdir2 } = await import("fs/promises");
2321
+ const { homedir } = await import("os");
2322
+ const name = process.platform === "win32" ? "duckdb.exe" : "duckdb";
2323
+ const homes = [env.GEZEL_HOME, join4(homedir(), ".gezel-dev"), join4(homedir(), ".gezel")].filter(
2324
+ (h) => Boolean(h)
2325
+ );
2326
+ for (const home of homes) {
2327
+ const engines = join4(home, "engines", "duckdb");
2328
+ const versions = (await readdir2(engines).catch(() => [])).sort().reverse();
2329
+ for (const version of versions) {
2330
+ const candidate = join4(engines, version, name);
2331
+ if (await stat2(candidate).catch(() => null)) return candidate;
2332
+ }
2333
+ }
2334
+ const pathDirs = (env.PATH ?? "").split(process.platform === "win32" ? ";" : ":");
2335
+ for (const dir of pathDirs) {
2336
+ if (!dir) continue;
2337
+ const candidate = join4(dir, name);
2338
+ if (await stat2(candidate).catch(() => null)) return candidate;
2339
+ }
2340
+ return null;
2341
+ }
2342
+
1771
2343
  // src/profiles/registry.ts
1772
2344
  var SHARED_ENCODING = {
1773
2345
  pooling: "mean",
1774
2346
  normalized: true,
1775
2347
  dimensions: 384,
1776
2348
  maxTokens: 512,
1777
- vectorEncoding: "bit384+int8",
2349
+ vectorEncoding: "bit+int8",
1778
2350
  distance: { stage1: "hamming", stage2: "cosine" },
1779
2351
  quantization: {
1780
2352
  int8: { method: "symmetric-linear", scale: 127 },
1781
2353
  binary: { method: "sign", threshold: 0, packing: "lsb-first" }
1782
2354
  }
1783
2355
  };
1784
- var GEZEL_MULTILINGUAL_E5_SMALL_1 = {
1785
- id: "gezel-multilingual-e5-small@1",
2356
+ var MULTILINGUAL_E5_SMALL_1 = {
2357
+ id: "multilingual-e5-small@1",
1786
2358
  model: {
1787
2359
  repo: "Xenova/multilingual-e5-small",
1788
- revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78"
2360
+ revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
2361
+ onnxFile: "onnx/model.onnx",
2362
+ onnxDigest: "sha256:4aa845c27760e06e9a686b9d8b5d440eae4b6612cd09e5b522b716d3941f77ff"
2363
+ },
2364
+ tokenizer: {
2365
+ kind: "sentencepiece-xlmr",
2366
+ file: "tokenizer.json",
2367
+ digest: "sha256:0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39"
1789
2368
  },
1790
- tokenizer: { kind: "sentencepiece-xlmr" },
1791
2369
  queryInstruction: "query: ",
1792
2370
  passageInstruction: "passage: ",
1793
2371
  ...SHARED_ENCODING
1794
2372
  };
1795
- var GEZEL_BGE_SMALL_EN_V15_1 = {
1796
- id: "gezel-bge-small-en-v1.5@1",
2373
+ var BGE_SMALL_EN_V15_1 = {
2374
+ id: "bge-small-en-v1.5@1",
1797
2375
  model: {
1798
2376
  repo: "Xenova/bge-small-en-v1.5",
1799
- revision: "ea104dacec62c0de699686887e3f920caeb4f3e3"
2377
+ revision: "ea104dacec62c0de699686887e3f920caeb4f3e3",
2378
+ onnxFile: "onnx/model.onnx",
2379
+ onnxDigest: "sha256:828e1496d7fabb79cfa4dcd84fa38625c0d3d21da474a00f08db0f559940cf35"
2380
+ },
2381
+ tokenizer: {
2382
+ kind: "wordpiece-bert",
2383
+ file: "tokenizer.json",
2384
+ digest: "sha256:d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66"
1800
2385
  },
1801
- tokenizer: { kind: "wordpiece-bert" },
1802
2386
  queryInstruction: "Represent this sentence for searching relevant passages: ",
1803
2387
  passageInstruction: "",
1804
2388
  ...SHARED_ENCODING
1805
2389
  };
1806
2390
  var KNOWLEDGE_EMBEDDING_PROFILES = [
1807
- GEZEL_MULTILINGUAL_E5_SMALL_1,
1808
- GEZEL_BGE_SMALL_EN_V15_1
2391
+ MULTILINGUAL_E5_SMALL_1,
2392
+ BGE_SMALL_EN_V15_1
1809
2393
  ];
1810
2394
  function knowledgeEmbeddingProfile(id) {
1811
2395
  return KNOWLEDGE_EMBEDDING_PROFILES.find((p) => p.id === id) ?? null;
1812
2396
  }
1813
- var GEZEL_MARKDOWN_CHUNKS_2 = {
1814
- id: "gezel-markdown-chunks@2",
2397
+ var MARKDOWN_CHUNKS_2 = {
2398
+ id: "markdown-chunks@2",
1815
2399
  unit: "tokens",
1816
2400
  tokenizer: "profile",
1817
- targetTokens: 420,
1818
- overlapTokens: 64,
1819
- contextHeader: { maxTokens: 64 }
2401
+ target: 420,
2402
+ overlap: 64,
2403
+ contextHeader: { max: 64 }
1820
2404
  };
2405
+ var GEZEL_MULTILINGUAL_E5_SMALL_1 = MULTILINGUAL_E5_SMALL_1;
2406
+ var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
2407
+ var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
2408
+
2409
+ // src/embedding/profile-embedder.ts
2410
+ import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
2411
+
2412
+ // src/embedding/artifact-verify.ts
2413
+ import { createHash as createHash4 } from "crypto";
2414
+ import { createReadStream } from "fs";
2415
+ import { stat as stat3 } from "fs/promises";
2416
+ import { join as join5 } from "path";
2417
+ import { embeddingProfileArtifacts } from "@bendyline/gezk";
2418
+ var EmbedderArtifactError = class extends Error {
2419
+ constructor(reason, profileId, checks) {
2420
+ super(describeFailure(reason, profileId, checks));
2421
+ this.reason = reason;
2422
+ this.profileId = profileId;
2423
+ this.checks = checks;
2424
+ this.name = "EmbedderArtifactError";
2425
+ }
2426
+ reason;
2427
+ profileId;
2428
+ checks;
2429
+ isActionable = true;
2430
+ };
2431
+ function describeFailure(reason, profileId, checks) {
2432
+ const detail = checks.filter((c) => c.actual !== c.expected).map(
2433
+ (c) => c.actual === null ? `${c.file} not found at ${c.path}` : `${c.file} is ${c.actual}, profile pins ${c.expected}`
2434
+ ).join("; ");
2435
+ return reason === "missing" ? `cannot verify the model files for profile ${profileId}: ${detail}` : `the model files loaded for profile ${profileId} differ from the ones it pins: ${detail}`;
2436
+ }
2437
+ function transformersCachePath(cacheDir, repo, revision, file) {
2438
+ return revision === "main" ? join5(cacheDir, repo, file) : join5(cacheDir, repo, revision, file);
2439
+ }
2440
+ var digestMemo = /* @__PURE__ */ new Map();
2441
+ var DIGEST_MEMO_MAX = 32;
2442
+ async function fileDigest(path) {
2443
+ let size;
2444
+ let mtimeMs;
2445
+ try {
2446
+ const info = await stat3(path);
2447
+ if (!info.isFile()) return null;
2448
+ size = info.size;
2449
+ mtimeMs = info.mtimeMs;
2450
+ } catch {
2451
+ return null;
2452
+ }
2453
+ const key = `${path} ${size} ${mtimeMs}`;
2454
+ const memo = digestMemo.get(key);
2455
+ if (memo) return memo;
2456
+ const digest = await new Promise((resolve4, reject) => {
2457
+ const hash = createHash4("sha256");
2458
+ createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve4(`sha256:${hash.digest("hex")}`));
2459
+ });
2460
+ if (digestMemo.size >= DIGEST_MEMO_MAX) {
2461
+ const oldest = digestMemo.keys().next().value;
2462
+ if (oldest !== void 0) digestMemo.delete(oldest);
2463
+ }
2464
+ digestMemo.set(key, digest);
2465
+ return digest;
2466
+ }
2467
+ async function verifyProfileArtifacts(profile, opts) {
2468
+ const artifacts = embeddingProfileArtifacts(profile);
2469
+ const revision = opts.revision ?? profile.model.revision;
2470
+ const pinned = [];
2471
+ if (artifacts.onnxDigest) {
2472
+ pinned.push({ role: "onnx", file: artifacts.onnxFile, expected: artifacts.onnxDigest });
2473
+ }
2474
+ if (artifacts.tokenizerDigest) {
2475
+ pinned.push({
2476
+ role: "tokenizer",
2477
+ file: artifacts.tokenizerFile,
2478
+ expected: artifacts.tokenizerDigest
2479
+ });
2480
+ }
2481
+ if (pinned.length === 0) return { status: "unpinned", checks: [] };
2482
+ const checks = [];
2483
+ for (const { role, file, expected } of pinned) {
2484
+ const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file);
2485
+ checks.push({ role, file, path, expected, actual: await fileDigest(path) });
2486
+ }
2487
+ if (checks.some((c) => c.actual === null)) {
2488
+ throw new EmbedderArtifactError("missing", profile.id, checks);
2489
+ }
2490
+ if (checks.some((c) => c.actual !== c.expected)) {
2491
+ throw new EmbedderArtifactError("mismatch", profile.id, checks);
2492
+ }
2493
+ return { status: "verified", checks };
2494
+ }
1821
2495
 
1822
2496
  // src/embedding/profile-embedder.ts
1823
2497
  var MAX_BATCH = 8;
@@ -1829,6 +2503,42 @@ var EmbedderUnavailableError = class extends Error {
1829
2503
  this.name = "EmbedderUnavailableError";
1830
2504
  }
1831
2505
  };
2506
+ var ONNX_DTYPE_BY_SUFFIX = [
2507
+ ["_q4f16", "q4f16"],
2508
+ ["_quantized", "q8"],
2509
+ ["_uint8", "uint8"],
2510
+ ["_int8", "int8"],
2511
+ ["_bnb4", "bnb4"],
2512
+ ["_fp16", "fp16"],
2513
+ ["_q4", "q4"]
2514
+ ];
2515
+ var POOLING_BY_PROFILE = {
2516
+ mean: "mean",
2517
+ cls: "cls"
2518
+ };
2519
+ function resolveTransformersModelOptions(profile) {
2520
+ const { onnxFile, tokenizerFile } = embeddingProfileArtifacts2(profile);
2521
+ if (tokenizerFile !== "tokenizer.json") {
2522
+ throw new EmbedderUnavailableError(
2523
+ `profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
2524
+ );
2525
+ }
2526
+ if (!onnxFile.endsWith(".onnx")) {
2527
+ throw new EmbedderUnavailableError(
2528
+ `profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
2529
+ );
2530
+ }
2531
+ const slash = onnxFile.lastIndexOf("/");
2532
+ const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
2533
+ const base = onnxFile.slice(slash + 1, -".onnx".length);
2534
+ const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
2535
+ const [suffix, dtype] = variant ?? ["", "fp32"];
2536
+ const modelFileName = base.slice(0, base.length - suffix.length);
2537
+ if (modelFileName === "") {
2538
+ throw new EmbedderUnavailableError(`profile ${profile.id} pins an unnamed graph ${onnxFile}`);
2539
+ }
2540
+ return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
2541
+ }
1832
2542
  async function loadTransformers() {
1833
2543
  try {
1834
2544
  const specifier = "@huggingface/transformers";
@@ -1844,33 +2554,52 @@ async function loadTransformers() {
1844
2554
  }
1845
2555
  }
1846
2556
  async function createProfileEmbedder(profile, opts = {}) {
1847
- const transformers = await loadTransformers();
2557
+ const pooling = POOLING_BY_PROFILE[profile.pooling];
2558
+ if (!pooling) {
2559
+ throw new EmbedderUnavailableError(
2560
+ `profile ${profile.id} uses ${profile.pooling} pooling, which this runtime does not implement`
2561
+ );
2562
+ }
2563
+ const modelOptions = resolveTransformersModelOptions(profile);
2564
+ const transformers = opts.transformers ?? await loadTransformers();
1848
2565
  const cacheDir = opts.cacheDir ?? process.env.GEZEL_HF_CACHE_DIR;
1849
2566
  if (cacheDir) {
1850
2567
  transformers.env.cacheDir = cacheDir;
1851
2568
  transformers.env.useFSCache = true;
1852
2569
  transformers.env.allowRemoteModels = true;
1853
2570
  }
1854
- const modelOpts = {
1855
- revision: profile.model.revision,
1856
- ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
1857
- };
1858
2571
  const [pipe, tokenizer] = await Promise.all([
1859
- transformers.pipeline("feature-extraction", profile.model.repo, modelOpts),
1860
- transformers.AutoTokenizer.from_pretrained(profile.model.repo, modelOpts)
2572
+ transformers.pipeline("feature-extraction", profile.model.repo, {
2573
+ ...modelOptions,
2574
+ ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
2575
+ }),
2576
+ transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
2577
+ revision: modelOptions.revision
2578
+ })
1861
2579
  ]);
2580
+ let verification;
2581
+ try {
2582
+ verification = await verifyProfileArtifacts(profile, {
2583
+ cacheDir: cacheDir ?? transformers.env.cacheDir ?? ""
2584
+ });
2585
+ } catch (err) {
2586
+ await pipe.dispose?.().catch(() => {
2587
+ });
2588
+ throw err;
2589
+ }
1862
2590
  const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
1863
2591
  const embed = async (texts) => {
1864
2592
  const out = [];
1865
2593
  for (let i = 0; i < texts.length; i += MAX_BATCH) {
1866
2594
  const slice = texts.slice(i, i + MAX_BATCH).map(cap);
1867
- const result = await pipe(slice, { pooling: "mean", normalize: true });
2595
+ const result = await pipe(slice, { pooling, normalize: true });
1868
2596
  out.push(...result.tolist());
1869
2597
  }
1870
2598
  return out;
1871
2599
  };
1872
2600
  return {
1873
2601
  profile,
2602
+ verification,
1874
2603
  embed,
1875
2604
  embedQuery: async (text) => {
1876
2605
  const [vector] = await embed([`${profile.queryInstruction}${text}`]);
@@ -1884,9 +2613,9 @@ async function createProfileEmbedder(profile, opts = {}) {
1884
2613
  }
1885
2614
 
1886
2615
  // src/markdown-adapter/load.ts
1887
- import { readFile as readFile2, readdir } from "fs/promises";
1888
- import { join as join4, relative as relative3, sep } from "path";
1889
- import { CatalogDocumentSchema } from "@bendyline/gezel";
2616
+ import { readFile as readFile3, readdir } from "fs/promises";
2617
+ import { join as join6, relative as relative3, sep } from "path";
2618
+ import { CatalogDocumentSchema } from "@bendyline/gezk";
1890
2619
  var ROOT_TOPIC_ID = "general";
1891
2620
  var SUMMARY_MAX_CHARS = 280;
1892
2621
  function parseFrontMatter(markdown) {
@@ -1940,7 +2669,7 @@ async function walkMarkdownFiles(root) {
1940
2669
  );
1941
2670
  for (const entry of entries) {
1942
2671
  if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
1943
- const abs = join4(dir, entry.name);
2672
+ const abs = join6(dir, entry.name);
1944
2673
  if (entry.isDirectory()) await visit(abs);
1945
2674
  else if (entry.isFile() && /\.(md|markdown)$/i.test(entry.name)) out.push(abs);
1946
2675
  }
@@ -1991,7 +2720,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
1991
2720
  const id = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
1992
2721
  if (seenIds.has(id)) throw new Error(`duplicate document id from path: ${rel}`);
1993
2722
  seenIds.add(id);
1994
- const raw = (await readFile2(abs, "utf8")).replace(/^\uFEFF/, "");
2723
+ const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
1995
2724
  const fm = parseFrontMatter(raw);
1996
2725
  const stem = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
1997
2726
  const title = fm.title ?? firstHeading(fm.body) ?? stem;
@@ -2021,131 +2750,18 @@ async function loadMarkdownCatalog(rootDir, opts) {
2021
2750
  }
2022
2751
 
2023
2752
  // src/registry-client/fetch.ts
2024
- import { awakeTimeoutSignal } from "@bendyline/gezel";
2025
- import { KnowledgeRegistryIndexSchema } from "@bendyline/gezel";
2753
+ import { KnowledgeRegistryIndexSchema } from "@bendyline/gezk";
2026
2754
 
2027
2755
  // src/signatures/signing.ts
2028
2756
  import {
2029
- createHash as createHash5,
2030
- createPrivateKey,
2031
- createPublicKey,
2032
- sign as cryptoSign,
2033
- verify as cryptoVerify,
2034
- generateKeyPairSync
2035
- } from "crypto";
2036
-
2037
- // src/signatures/jcs.ts
2038
- function canonicalizeJson(value) {
2039
- return serialize(value, "$");
2040
- }
2041
- function serialize(value, path) {
2042
- if (value === null) return "null";
2043
- switch (typeof value) {
2044
- case "boolean":
2045
- return value ? "true" : "false";
2046
- case "number":
2047
- if (!Number.isFinite(value)) throw new Error(`non-finite number at ${path}`);
2048
- return JSON.stringify(value);
2049
- case "string":
2050
- return JSON.stringify(value);
2051
- case "object":
2052
- break;
2053
- default:
2054
- throw new Error(`value of type ${typeof value} at ${path} has no JSON identity`);
2055
- }
2056
- if (Array.isArray(value)) {
2057
- return `[${value.map((item, i) => serialize(item ?? null, `${path}[${i}]`)).join(",")}]`;
2058
- }
2059
- const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => compareUtf16(a, b));
2060
- return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${serialize(v, `${path}.${k}`)}`).join(",")}}`;
2061
- }
2062
- function compareUtf16(a, b) {
2063
- const n = Math.min(a.length, b.length);
2064
- for (let i = 0; i < n; i++) {
2065
- const d = a.charCodeAt(i) - b.charCodeAt(i);
2066
- if (d !== 0) return d;
2067
- }
2068
- return a.length - b.length;
2069
- }
2070
-
2071
- // src/signatures/signing.ts
2072
- function knowledgeKeyId(publicKeyPem) {
2073
- const der = createPublicKey(publicKeyPem).export({ type: "spki", format: "der" });
2074
- return createHash5("sha256").update(der).digest("hex").slice(0, 16);
2075
- }
2076
- function generateKnowledgeSigningKeyPair() {
2077
- const { publicKey, privateKey } = generateKeyPairSync("ed25519");
2078
- const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }).toString();
2079
- return {
2080
- publicKeyPem,
2081
- privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
2082
- keyId: knowledgeKeyId(publicKeyPem)
2083
- };
2084
- }
2085
- function manifestSigningPayload(manifest) {
2086
- const { signature: _omitted, ...unsigned } = manifest;
2087
- return Buffer.from(canonicalizeJson(unsigned), "utf8");
2088
- }
2089
- function signManifest(manifest, privateKeyPem) {
2090
- const key = createPrivateKey(privateKeyPem);
2091
- const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
2092
- const value = cryptoSign(null, manifestSigningPayload(manifest), key).toString("base64");
2093
- return {
2094
- ...manifest,
2095
- signature: {
2096
- algorithm: "ed25519",
2097
- keyId: knowledgeKeyId(publicKeyPem),
2098
- canonicalization: "rfc8785",
2099
- value
2100
- }
2101
- };
2102
- }
2103
- function verifyManifestSignature(manifest, anchors) {
2104
- return verifySignedDocument(manifest, anchors);
2105
- }
2106
- function signRegistryIndex(index, privateKeyPem) {
2107
- const key = createPrivateKey(privateKeyPem);
2108
- const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
2109
- const { signature: _omitted, ...unsigned } = index;
2110
- const value = cryptoSign(null, Buffer.from(canonicalizeJson(unsigned), "utf8"), key).toString(
2111
- "base64"
2112
- );
2113
- return {
2114
- ...index,
2115
- signature: {
2116
- algorithm: "ed25519",
2117
- keyId: knowledgeKeyId(publicKeyPem),
2118
- canonicalization: "rfc8785",
2119
- value
2120
- }
2121
- };
2122
- }
2123
- function verifyRegistryIndex(index, anchors) {
2124
- return verifySignedDocument(index, anchors);
2125
- }
2126
- function verifySignedDocument(document, anchors) {
2127
- const signature = document.signature;
2128
- if (!signature) return { ok: false, reason: "unsigned" };
2129
- const anchor = anchors.find((a) => a.keyId === signature.keyId);
2130
- if (!anchor) {
2131
- return { ok: false, reason: "unknown-key", detail: `no trust anchor for ${signature.keyId}` };
2132
- }
2133
- try {
2134
- if (knowledgeKeyId(anchor.publicKeyPem) !== signature.keyId) {
2135
- return { ok: false, reason: "unknown-key", detail: "anchor keyId does not match its key" };
2136
- }
2137
- const { signature: _omitted, ...unsigned } = document;
2138
- const ok = cryptoVerify(
2139
- null,
2140
- Buffer.from(canonicalizeJson(unsigned), "utf8"),
2141
- createPublicKey(anchor.publicKeyPem),
2142
- Buffer.from(signature.value, "base64")
2143
- );
2144
- return ok ? { ok: true, keyId: signature.keyId } : { ok: false, reason: "bad-signature" };
2145
- } catch (err) {
2146
- return { ok: false, reason: "error", detail: err instanceof Error ? err.message : String(err) };
2147
- }
2148
- }
2757
+ generateKnowledgeSigningKeyPair,
2758
+ knowledgeKeyId,
2759
+ manifestSigningPayload,
2760
+ signManifest,
2761
+ signRegistryIndex,
2762
+ verifyManifestSignature,
2763
+ verifyRegistryIndex
2764
+ } from "@bendyline/gezk/node";
2149
2765
 
2150
2766
  // src/registry-client/fetch.ts
2151
2767
  var DEFAULT_MAX_REGISTRY_BYTES = 4 * 1024 * 1024;
@@ -2164,7 +2780,7 @@ async function fetchKnowledgeRegistry(url, opts) {
2164
2780
  let response;
2165
2781
  try {
2166
2782
  response = await fetchImpl(url, {
2167
- signal: awakeTimeoutSignal(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
2783
+ signal: opts.signal ?? AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
2168
2784
  redirect: "follow",
2169
2785
  headers: { accept: "application/json" }
2170
2786
  });
@@ -2197,7 +2813,7 @@ async function fetchKnowledgeRegistry(url, opts) {
2197
2813
  } catch (err) {
2198
2814
  throw new KnowledgeRegistryFetchError(
2199
2815
  "invalid",
2200
- `registry is not a valid gezel-knowledge-registry document: ${err instanceof Error ? err.message : String(err)}`
2816
+ `registry is not a valid gezk-registry document: ${err instanceof Error ? err.message : String(err)}`
2201
2817
  );
2202
2818
  }
2203
2819
  const verdict = verifyRegistryIndex(registry, opts.anchors);
@@ -2243,7 +2859,11 @@ function newerRegistryEntries(registry, installed) {
2243
2859
 
2244
2860
  // src/signatures/anchors.ts
2245
2861
  var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
2862
+
2863
+ // src/signatures/jcs.ts
2864
+ import { canonicalizeJson } from "@bendyline/gezk";
2246
2865
  export {
2866
+ BGE_SMALL_EN_V15_1,
2247
2867
  BODY_CODEC_MIN_BYTES,
2248
2868
  BUILTIN_KNOWLEDGE_TRUST_ANCHORS,
2249
2869
  CENTROID_CHUNKS_PER,
@@ -2252,6 +2872,8 @@ export {
2252
2872
  CENTROID_SAMPLE_MAX,
2253
2873
  CatalogHandle,
2254
2874
  CatalogOpenError,
2875
+ DuckdbError,
2876
+ EmbedderArtifactError,
2255
2877
  EmbedderUnavailableError,
2256
2878
  GEZEL_BGE_SMALL_EN_V15_1,
2257
2879
  GEZEL_MARKDOWN_CHUNKS_2,
@@ -2260,11 +2882,20 @@ export {
2260
2882
  GEZK_ARCHIVE_LIMITS,
2261
2883
  GEZK_FORMAT_VERSION,
2262
2884
  GEZK_INDEX_SCHEMA_VERSION,
2885
+ GEZK_MIME_TYPE,
2263
2886
  GezkArchiveError,
2264
2887
  KNOWLEDGE_EMBEDDING_PROFILES,
2888
+ KNOWLEDGE_TOOLCHAIN,
2265
2889
  KnowledgeRegistryFetchError,
2890
+ LICENSE_NOTICE_PATH,
2266
2891
  MANIFEST_PATH,
2892
+ MARKDOWN_CHUNKS_2,
2267
2893
  MAX_KNOWLEDGE_DOCUMENT_BYTES,
2894
+ MIMETYPE_PATH,
2895
+ MULTILINGUAL_E5_SMALL_1,
2896
+ PARQUET_EXPORT_VERSION,
2897
+ PARQUET_REPORT_PATH,
2898
+ README_PATH,
2268
2899
  RERANK_FINAL_K,
2269
2900
  ROUTER_DB_PATH,
2270
2901
  ROUTER_DDL,
@@ -2273,7 +2904,9 @@ export {
2273
2904
  SHARD_DDL,
2274
2905
  SHARD_MAX_CHUNKS,
2275
2906
  SHARD_TARGET_CHUNKS,
2907
+ SOURCE_NOTICES_PATH,
2276
2908
  ZIP_FIXED_MTIME,
2909
+ assertDuckdbCli,
2277
2910
  canonicalizeJson,
2278
2911
  chunkContentHash,
2279
2912
  chunkMarkdownProfile,
@@ -2282,10 +2915,14 @@ export {
2282
2915
  compileKnowledgeCatalog,
2283
2916
  createProfileEmbedder,
2284
2917
  documentSlug,
2918
+ duckdbVersion,
2919
+ exportCatalogParquet,
2285
2920
  extractGezkVerified,
2286
2921
  fetchKnowledgeRegistry,
2922
+ findDuckdbBinary,
2287
2923
  findRegistryEntry,
2288
2924
  generateKnowledgeSigningKeyPair,
2925
+ hammingTopK,
2289
2926
  inspectGezkArchive,
2290
2927
  knowledgeEmbeddingProfile,
2291
2928
  knowledgeKeyId,
@@ -2299,11 +2936,14 @@ export {
2299
2936
  readGezkManifest,
2300
2937
  rerankK,
2301
2938
  rerankScore,
2939
+ resolveTransformersModelOptions,
2940
+ runDuckdbScript,
2302
2941
  signManifest,
2303
2942
  signRegistryIndex,
2943
+ transformersCachePath,
2304
2944
  validateExtractedCatalog,
2305
- vecChunksDdl,
2306
2945
  verifyManifestSignature,
2946
+ verifyProfileArtifacts,
2307
2947
  verifyRegistryIndex,
2308
2948
  writeGezkArchive
2309
2949
  };