@bendyline/gezel-knowledge 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,2113 @@
1
+ // src/format/constants.ts
2
+ var GEZK_FORMAT_VERSION = 1;
3
+ var GEZK_INDEX_SCHEMA_VERSION = 1;
4
+ var GEZK_APPLICATION_ID = 1195727435;
5
+ var ROUTER_DB_PATH = "index/router.db";
6
+ var MANIFEST_PATH = "manifest.json";
7
+ var SHARD_TARGET_CHUNKS = 2e5;
8
+ var SHARD_MAX_CHUNKS = 25e4;
9
+ var BODY_CODEC_MIN_BYTES = 512;
10
+ var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
11
+ var CENTROID_CHUNKS_PER = 12500;
12
+ var CENTROID_MAX_PER_SHARD = 32;
13
+ var CENTROID_KMEANS_MAX_ITER = 25;
14
+ var CENTROID_SAMPLE_MAX = 65536;
15
+ var ROUTE_SHARDS_PROACTIVE = 3;
16
+ var ROUTE_SHARDS_EXPLICIT = 6;
17
+ var RERANK_FINAL_K = 24;
18
+ function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
19
+ return Math.min(512, Math.max(128, 8 * finalK), chunkCount);
20
+ }
21
+ var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
22
+
23
+ // src/format/ddl.ts
24
+ var ROUTER_DDL = `
25
+ CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
26
+
27
+ CREATE TABLE topics(
28
+ id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,
29
+ description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL
30
+ );
31
+
32
+ CREATE TABLE documents(
33
+ id TEXT PRIMARY KEY,
34
+ title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
35
+ language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
36
+ shard_id INTEGER NOT NULL,
37
+ chunk_count INTEGER NOT NULL,
38
+ source_url TEXT, source_revision TEXT, source_updated_at TEXT,
39
+ attribution_json TEXT,
40
+ body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
41
+ body_blob BLOB NOT NULL
42
+ );
43
+ CREATE INDEX documents_topic ON documents(topic_id, slug);
44
+ CREATE INDEX documents_shard ON documents(shard_id);
45
+
46
+ CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
47
+ PRIMARY KEY (alias, document_id)) WITHOUT ROWID;
48
+
49
+ CREATE TABLE shards(
50
+ id INTEGER PRIMARY KEY,
51
+ path TEXT NOT NULL,
52
+ chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,
53
+ topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,
54
+ bytes INTEGER NOT NULL
55
+ );
56
+
57
+ CREATE TABLE route_centroids(
58
+ id INTEGER PRIMARY KEY,
59
+ shard_id INTEGER NOT NULL REFERENCES shards(id),
60
+ embedding BLOB NOT NULL,
61
+ weight INTEGER NOT NULL
62
+ );
63
+ CREATE INDEX route_centroids_shard ON route_centroids(shard_id);
64
+
65
+ CREATE VIRTUAL TABLE fts_documents USING fts5(
66
+ title, summary, aliases, document_id UNINDEXED,
67
+ tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
68
+ );
69
+ `;
70
+ var SHARD_DDL = `
71
+ CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
72
+
73
+ CREATE TABLE chunks(
74
+ id INTEGER PRIMARY KEY,
75
+ chunk_uid TEXT NOT NULL,
76
+ document_id TEXT NOT NULL,
77
+ ordinal INTEGER NOT NULL,
78
+ title TEXT NOT NULL,
79
+ heading_path TEXT NOT NULL,
80
+ heading_text TEXT NOT NULL,
81
+ line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,
82
+ token_count INTEGER NOT NULL,
83
+ content_hash TEXT NOT NULL,
84
+ text TEXT NOT NULL
85
+ );
86
+ CREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);
87
+ CREATE INDEX chunks_document ON chunks(document_id, ordinal);
88
+
89
+ CREATE VIRTUAL TABLE fts_chunks USING fts5(
90
+ title, heading_text, text,
91
+ content = 'chunks', content_rowid = 'id',
92
+ tokenize = 'unicode61 remove_diacritics 2'
93
+ );
94
+
95
+ CREATE TABLE chunk_vectors_int8(
96
+ chunk_id INTEGER PRIMARY KEY,
97
+ v BLOB NOT NULL
98
+ );
99
+ `;
100
+ function vecChunksDdl(dim) {
101
+ return `CREATE VIRTUAL TABLE vec_chunks USING vec0(
102
+ chunk_id INTEGER PRIMARY KEY,
103
+ embedding bit[${dim}],
104
+ chunk_size=1024
105
+ );`;
106
+ }
107
+
108
+ // src/format/ids.ts
109
+ import { createHash } from "crypto";
110
+ function chunkUid(documentId, ordinal, chunkText) {
111
+ const textHash = createHash("sha256").update(chunkText, "utf8").digest();
112
+ return createHash("sha256").update(documentId, "utf8").update(Buffer.from([0])).update(String(ordinal), "utf8").update(Buffer.from([0])).update(textHash).digest("hex").slice(0, 32);
113
+ }
114
+ function chunkContentHash(chunkText) {
115
+ return createHash("sha256").update(chunkText, "utf8").digest("hex");
116
+ }
117
+ function documentSlug(title) {
118
+ const slug = title.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
119
+ return slug || "untitled";
120
+ }
121
+
122
+ // src/format/quantize.ts
123
+ function quantizeInt8(unitVector) {
124
+ const out = new Int8Array(unitVector.length);
125
+ for (let i = 0; i < unitVector.length; i++) {
126
+ const q = Math.round(unitVector[i] * 127);
127
+ out[i] = q > 127 ? 127 : q < -127 ? -127 : q;
128
+ }
129
+ return out;
130
+ }
131
+ function quantizeBinary(unitVector) {
132
+ const bytes = Math.ceil(unitVector.length / 8);
133
+ const out = new Uint8Array(bytes);
134
+ for (let i = 0; i < unitVector.length; i++) {
135
+ if (unitVector[i] > 0) {
136
+ out[i >> 3] = out[i >> 3] | 1 << (i & 7);
137
+ }
138
+ }
139
+ return out;
140
+ }
141
+ function rerankScore(query, passageInt8) {
142
+ let dot = 0;
143
+ const n = Math.min(query.length, passageInt8.length);
144
+ for (let i = 0; i < n; i++) dot += query[i] * (passageInt8[i] / 127);
145
+ return dot;
146
+ }
147
+ function l2Normalize(vector) {
148
+ let sum = 0;
149
+ for (let i = 0; i < vector.length; i++) {
150
+ const v = vector[i];
151
+ sum += v * v;
152
+ }
153
+ const norm = Math.sqrt(sum);
154
+ if (!Number.isFinite(norm) || norm === 0) throw new Error("degenerate vector (zero norm)");
155
+ const out = new Float32Array(vector.length);
156
+ for (let i = 0; i < vector.length; i++) out[i] = vector[i] / norm;
157
+ return out;
158
+ }
159
+
160
+ // src/chunking/markdown-chunker.ts
161
+ var HEADING = /^(#{1,6})\s+(.*)$/;
162
+ function chunkMarkdownProfile(md, opts) {
163
+ const lines = md.split(/\r?\n/);
164
+ const headIdx = [];
165
+ for (let i = 0; i < lines.length; i++) {
166
+ if (HEADING.test(lines[i])) headIdx.push(i);
167
+ }
168
+ const out = [];
169
+ if (headIdx.length === 0) {
170
+ return chunkRange(lines, 0, lines.length - 1, "doc", [], opts);
171
+ }
172
+ if (headIdx[0] > 0) {
173
+ out.push(...chunkRange(lines, 0, headIdx[0] - 1, "preamble", [], opts));
174
+ }
175
+ const stack = [];
176
+ for (let h = 0; h < headIdx.length; h++) {
177
+ const start = headIdx[h];
178
+ const end = h + 1 < headIdx.length ? headIdx[h + 1] - 1 : lines.length - 1;
179
+ const m = HEADING.exec(lines[start]);
180
+ const depth = m ? m[1]?.length ?? 1 : 1;
181
+ const text = m ? (m[2] ?? "").trim() : "";
182
+ while (stack.length > 0 && stack[stack.length - 1].depth >= depth) {
183
+ stack.pop();
184
+ }
185
+ stack.push({ depth, text });
186
+ const headingPath = stack.map((s) => s.text).filter((t) => t.length > 0);
187
+ out.push(...chunkRange(lines, start, end, "section", headingPath, opts));
188
+ }
189
+ return out;
190
+ }
191
+ function measure(text, opts) {
192
+ return opts.unit === "chars" ? text.length : opts.countTokens(text);
193
+ }
194
+ function chunkRange(lines, start, end, kind, headingPath, opts) {
195
+ const out = [];
196
+ const lineSize = /* @__PURE__ */ new Map();
197
+ const sizeOf = (i) => {
198
+ let s = lineSize.get(i);
199
+ if (s === void 0) {
200
+ s = measure(lines[i] ?? "", opts);
201
+ lineSize.set(i, s);
202
+ }
203
+ return s;
204
+ };
205
+ const joinCost = opts.unit === "chars" ? 1 : 0;
206
+ let cursor = start;
207
+ while (cursor <= end) {
208
+ const first = lines[cursor] ?? "";
209
+ if (sizeOf(cursor) > opts.target) {
210
+ for (const text2 of splitOversizeLine(first, opts)) {
211
+ if (text2) out.push({ kind, lineStart: cursor + 1, lineEnd: cursor + 1, text: text2, headingPath });
212
+ }
213
+ cursor++;
214
+ continue;
215
+ }
216
+ let chunkEnd = cursor;
217
+ let size = sizeOf(cursor);
218
+ while (chunkEnd + 1 <= end) {
219
+ const nextSize = sizeOf(chunkEnd + 1) + joinCost;
220
+ if (size + nextSize > opts.target) break;
221
+ size += nextSize;
222
+ chunkEnd++;
223
+ }
224
+ const text = lines.slice(cursor, chunkEnd + 1).join("\n").trim();
225
+ if (text) {
226
+ out.push({ kind, lineStart: cursor + 1, lineEnd: chunkEnd + 1, text, headingPath });
227
+ }
228
+ if (chunkEnd >= end) break;
229
+ let nextCursor = chunkEnd + 1;
230
+ let overlap = 0;
231
+ for (let i = chunkEnd; i > cursor; i--) {
232
+ const length = sizeOf(i) + joinCost;
233
+ if (overlap + length > opts.overlap) break;
234
+ overlap += length;
235
+ nextCursor = i;
236
+ }
237
+ cursor = Math.max(cursor + 1, nextCursor);
238
+ }
239
+ return out;
240
+ }
241
+ function splitOversizeLine(line, opts) {
242
+ if (opts.unit === "chars") {
243
+ const out2 = [];
244
+ let offset = 0;
245
+ while (offset < line.length) {
246
+ out2.push(line.slice(offset, offset + opts.target).trim());
247
+ if (offset + opts.target >= line.length) break;
248
+ offset += opts.target - opts.overlap;
249
+ }
250
+ return out2;
251
+ }
252
+ const sentences = line.split(/(?<=[.!?])\s+/);
253
+ const out = [];
254
+ let current = [];
255
+ let currentTokens = 0;
256
+ const flush = () => {
257
+ if (current.length > 0) {
258
+ const text = current.join(" ").trim();
259
+ if (text) out.push(text);
260
+ current = [];
261
+ currentTokens = 0;
262
+ }
263
+ };
264
+ for (const sentence of sentences) {
265
+ const tokens = opts.countTokens(sentence);
266
+ if (tokens > opts.target) {
267
+ flush();
268
+ const charsPerToken = Math.max(1, sentence.length / tokens);
269
+ const window = Math.max(1, Math.floor(opts.target * charsPerToken));
270
+ const step = Math.max(1, window - Math.floor(opts.overlap * charsPerToken));
271
+ for (let offset = 0; offset < sentence.length; offset += step) {
272
+ const piece = sentence.slice(offset, offset + window).trim();
273
+ if (piece) out.push(piece);
274
+ if (offset + window >= sentence.length) break;
275
+ }
276
+ continue;
277
+ }
278
+ if (currentTokens + tokens > opts.target) flush();
279
+ current.push(sentence);
280
+ currentTokens += tokens;
281
+ }
282
+ flush();
283
+ return out;
284
+ }
285
+
286
+ // src/compiler/compile.ts
287
+ import { createHash as createHash3 } from "crypto";
288
+ import { mkdirSync, rmSync } from "fs";
289
+ import { stat } from "fs/promises";
290
+ import { join } from "path";
291
+ import { brotliCompressSync, constants as zlibConstants } from "zlib";
292
+ import { KnowledgeCatalogManifestSchema } from "@bendyline/gezel";
293
+
294
+ // src/archive/write.ts
295
+ import { createWriteStream } from "fs";
296
+ import { mkdir } from "fs/promises";
297
+ import { createRequire } from "module";
298
+ import { dirname } from "path";
299
+ var nodeRequire = createRequire(import.meta.url);
300
+ async function writeGezkArchive(outputPath, entries) {
301
+ const yazl = nodeRequire("yazl");
302
+ await mkdir(dirname(outputPath), { recursive: true });
303
+ const zip = new yazl.ZipFile();
304
+ const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress: true };
305
+ const sorted = [...entries].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
306
+ for (const entry of sorted) {
307
+ if (entry.path.includes("\\") || entry.path.startsWith("/") || entry.path.includes("..")) {
308
+ throw new Error(`unsafe archive path: ${entry.path}`);
309
+ }
310
+ if (entry.content) zip.addBuffer(entry.content, entry.path, options);
311
+ else if (entry.absPath) zip.addFile(entry.absPath, entry.path, options);
312
+ else throw new Error(`archive entry ${entry.path} has neither content nor absPath`);
313
+ }
314
+ await new Promise((resolve3, reject) => {
315
+ const out = createWriteStream(outputPath);
316
+ out.on("error", reject);
317
+ out.on("close", () => resolve3());
318
+ zip.outputStream.on("error", reject);
319
+ zip.outputStream.pipe(out);
320
+ zip.end();
321
+ });
322
+ }
323
+
324
+ // src/format/file-hash.ts
325
+ import { createHash as createHash2 } from "crypto";
326
+ import { createReadStream } from "fs";
327
+ async function hashFileStreaming(path, maxBytes = Number.MAX_SAFE_INTEGER) {
328
+ const hasher = createHash2("sha256");
329
+ let sizeBytes = 0;
330
+ await new Promise((resolve3, reject) => {
331
+ const stream = createReadStream(path, { highWaterMark: 4 * 1024 * 1024 });
332
+ stream.on("data", (chunk) => {
333
+ const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
334
+ sizeBytes += bytes.byteLength;
335
+ if (sizeBytes > maxBytes) {
336
+ stream.destroy(new Error(`file exceeds ${maxBytes} byte limit`));
337
+ return;
338
+ }
339
+ hasher.update(bytes);
340
+ });
341
+ stream.on("error", reject);
342
+ stream.on("end", resolve3);
343
+ });
344
+ return { sha256: hasher.digest("hex"), sizeBytes };
345
+ }
346
+
347
+ // src/format/node-sqlite.ts
348
+ import { createRequire as createRequire2 } from "module";
349
+ var nodeRequire2 = createRequire2(import.meta.url);
350
+ var sqlite = nodeRequire2("node:sqlite");
351
+ var DatabaseSync = sqlite.DatabaseSync;
352
+
353
+ // src/compiler/vec-load.ts
354
+ import { createRequire as createRequire3 } from "module";
355
+ var nodeRequire3 = createRequire3(import.meta.url);
356
+ function loadVecExtension(db) {
357
+ try {
358
+ db.enableLoadExtension(true);
359
+ const sqliteVec = nodeRequire3("sqlite-vec");
360
+ db.loadExtension(sqliteVec.getLoadablePath());
361
+ } catch (err) {
362
+ throw new Error(
363
+ `sqlite-vec extension failed to load \u2014 the compiler cannot build vector tables: ${err instanceof Error ? err.message : String(err)}`
364
+ );
365
+ } finally {
366
+ try {
367
+ db.enableLoadExtension(false);
368
+ } catch {
369
+ }
370
+ }
371
+ }
372
+
373
+ // src/compiler/compile.ts
374
+ async function compileKnowledgeCatalog(opts) {
375
+ if (opts.topics.length === 0) {
376
+ throw new Error(
377
+ "a catalog must ship a table of contents: provide at least one topic (a flat corpus gets a single root topic)"
378
+ );
379
+ }
380
+ const topicIds = new Set(opts.topics.map((t) => t.id));
381
+ const profile = opts.embeddingProfile;
382
+ const chunkerOpts = {
383
+ unit: "tokens",
384
+ target: opts.chunkingProfile.targetTokens,
385
+ overlap: opts.chunkingProfile.overlapTokens,
386
+ countTokens: opts.countTokens
387
+ };
388
+ const prepared = [];
389
+ const seenIds = /* @__PURE__ */ new Set();
390
+ for await (const doc of opts.documents) {
391
+ if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
392
+ seenIds.add(doc.id);
393
+ const rootTopic = doc.topicPath[0];
394
+ if (!rootTopic || !topicIds.has(rootTopic)) {
395
+ throw new Error(`document ${doc.id} names unknown topic '${rootTopic ?? ""}'`);
396
+ }
397
+ prepared.push({
398
+ doc,
399
+ topicPathKey: doc.topicPath.join("/"),
400
+ chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
401
+ });
402
+ }
403
+ prepared.sort(
404
+ (a, b) => a.topicPathKey === b.topicPathKey ? a.doc.id < b.doc.id ? -1 : a.doc.id > b.doc.id ? 1 : 0 : a.topicPathKey < b.topicPathKey ? -1 : 1
405
+ );
406
+ const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
407
+ const embedded = totalChunks <= SHARD_TARGET_CHUNKS;
408
+ const shardOf = /* @__PURE__ */ new Map();
409
+ let shardCount = 1;
410
+ if (embedded) {
411
+ for (const p of prepared) shardOf.set(p.doc.id, 0);
412
+ } else {
413
+ let current = 0;
414
+ let filled = 0;
415
+ for (const p of prepared) {
416
+ if (filled > 0 && filled + p.chunks.length > SHARD_TARGET_CHUNKS) {
417
+ current++;
418
+ filled = 0;
419
+ }
420
+ shardOf.set(p.doc.id, current);
421
+ filled += p.chunks.length;
422
+ }
423
+ shardCount = current + 1;
424
+ }
425
+ rmSync(opts.workDir, { recursive: true, force: true });
426
+ mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
427
+ const routerStaged = join(opts.workDir, ROUTER_DB_PATH);
428
+ const shardPath = (id) => embedded ? ROUTER_DB_PATH : `index/shards/${String(id).padStart(3, "0")}.db`;
429
+ const shardDbs = /* @__PURE__ */ new Map();
430
+ const router = createCatalogDb(routerStaged);
431
+ const closeAll = () => {
432
+ for (const db of [router, ...shardDbs.values()]) {
433
+ try {
434
+ db.close();
435
+ } catch {
436
+ }
437
+ }
438
+ };
439
+ try {
440
+ router.exec(ROUTER_DDL);
441
+ router.exec("BEGIN");
442
+ if (embedded) {
443
+ router.exec(SHARD_DDL.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
444
+ router.exec(vecChunksDdl(profile.dimensions));
445
+ }
446
+ const commonMeta = {
447
+ format_version: String(GEZK_FORMAT_VERSION),
448
+ index_schema_version: String(GEZK_INDEX_SCHEMA_VERSION),
449
+ catalog_id: opts.catalog.id,
450
+ catalog_version: opts.catalog.version,
451
+ embedding_profile_id: profile.id,
452
+ chunking_profile_id: opts.chunkingProfile.id
453
+ };
454
+ const putMeta = (db, extra) => {
455
+ const stmt = db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)");
456
+ for (const [k, v] of Object.entries({ ...commonMeta, ...extra })) stmt.run(k, v);
457
+ };
458
+ putMeta(router, {
459
+ embedding_profile_json: JSON.stringify(profile),
460
+ chunking_profile_json: JSON.stringify(opts.chunkingProfile),
461
+ created_at: opts.catalog.createdAt,
462
+ ...opts.toolchain ? { toolchain_json: JSON.stringify(opts.toolchain) } : {}
463
+ });
464
+ const topicDocCount = /* @__PURE__ */ new Map();
465
+ for (const p of prepared) {
466
+ const t = p.doc.topicPath[0];
467
+ topicDocCount.set(t, (topicDocCount.get(t) ?? 0) + 1);
468
+ }
469
+ {
470
+ const stmt = router.prepare(
471
+ "INSERT INTO topics (id, parent_id, name, description, sort_key, document_count) VALUES (?, ?, ?, ?, ?, ?)"
472
+ );
473
+ for (const t of [...opts.topics].sort((a, b) => a.id < b.id ? -1 : 1)) {
474
+ stmt.run(
475
+ t.id,
476
+ t.parentId ?? null,
477
+ t.name,
478
+ t.description ?? null,
479
+ t.sortKey ?? t.name.toLowerCase(),
480
+ topicDocCount.get(t.id) ?? 0
481
+ );
482
+ }
483
+ }
484
+ {
485
+ const insertDoc = router.prepare(
486
+ `INSERT INTO documents (id, title, slug, summary, language, topic_id, shard_id, chunk_count,
487
+ source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob)
488
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
489
+ );
490
+ const insertAlias = router.prepare(
491
+ "INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
492
+ );
493
+ const insertFts = router.prepare(
494
+ "INSERT INTO fts_documents (title, summary, aliases, document_id) VALUES (?, ?, ?, ?)"
495
+ );
496
+ for (const p of prepared) {
497
+ const body = Buffer.from(normalizeMarkdown(p.doc.markdown), "utf8");
498
+ if (body.byteLength > MAX_KNOWLEDGE_DOCUMENT_BYTES) {
499
+ throw new Error(
500
+ `document ${p.doc.id} exceeds the ${MAX_KNOWLEDGE_DOCUMENT_BYTES} byte body limit`
501
+ );
502
+ }
503
+ const useBrotli = body.length >= BODY_CODEC_MIN_BYTES;
504
+ const blob = useBrotli ? brotliCompressSync(body, {
505
+ params: {
506
+ [zlibConstants.BROTLI_PARAM_QUALITY]: 5,
507
+ [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT
508
+ }
509
+ }) : body;
510
+ insertDoc.run(
511
+ p.doc.id,
512
+ p.doc.title,
513
+ p.doc.slug || documentSlug(p.doc.title),
514
+ p.doc.summary ?? null,
515
+ p.doc.language,
516
+ p.doc.topicPath[0],
517
+ BigInt(shardOf.get(p.doc.id)),
518
+ BigInt(p.chunks.length),
519
+ p.doc.sourceUrl ?? null,
520
+ p.doc.sourceRevision ?? null,
521
+ p.doc.sourceUpdatedAt ?? null,
522
+ p.doc.attribution ? JSON.stringify(p.doc.attribution) : null,
523
+ useBrotli ? "br" : "none",
524
+ blob
525
+ );
526
+ const aliases = [...new Set(p.doc.aliases ?? [])].sort();
527
+ for (const alias of aliases) insertAlias.run(alias, p.doc.id);
528
+ insertFts.run(p.doc.title, p.doc.summary ?? "", aliases.join(" "), p.doc.id);
529
+ }
530
+ }
531
+ const shardChunkCounts = /* @__PURE__ */ new Map();
532
+ const shardDocCounts = /* @__PURE__ */ new Map();
533
+ const shardTopicIds = /* @__PURE__ */ new Map();
534
+ const shardSamples = /* @__PURE__ */ new Map();
535
+ const shardDb = (id) => {
536
+ if (embedded) return router;
537
+ let db = shardDbs.get(id);
538
+ if (!db) {
539
+ db = createCatalogDb(join(opts.workDir, shardPath(id)));
540
+ db.exec(SHARD_DDL);
541
+ db.exec(vecChunksDdl(profile.dimensions));
542
+ db.exec("BEGIN");
543
+ shardDbs.set(id, db);
544
+ }
545
+ return db;
546
+ };
547
+ const vecStmts = /* @__PURE__ */ new Map();
548
+ const vecStmtsFor = (db) => {
549
+ let stmts = vecStmts.get(db);
550
+ if (!stmts) {
551
+ stmts = {
552
+ vec: db.prepare("INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, vec_bit(?))"),
553
+ int8: db.prepare("INSERT INTO chunk_vectors_int8 (chunk_id, v) VALUES (?, ?)")
554
+ };
555
+ vecStmts.set(db, stmts);
556
+ }
557
+ return stmts;
558
+ };
559
+ const embedBatchSize = opts.embedBatchSize ?? 32;
560
+ let pendingTexts = [];
561
+ let pendingRows = [];
562
+ let processedChunks = 0;
563
+ const flushEmbeds = async () => {
564
+ if (pendingTexts.length === 0) return;
565
+ const vectors = await opts.embed(pendingTexts);
566
+ if (vectors.length !== pendingTexts.length) {
567
+ throw new Error(
568
+ `embedder returned ${vectors.length} vectors for ${pendingTexts.length} texts`
569
+ );
570
+ }
571
+ for (let i = 0; i < vectors.length; i++) {
572
+ const unit = l2Normalize(vectors[i]);
573
+ if (unit.length !== profile.dimensions) {
574
+ throw new Error(
575
+ `embedder returned dim ${unit.length}, profile expects ${profile.dimensions}`
576
+ );
577
+ }
578
+ const row = pendingRows[i];
579
+ const stmts = vecStmtsFor(shardDb(row.shardId));
580
+ stmts.vec.run(BigInt(row.chunkId), Buffer.from(quantizeBinary(unit)));
581
+ stmts.int8.run(BigInt(row.chunkId), Buffer.from(quantizeInt8(unit).buffer));
582
+ const shardTotal = shardChunkCounts.get(row.shardId) ?? 0;
583
+ void shardTotal;
584
+ const samples = shardSamples.get(row.shardId) ?? [];
585
+ shardSamples.set(row.shardId, samples);
586
+ const stride = Math.max(1, Math.ceil(totalChunks / CENTROID_SAMPLE_MAX));
587
+ if ((row.chunkId - 1) % stride === 0) samples.push(unit);
588
+ }
589
+ processedChunks += pendingTexts.length;
590
+ opts.onProgress?.({ phase: "embed", done: processedChunks, total: totalChunks });
591
+ pendingTexts = [];
592
+ pendingRows = [];
593
+ };
594
+ for (const p of prepared) {
595
+ const shardId = shardOf.get(p.doc.id);
596
+ const db = shardDb(shardId);
597
+ shardDocCounts.set(shardId, (shardDocCounts.get(shardId) ?? 0) + 1);
598
+ const topics = shardTopicIds.get(shardId) ?? /* @__PURE__ */ new Set();
599
+ topics.add(p.doc.topicPath[0]);
600
+ shardTopicIds.set(shardId, topics);
601
+ const insertChunk = db.prepare(
602
+ `INSERT INTO chunks (chunk_uid, document_id, ordinal, title, heading_path, heading_text,
603
+ line_start, line_end, token_count, content_hash, text)
604
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id`
605
+ );
606
+ const insertChunkFts = db.prepare(
607
+ "INSERT INTO fts_chunks (rowid, title, heading_text, text) VALUES (?, ?, ?, ?)"
608
+ );
609
+ for (let ordinal = 0; ordinal < p.chunks.length; ordinal++) {
610
+ const chunk = p.chunks[ordinal];
611
+ const headingText = chunk.headingPath.join(" > ");
612
+ const row = insertChunk.get(
613
+ chunkUid(p.doc.id, ordinal, chunk.text),
614
+ p.doc.id,
615
+ BigInt(ordinal),
616
+ p.doc.title,
617
+ JSON.stringify(chunk.headingPath),
618
+ headingText,
619
+ BigInt(chunk.lineStart),
620
+ BigInt(chunk.lineEnd),
621
+ BigInt(opts.countTokens(chunk.text)),
622
+ chunkContentHash(chunk.text),
623
+ chunk.text
624
+ );
625
+ const chunkId = Number(row.id);
626
+ insertChunkFts.run(BigInt(chunkId), p.doc.title, headingText, chunk.text);
627
+ shardChunkCounts.set(shardId, (shardChunkCounts.get(shardId) ?? 0) + 1);
628
+ const header = buildContextHeader(
629
+ p.doc.title,
630
+ chunk.headingPath,
631
+ opts.chunkingProfile.contextHeader.maxTokens,
632
+ opts.countTokens
633
+ );
634
+ pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
635
+ pendingRows.push({ shardId, chunkId });
636
+ if (pendingTexts.length >= embedBatchSize) await flushEmbeds();
637
+ }
638
+ }
639
+ await flushEmbeds();
640
+ {
641
+ const insertShard = router.prepare(
642
+ "INSERT INTO shards (id, path, chunk_count, document_count, topic_ids_json, centroid_count, bytes) VALUES (?, ?, ?, ?, ?, ?, ?)"
643
+ );
644
+ const insertCentroid = router.prepare(
645
+ "INSERT INTO route_centroids (shard_id, embedding, weight) VALUES (?, ?, ?)"
646
+ );
647
+ for (let id = 0; id < shardCount; id++) {
648
+ const chunkCount = shardChunkCounts.get(id) ?? 0;
649
+ const samples = shardSamples.get(id) ?? [];
650
+ const k = Math.max(
651
+ 1,
652
+ Math.min(CENTROID_MAX_PER_SHARD, Math.ceil(chunkCount / CENTROID_CHUNKS_PER))
653
+ );
654
+ const centroids = kMeans(samples, k, seedFor(opts.catalog.id, id));
655
+ insertShard.run(
656
+ BigInt(id),
657
+ shardPath(id),
658
+ BigInt(chunkCount),
659
+ BigInt(shardDocCounts.get(id) ?? 0),
660
+ JSON.stringify([...shardTopicIds.get(id) ?? []].sort()),
661
+ BigInt(centroids.length),
662
+ BigInt(0)
663
+ // authoritative sizes live in the manifest (post-VACUUM)
664
+ );
665
+ for (const c of centroids) {
666
+ insertCentroid.run(BigInt(id), Buffer.from(c.centroid.buffer.slice(0)), BigInt(c.weight));
667
+ }
668
+ }
669
+ }
670
+ for (let id = 0; id < shardCount; id++) {
671
+ if (!embedded)
672
+ putMeta(shardDb(id), {
673
+ shard_id: String(id),
674
+ chunk_count: String(shardChunkCounts.get(id) ?? 0)
675
+ });
676
+ }
677
+ if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
678
+ const finalDir = join(opts.workDir, "final");
679
+ mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
680
+ const vacuumed = [];
681
+ const vacuumInto = (db, archivePath) => {
682
+ db.exec("COMMIT");
683
+ const dest = join(finalDir, archivePath);
684
+ db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
685
+ vacuumed.push({ archivePath, absPath: dest });
686
+ };
687
+ vacuumInto(router, ROUTER_DB_PATH);
688
+ if (!embedded) {
689
+ for (let id = 0; id < shardCount; id++) vacuumInto(shardDb(id), shardPath(id));
690
+ }
691
+ router.close();
692
+ for (const db of shardDbs.values()) db.close();
693
+ const files = [];
694
+ const routerShardStats = [];
695
+ for (const f of vacuumed) {
696
+ const { sha256, sizeBytes } = await hashFileStreaming(f.absPath);
697
+ files.push({ path: f.archivePath, sizeBytes, sha256 });
698
+ const idMatch = /(\d+)\.db$/.exec(f.archivePath);
699
+ const shardId = f.archivePath === ROUTER_DB_PATH ? 0 : Number(idMatch?.[1] ?? -1);
700
+ if (f.archivePath !== ROUTER_DB_PATH || embedded) {
701
+ routerShardStats.push({
702
+ id: shardId,
703
+ path: f.archivePath,
704
+ chunks: shardChunkCounts.get(shardId) ?? 0,
705
+ documents: shardDocCounts.get(shardId) ?? 0,
706
+ centroids: Math.max(
707
+ 1,
708
+ Math.min(
709
+ CENTROID_MAX_PER_SHARD,
710
+ Math.ceil((shardChunkCounts.get(shardId) ?? 0) / CENTROID_CHUNKS_PER)
711
+ )
712
+ ),
713
+ sha256
714
+ });
715
+ }
716
+ }
717
+ const extraFiles = opts.extraFiles ?? {};
718
+ for (const [path, content] of Object.entries(extraFiles)) {
719
+ const bytes = Buffer.from(content, "utf8");
720
+ files.push({
721
+ path,
722
+ sizeBytes: bytes.length,
723
+ sha256: createHash3("sha256").update(bytes).digest("hex")
724
+ });
725
+ }
726
+ const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
727
+ kind: "gezel-knowledge-catalog",
728
+ formatVersion: GEZK_FORMAT_VERSION,
729
+ indexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION,
730
+ id: opts.catalog.id,
731
+ version: opts.catalog.version,
732
+ name: opts.catalog.name,
733
+ ...opts.catalog.description ? { description: opts.catalog.description } : {},
734
+ language: opts.catalog.language,
735
+ publisher: opts.catalog.publisher,
736
+ createdAt: opts.catalog.createdAt,
737
+ ...opts.catalog.sourceSnapshot ? { sourceSnapshot: opts.catalog.sourceSnapshot } : {},
738
+ license: opts.catalog.license,
739
+ embedding: profile,
740
+ chunking: opts.chunkingProfile,
741
+ topics: opts.topics.map((t) => ({ id: t.id, name: t.name })),
742
+ router: {
743
+ shardTargetChunks: SHARD_TARGET_CHUNKS,
744
+ shards: routerShardStats.sort((a, b) => a.id - b.id),
745
+ totalCentroids: routerShardStats.reduce((sum, s) => sum + s.centroids, 0)
746
+ },
747
+ counts: { documents: prepared.length, chunks: totalChunks, shards: shardCount },
748
+ files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
749
+ compatibility: { maximumIndexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION },
750
+ ...opts.smokeQueries ? { smokeQueries: opts.smokeQueries } : {},
751
+ ...opts.toolchain ? { toolchain: opts.toolchain } : {}
752
+ });
753
+ const manifest = opts.finalizeManifest ? KnowledgeCatalogManifestSchema.parse(opts.finalizeManifest(unsignedManifest)) : unsignedManifest;
754
+ const archiveEntries = [
755
+ {
756
+ path: MANIFEST_PATH,
757
+ content: Buffer.from(`${JSON.stringify(manifest, null, 2)}
758
+ `, "utf8")
759
+ },
760
+ ...vacuumed.map((f) => ({ path: f.archivePath, absPath: f.absPath })),
761
+ ...Object.entries(extraFiles).map(([path, content]) => ({
762
+ path,
763
+ content: Buffer.from(content, "utf8")
764
+ }))
765
+ ];
766
+ await writeGezkArchive(opts.outputPath, archiveEntries);
767
+ const archiveBytes = (await stat(opts.outputPath)).size;
768
+ rmSync(opts.workDir, { recursive: true, force: true });
769
+ return {
770
+ manifest,
771
+ documents: prepared.length,
772
+ chunks: totalChunks,
773
+ shards: shardCount,
774
+ archiveBytes
775
+ };
776
+ } finally {
777
+ closeAll();
778
+ }
779
+ }
780
+ function createCatalogDb(absPath) {
781
+ const db = new DatabaseSync(absPath, { allowExtension: true });
782
+ loadVecExtension(db);
783
+ db.exec("PRAGMA page_size=8192");
784
+ db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
785
+ db.exec(`PRAGMA user_version=${GEZK_INDEX_SCHEMA_VERSION}`);
786
+ db.exec("PRAGMA journal_mode=DELETE");
787
+ return db;
788
+ }
789
+ function normalizeMarkdown(md) {
790
+ return md.replace(/\r\n/g, "\n").normalize("NFC");
791
+ }
792
+ function buildContextHeader(title, headingPath, maxTokens, countTokens) {
793
+ const path = headingPath.filter((h) => h !== title);
794
+ let header = path.length > 0 ? `${title}
795
+ ${path.join(" > ")}
796
+ ` : `${title}
797
+ `;
798
+ if (countTokens(header) > maxTokens) {
799
+ header = `${title}
800
+ `;
801
+ if (countTokens(header) > maxTokens) header = "";
802
+ }
803
+ return header;
804
+ }
805
+ function seedFor(catalogId, shardId) {
806
+ const digest = createHash3("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
807
+ return digest.readBigUInt64LE(24);
808
+ }
809
+ function makePrng(seed) {
810
+ let state = seed === 0n ? 0x9e3779b97f4a7c15n : seed;
811
+ return () => {
812
+ state ^= state >> 12n;
813
+ state ^= state << 25n & 0xffffffffffffffffn;
814
+ state ^= state >> 27n;
815
+ const out = state * 0x2545f4914f6cdd1dn & 0xffffffffffffffffn;
816
+ return Number(out >> 11n) / 2 ** 53;
817
+ };
818
+ }
819
+ function kMeans(samples, k, seed) {
820
+ if (samples.length === 0) return [];
821
+ const kk = Math.min(k, samples.length);
822
+ const rand = makePrng(seed);
823
+ const dim = samples[0].length;
824
+ const centroids = [
825
+ Float32Array.from(samples[Math.floor(rand() * samples.length)])
826
+ ];
827
+ const dist2 = (a, b) => {
828
+ let dot = 0;
829
+ for (let i = 0; i < dim; i++) dot += a[i] * b[i];
830
+ return Math.max(0, 2 - 2 * dot);
831
+ };
832
+ while (centroids.length < kk) {
833
+ const weights2 = samples.map((s) => Math.min(...centroids.map((c) => dist2(s, c))));
834
+ const total = weights2.reduce((sum, w) => sum + w, 0);
835
+ if (total <= 0) break;
836
+ let target = rand() * total;
837
+ let pick = 0;
838
+ for (let i = 0; i < weights2.length; i++) {
839
+ target -= weights2[i];
840
+ if (target <= 0) {
841
+ pick = i;
842
+ break;
843
+ }
844
+ }
845
+ centroids.push(Float32Array.from(samples[pick]));
846
+ }
847
+ const assignment = new Array(samples.length).fill(0);
848
+ for (let iter = 0; iter < CENTROID_KMEANS_MAX_ITER; iter++) {
849
+ let moved = 0;
850
+ for (let s = 0; s < samples.length; s++) {
851
+ let best = 0;
852
+ let bestD = Number.POSITIVE_INFINITY;
853
+ for (let c = 0; c < centroids.length; c++) {
854
+ const d = dist2(samples[s], centroids[c]);
855
+ if (d < bestD) {
856
+ bestD = d;
857
+ best = c;
858
+ }
859
+ }
860
+ if (assignment[s] !== best) {
861
+ assignment[s] = best;
862
+ moved++;
863
+ }
864
+ }
865
+ const sums = centroids.map(() => new Float64Array(dim));
866
+ const counts = new Array(centroids.length).fill(0);
867
+ for (let s = 0; s < samples.length; s++) {
868
+ const c = assignment[s];
869
+ counts[c] = counts[c] + 1;
870
+ const sum = sums[c];
871
+ const sample = samples[s];
872
+ for (let i = 0; i < dim; i++) sum[i] = sum[i] + sample[i];
873
+ }
874
+ for (let c = 0; c < centroids.length; c++) {
875
+ if (counts[c] === 0) continue;
876
+ const sum = sums[c];
877
+ let norm = 0;
878
+ for (let i = 0; i < dim; i++) norm += sum[i] ** 2;
879
+ norm = Math.sqrt(norm) || 1;
880
+ const centroid = centroids[c];
881
+ for (let i = 0; i < dim; i++) centroid[i] = sum[i] / norm;
882
+ }
883
+ if (moved === 0) break;
884
+ }
885
+ const weights = new Array(centroids.length).fill(0);
886
+ for (const a of assignment) weights[a] = weights[a] + 1;
887
+ return centroids.map((centroid, i) => ({ centroid, weight: weights[i] })).filter((c) => c.weight > 0);
888
+ }
889
+
890
+ // src/archive/read.ts
891
+ import { createHash as createHash4 } from "crypto";
892
+ import { createWriteStream as createWriteStream2 } from "fs";
893
+ import { mkdir as mkdir2 } from "fs/promises";
894
+ import { createRequire as createRequire4 } from "module";
895
+ import { dirname as dirname2, isAbsolute, normalize, relative, resolve } from "path";
896
+ import { Transform } from "stream";
897
+ import { pipeline } from "stream/promises";
898
+ import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/gezel";
899
+ var nodeRequire4 = createRequire4(import.meta.url);
900
+ var GEZK_ARCHIVE_LIMITS = {
901
+ maxEntries: 16384,
902
+ maxManifestBytes: 16 * 1024 * 1024,
903
+ maxEntryBytes: 8 * 1024 * 1024 * 1024,
904
+ maxTotalBytes: 32 * 1024 * 1024 * 1024,
905
+ maxCompressionRatio: 400
906
+ };
907
+ var GezkArchiveError = class extends Error {
908
+ constructor(message, reason) {
909
+ super(message);
910
+ this.reason = reason;
911
+ this.name = "GezkArchiveError";
912
+ }
913
+ reason;
914
+ };
915
+ function openZip(archivePath) {
916
+ const yauzl = nodeRequire4("yauzl");
917
+ return new Promise((resolvePromise, reject) => {
918
+ yauzl.open(
919
+ archivePath,
920
+ { lazyEntries: true, decodeStrings: true, validateEntrySizes: true },
921
+ (err, zip) => {
922
+ if (err || !zip) reject(err ?? new Error("yauzl returned no zip"));
923
+ else resolvePromise(zip);
924
+ }
925
+ );
926
+ });
927
+ }
928
+ function assertSafeEntry(entry, seenLower) {
929
+ const name = entry.fileName;
930
+ if (name.endsWith("/")) throw new GezkArchiveError(`directory entry: ${name}`, "unsafe-entry");
931
+ if (name.includes("\\") || name.startsWith("/") || /(^|\/)\.\.(\/|$)/.test(name) || /^[a-zA-Z]:/.test(name)) {
932
+ throw new GezkArchiveError(`unsafe path: ${name}`, "unsafe-entry");
933
+ }
934
+ if ((entry.generalPurposeBitFlag & 1) !== 0) {
935
+ throw new GezkArchiveError(`encrypted entry: ${name}`, "unsafe-entry");
936
+ }
937
+ const unixMode = entry.externalFileAttributes >>> 16 & 61440;
938
+ if (unixMode === 40960) throw new GezkArchiveError(`symlink entry: ${name}`, "unsafe-entry");
939
+ const lower = name.toLowerCase();
940
+ if (seenLower.has(lower)) {
941
+ throw new GezkArchiveError(`duplicate entry (case-insensitive): ${name}`, "unsafe-entry");
942
+ }
943
+ seenLower.add(lower);
944
+ if (entry.uncompressedSize > GEZK_ARCHIVE_LIMITS.maxEntryBytes) {
945
+ throw new GezkArchiveError(`entry exceeds size limit: ${name}`, "limits");
946
+ }
947
+ if (entry.compressedSize > 0 && entry.uncompressedSize / entry.compressedSize > GEZK_ARCHIVE_LIMITS.maxCompressionRatio) {
948
+ throw new GezkArchiveError(`suspicious compression ratio: ${name}`, "limits");
949
+ }
950
+ }
951
+ async function inspectEntries(archivePath) {
952
+ const zip = await openZip(archivePath);
953
+ const entries = /* @__PURE__ */ new Map();
954
+ const seenLower = /* @__PURE__ */ new Set();
955
+ let total = 0;
956
+ return new Promise((resolvePromise, reject) => {
957
+ zip.on("entry", (entry) => {
958
+ try {
959
+ if (entries.size >= GEZK_ARCHIVE_LIMITS.maxEntries) {
960
+ throw new GezkArchiveError("too many archive entries", "limits");
961
+ }
962
+ assertSafeEntry(entry, seenLower);
963
+ total += entry.uncompressedSize;
964
+ if (total > GEZK_ARCHIVE_LIMITS.maxTotalBytes) {
965
+ throw new GezkArchiveError("archive exceeds total size limit", "limits");
966
+ }
967
+ entries.set(entry.fileName, entry);
968
+ zip.readEntry();
969
+ } catch (err) {
970
+ zip.close();
971
+ reject(err);
972
+ }
973
+ });
974
+ zip.on("end", () => {
975
+ zip.close();
976
+ resolvePromise({ entries, totalUncompressedBytes: total });
977
+ });
978
+ zip.on("error", (err) => {
979
+ zip.close();
980
+ reject(new GezkArchiveError(`archive read failed: ${err.message}`, "io"));
981
+ });
982
+ zip.readEntry();
983
+ });
984
+ }
985
+ function reconcileEntries(manifest, entries) {
986
+ const declared = new Map(manifest.files.map((file) => [file.path, file]));
987
+ for (const path of entries.keys()) {
988
+ if (path === MANIFEST_PATH) continue;
989
+ if (!declared.has(path)) {
990
+ throw new GezkArchiveError(`archive contains undeclared file: ${path}`, "undeclared-file");
991
+ }
992
+ }
993
+ for (const [path, file] of declared) {
994
+ const entry = entries.get(path);
995
+ if (!entry) {
996
+ throw new GezkArchiveError(`archive is missing declared file: ${path}`, "missing-file");
997
+ }
998
+ if (entry.uncompressedSize !== file.sizeBytes) {
999
+ throw new GezkArchiveError(
1000
+ `declared size does not match ZIP metadata for ${path}: expected ${file.sizeBytes}, archive has ${entry.uncompressedSize}`,
1001
+ "hash-mismatch"
1002
+ );
1003
+ }
1004
+ }
1005
+ }
1006
+ async function inspectGezkArchive(archivePath) {
1007
+ const [manifest, inspected] = await Promise.all([
1008
+ readGezkManifest(archivePath),
1009
+ inspectEntries(archivePath)
1010
+ ]);
1011
+ reconcileEntries(manifest, inspected.entries);
1012
+ return {
1013
+ manifest,
1014
+ entryCount: inspected.entries.size,
1015
+ totalUncompressedBytes: inspected.totalUncompressedBytes
1016
+ };
1017
+ }
1018
+ async function readGezkManifest(archivePath) {
1019
+ const zip = await openZip(archivePath);
1020
+ return new Promise((resolvePromise, reject) => {
1021
+ zip.on("entry", (entry) => {
1022
+ if (entry.fileName !== MANIFEST_PATH) {
1023
+ zip.readEntry();
1024
+ return;
1025
+ }
1026
+ if (entry.uncompressedSize > GEZK_ARCHIVE_LIMITS.maxManifestBytes) {
1027
+ zip.close();
1028
+ reject(new GezkArchiveError("manifest exceeds size limit", "limits"));
1029
+ return;
1030
+ }
1031
+ zip.openReadStream(entry, (err, stream) => {
1032
+ if (err || !stream) {
1033
+ zip.close();
1034
+ reject(new GezkArchiveError(`manifest read failed: ${err?.message}`, "io"));
1035
+ return;
1036
+ }
1037
+ const parts = [];
1038
+ stream.on("data", (d) => parts.push(d));
1039
+ stream.on("end", () => {
1040
+ zip.close();
1041
+ try {
1042
+ resolvePromise(
1043
+ KnowledgeCatalogManifestSchema2.parse(
1044
+ JSON.parse(Buffer.concat(parts).toString("utf8"))
1045
+ )
1046
+ );
1047
+ } catch (parseErr) {
1048
+ reject(
1049
+ new GezkArchiveError(
1050
+ `invalid manifest: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,
1051
+ "manifest"
1052
+ )
1053
+ );
1054
+ }
1055
+ });
1056
+ stream.on("error", (streamErr) => {
1057
+ zip.close();
1058
+ reject(new GezkArchiveError(`manifest read failed: ${streamErr.message}`, "io"));
1059
+ });
1060
+ });
1061
+ });
1062
+ zip.on("end", () => {
1063
+ zip.close();
1064
+ reject(new GezkArchiveError("archive has no manifest.json", "manifest"));
1065
+ });
1066
+ zip.on("error", (err) => {
1067
+ zip.close();
1068
+ reject(new GezkArchiveError(`archive read failed: ${err.message}`, "io"));
1069
+ });
1070
+ zip.readEntry();
1071
+ });
1072
+ }
1073
+ async function extractGezkVerified(archivePath, destDir) {
1074
+ const manifest = await readGezkManifest(archivePath);
1075
+ const { entries } = await inspectEntries(archivePath);
1076
+ reconcileEntries(manifest, entries);
1077
+ const declared = new Map(manifest.files.map((file) => [file.path, file]));
1078
+ const destRoot = normalize(resolve(destDir));
1079
+ const zip = await openZip(archivePath);
1080
+ await new Promise((resolvePromise, reject) => {
1081
+ const next = () => zip.readEntry();
1082
+ zip.on("entry", (entry) => {
1083
+ void (async () => {
1084
+ try {
1085
+ const target = resolve(destDir, entry.fileName);
1086
+ const targetRel = relative(destRoot, normalize(target));
1087
+ if (targetRel.startsWith("..") || isAbsolute(targetRel)) {
1088
+ throw new GezkArchiveError(
1089
+ `entry escapes destination: ${entry.fileName}`,
1090
+ "unsafe-entry"
1091
+ );
1092
+ }
1093
+ await mkdir2(dirname2(target), { recursive: true });
1094
+ await new Promise((res, rej) => {
1095
+ zip.openReadStream(entry, (err, stream) => {
1096
+ if (err || !stream) {
1097
+ rej(new GezkArchiveError(`read failed: ${err?.message}`, "io"));
1098
+ return;
1099
+ }
1100
+ const expected = declared.get(entry.fileName);
1101
+ if (!expected) {
1102
+ pipeline(stream, createWriteStream2(target)).then(res, rej);
1103
+ return;
1104
+ }
1105
+ const hasher = createHash4("sha256");
1106
+ let sizeBytes = 0;
1107
+ const verify = new Transform({
1108
+ transform(chunk, _encoding, callback) {
1109
+ sizeBytes += chunk.byteLength;
1110
+ if (sizeBytes > expected.sizeBytes) {
1111
+ callback(
1112
+ new GezkArchiveError(
1113
+ `size mismatch for ${entry.fileName}: expected ${expected.sizeBytes}, got more data`,
1114
+ "hash-mismatch"
1115
+ )
1116
+ );
1117
+ return;
1118
+ }
1119
+ hasher.update(chunk);
1120
+ callback(null, chunk);
1121
+ }
1122
+ });
1123
+ pipeline(stream, verify, createWriteStream2(target)).then(() => {
1124
+ const actual = hasher.digest("hex");
1125
+ if (sizeBytes !== expected.sizeBytes) {
1126
+ throw new GezkArchiveError(
1127
+ `size mismatch for ${entry.fileName}: expected ${expected.sizeBytes}, got ${sizeBytes}`,
1128
+ "hash-mismatch"
1129
+ );
1130
+ }
1131
+ if (actual !== expected.sha256) {
1132
+ throw new GezkArchiveError(
1133
+ `sha256 mismatch for ${entry.fileName}: expected ${expected.sha256}, got ${actual}`,
1134
+ "hash-mismatch"
1135
+ );
1136
+ }
1137
+ }).then(res, rej);
1138
+ });
1139
+ });
1140
+ next();
1141
+ } catch (err) {
1142
+ zip.close();
1143
+ reject(
1144
+ err instanceof GezkArchiveError ? err : new GezkArchiveError(
1145
+ `extraction failed for ${entry.fileName}: ${err instanceof Error ? err.message : String(err)}`,
1146
+ "io"
1147
+ )
1148
+ );
1149
+ }
1150
+ })();
1151
+ });
1152
+ zip.on("end", () => {
1153
+ zip.close();
1154
+ resolvePromise();
1155
+ });
1156
+ zip.on("error", (err) => {
1157
+ zip.close();
1158
+ reject(new GezkArchiveError(`archive read failed: ${err.message}`, "io"));
1159
+ });
1160
+ zip.readEntry();
1161
+ });
1162
+ return manifest;
1163
+ }
1164
+
1165
+ // src/reader/open.ts
1166
+ import { createRequire as createRequire5 } from "module";
1167
+ import { pathToFileURL } from "url";
1168
+ var nodeRequire5 = createRequire5(import.meta.url);
1169
+ var CatalogOpenError = class extends Error {
1170
+ constructor(message, reason) {
1171
+ super(message);
1172
+ this.reason = reason;
1173
+ this.name = "CatalogOpenError";
1174
+ }
1175
+ reason;
1176
+ };
1177
+ function openCatalogDatabase(absPath) {
1178
+ const uri = `${pathToFileURL(absPath).href}?immutable=1`;
1179
+ let db;
1180
+ try {
1181
+ db = new DatabaseSync(uri, { readOnly: true, allowExtension: true });
1182
+ } catch (err) {
1183
+ throw new CatalogOpenError(
1184
+ `cannot open catalog database ${absPath}: ${err instanceof Error ? err.message : String(err)}`,
1185
+ "not-found"
1186
+ );
1187
+ }
1188
+ try {
1189
+ let vecLoaded = false;
1190
+ try {
1191
+ db.enableLoadExtension(true);
1192
+ const sqliteVec = nodeRequire5("sqlite-vec");
1193
+ db.loadExtension(sqliteVec.getLoadablePath());
1194
+ vecLoaded = true;
1195
+ } catch {
1196
+ vecLoaded = false;
1197
+ } finally {
1198
+ try {
1199
+ db.enableLoadExtension(false);
1200
+ } catch {
1201
+ }
1202
+ }
1203
+ const appId = readPragmaNumber(db, "application_id");
1204
+ if (appId !== GEZK_APPLICATION_ID) {
1205
+ throw new CatalogOpenError(
1206
+ `not a .gezk catalog database (application_id ${appId})`,
1207
+ "not-a-catalog"
1208
+ );
1209
+ }
1210
+ const schemaVersion = readPragmaNumber(db, "user_version");
1211
+ if (schemaVersion !== GEZK_INDEX_SCHEMA_VERSION) {
1212
+ throw new CatalogOpenError(
1213
+ `unsupported index schema version ${schemaVersion} (reader supports ${GEZK_INDEX_SCHEMA_VERSION})`,
1214
+ "schema-version"
1215
+ );
1216
+ }
1217
+ if (!vecLoaded) {
1218
+ throw new CatalogOpenError("sqlite-vec extension failed to load", "vec-unavailable");
1219
+ }
1220
+ db.exec("PRAGMA mmap_size=536870912");
1221
+ db.exec("PRAGMA cache_size=-8192");
1222
+ return {
1223
+ db,
1224
+ close: () => {
1225
+ try {
1226
+ db.close();
1227
+ } catch {
1228
+ }
1229
+ }
1230
+ };
1231
+ } catch (err) {
1232
+ try {
1233
+ db.close();
1234
+ } catch {
1235
+ }
1236
+ throw err;
1237
+ }
1238
+ }
1239
+ function readPragmaNumber(db, pragma) {
1240
+ const row = db.prepare(`PRAGMA ${pragma}`).get();
1241
+ const value = row ? Object.values(row)[0] : void 0;
1242
+ return typeof value === "bigint" ? Number(value) : Number(value ?? Number.NaN);
1243
+ }
1244
+
1245
+ // src/reader/catalog-handle.ts
1246
+ import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1247
+ import { brotliDecompressSync } from "zlib";
1248
+ var CatalogHandle = class _CatalogHandle {
1249
+ constructor(rootDir, router) {
1250
+ this.rootDir = rootDir;
1251
+ this.router = router;
1252
+ this.meta = {};
1253
+ for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
1254
+ this.meta[row.key] = row.value;
1255
+ }
1256
+ this.shards = router.db.prepare("SELECT id, path, chunk_count FROM shards ORDER BY id").all().map((r) => ({ id: Number(r.id), path: r.path, chunkCount: Number(r.chunk_count) }));
1257
+ this.connections.set(ROUTER_DB_PATH, router);
1258
+ }
1259
+ rootDir;
1260
+ router;
1261
+ connections = /* @__PURE__ */ new Map();
1262
+ meta;
1263
+ shards;
1264
+ /** Open a catalog's extracted version directory (the dir holding manifest.json). */
1265
+ static open(rootDir) {
1266
+ const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
1267
+ return new _CatalogHandle(rootDir, router);
1268
+ }
1269
+ close() {
1270
+ for (const conn of this.connections.values()) conn.close();
1271
+ this.connections.clear();
1272
+ }
1273
+ /** Integrity check for the router connection, used by deep validation. */
1274
+ routerQuickCheck() {
1275
+ const row = this.router.db.prepare("PRAGMA quick_check").get();
1276
+ return row.quick_check ?? "no result";
1277
+ }
1278
+ /** Metadata-only body profile; SQLite's length() does not materialize blobs. */
1279
+ documentBodyProfile() {
1280
+ const row = this.router.db.prepare(
1281
+ `SELECT
1282
+ COALESCE(MAX(CASE WHEN body_codec = 'none' THEN length(body_blob) ELSE 0 END), 0) AS max_raw,
1283
+ COALESCE(MAX(CASE WHEN body_codec = 'br' THEN length(body_blob) ELSE 0 END), 0) AS max_br,
1284
+ COALESCE(SUM(CASE WHEN body_codec IN ('none', 'br') THEN 0 ELSE 1 END), 0) AS unknown
1285
+ FROM documents`
1286
+ ).get();
1287
+ return {
1288
+ maxRawBytes: Number(row.max_raw),
1289
+ maxCompressedBytes: Number(row.max_br),
1290
+ unknownCodecs: Number(row.unknown)
1291
+ };
1292
+ }
1293
+ shardDb(shard) {
1294
+ let conn = this.connections.get(shard.path);
1295
+ if (!conn) {
1296
+ const abs = this.resolveCatalogPath(shard.path);
1297
+ conn = openCatalogDatabase(abs);
1298
+ this.connections.set(shard.path, conn);
1299
+ }
1300
+ return conn.db;
1301
+ }
1302
+ /** Resolve a manifest/router path without allowing it to escape the catalog. */
1303
+ resolveCatalogPath(path) {
1304
+ const root = resolve2(this.rootDir);
1305
+ const target = resolve2(root, path);
1306
+ const rel = relative2(root, target);
1307
+ if (rel.startsWith("..") || isAbsolute2(rel)) {
1308
+ throw new CatalogOpenError(`path escapes catalog root: ${path}`, "corrupt");
1309
+ }
1310
+ return target;
1311
+ }
1312
+ // ── browsing (the shipped table of contents) ──────────────────────────────
1313
+ topics() {
1314
+ return this.router.db.prepare(
1315
+ "SELECT id, parent_id, name, description, sort_key, document_count FROM topics ORDER BY sort_key"
1316
+ ).all().map((r) => ({
1317
+ id: r.id,
1318
+ parentId: r.parent_id,
1319
+ name: r.name,
1320
+ description: r.description,
1321
+ sortKey: r.sort_key,
1322
+ documentCount: Number(r.document_count)
1323
+ }));
1324
+ }
1325
+ documentsPage(opts = {}) {
1326
+ const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
1327
+ const offset = Math.max(0, opts.offset ?? 0);
1328
+ const where = opts.topicId ? "WHERE topic_id = ?" : "";
1329
+ const params = opts.topicId ? [opts.topicId] : [];
1330
+ const total = Number(
1331
+ this.router.db.prepare(`SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
1332
+ );
1333
+ const documents = this.router.db.prepare(
1334
+ `SELECT id, title, slug, summary, language, topic_id, source_url, source_revision,
1335
+ source_updated_at, attribution_json
1336
+ FROM documents ${where} ORDER BY slug, id LIMIT ? OFFSET ?`
1337
+ ).all(...params, limit, offset).map(rowToDocumentMeta);
1338
+ return { documents, total };
1339
+ }
1340
+ getDocument(id) {
1341
+ const row = this.router.db.prepare(
1342
+ `SELECT id, title, slug, summary, language, topic_id, source_url, source_revision,
1343
+ source_updated_at, attribution_json, body_codec, body_blob
1344
+ FROM documents WHERE id = ?`
1345
+ ).get(id);
1346
+ if (!row) return null;
1347
+ const blob = row.body_blob;
1348
+ if (blob.byteLength > MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024) {
1349
+ throw new CatalogOpenError(`document body exceeds the stored-size limit: ${id}`, "corrupt");
1350
+ }
1351
+ let body;
1352
+ if (row.body_codec === "br") {
1353
+ try {
1354
+ body = brotliDecompressSync(blob, { maxOutputLength: MAX_KNOWLEDGE_DOCUMENT_BYTES });
1355
+ } catch (error) {
1356
+ throw new CatalogOpenError(
1357
+ `document body is invalid or exceeds the decompression limit: ${id} (${error instanceof Error ? error.message : String(error)})`,
1358
+ "corrupt"
1359
+ );
1360
+ }
1361
+ } else if (row.body_codec === "none") {
1362
+ body = Buffer.from(blob);
1363
+ if (body.byteLength > MAX_KNOWLEDGE_DOCUMENT_BYTES) {
1364
+ throw new CatalogOpenError(`document body exceeds the size limit: ${id}`, "corrupt");
1365
+ }
1366
+ } else {
1367
+ throw new CatalogOpenError(`unknown document body codec for ${id}`, "corrupt");
1368
+ }
1369
+ const markdown = body.toString("utf8");
1370
+ return { ...rowToDocumentMeta(row), markdown };
1371
+ }
1372
+ // ── search ────────────────────────────────────────────────────────────────
1373
+ /** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
1374
+ searchDocumentsFts(query, limit = 10) {
1375
+ const match = ftsQuery(query);
1376
+ if (!match) return [];
1377
+ try {
1378
+ return this.router.db.prepare(
1379
+ "SELECT document_id FROM fts_documents WHERE fts_documents MATCH ? ORDER BY rank LIMIT ?"
1380
+ ).all(match, limit).map((r, i) => ({ documentId: r.document_id, rank: i }));
1381
+ } catch {
1382
+ return [];
1383
+ }
1384
+ }
1385
+ /**
1386
+ * Score every shard against the (unit float32) query: shard score = max
1387
+ * cosine over its centroids (§3.4). This is the cross-catalog routing
1388
+ * primitive — the daemon merges scores from every active catalog and
1389
+ * takes the top-S GLOBALLY, so multiple catalogs share one scan budget.
1390
+ */
1391
+ scoreShards(queryVector) {
1392
+ const rows = this.router.db.prepare("SELECT shard_id, embedding FROM route_centroids").all();
1393
+ const best = /* @__PURE__ */ new Map();
1394
+ for (const row of rows) {
1395
+ const centroid = new Float32Array(
1396
+ row.embedding.buffer,
1397
+ row.embedding.byteOffset,
1398
+ Math.floor(row.embedding.byteLength / 4)
1399
+ );
1400
+ let dot = 0;
1401
+ const n = Math.min(centroid.length, queryVector.length);
1402
+ for (let i = 0; i < n; i++) dot += centroid[i] * queryVector[i];
1403
+ const id = Number(row.shard_id);
1404
+ if (dot > (best.get(id) ?? Number.NEGATIVE_INFINITY)) best.set(id, dot);
1405
+ }
1406
+ return [...best.entries()].map(([shardId, score]) => ({ shardId, score }));
1407
+ }
1408
+ /** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
1409
+ routeShards(queryVector, shardBudget) {
1410
+ if (this.shards.length <= shardBudget) return this.shards.map((s) => s.id);
1411
+ return this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
1412
+ }
1413
+ /**
1414
+ * Two-stage semantic search over the routed shards: bit-hamming KNN
1415
+ * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
1416
+ */
1417
+ searchSemantic(queryVector, opts = {}) {
1418
+ const shardIds = this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT);
1419
+ return this.searchShards(queryVector, shardIds, opts.finalK ?? RERANK_FINAL_K);
1420
+ }
1421
+ /**
1422
+ * The scan stage against an EXPLICIT shard list — what the daemon calls
1423
+ * after global cross-catalog routing has already spent the S budget.
1424
+ */
1425
+ searchShards(queryVector, shardIds, finalK) {
1426
+ const queryBits = Buffer.from(quantizeBinary(queryVector));
1427
+ const hits = [];
1428
+ for (const shardId of shardIds) {
1429
+ const shard = this.shards.find((s) => s.id === shardId);
1430
+ if (!shard) continue;
1431
+ const db = this.shardDb(shard);
1432
+ const k = rerankK(finalK, shard.chunkCount);
1433
+ const candidates = db.prepare(
1434
+ "SELECT chunk_id FROM vec_chunks WHERE embedding MATCH vec_bit(?) AND k = ? ORDER BY distance"
1435
+ ).all(queryBits, BigInt(k));
1436
+ if (candidates.length === 0) continue;
1437
+ const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
1438
+ const reranked = candidates.map((c) => {
1439
+ const row = getInt8.get(BigInt(c.chunk_id));
1440
+ if (!row) return null;
1441
+ const int8 = new Int8Array(row.v.buffer, row.v.byteOffset, row.v.byteLength);
1442
+ return { chunkId: Number(c.chunk_id), cosine: rerankScore(queryVector, int8) };
1443
+ }).filter((r) => r !== null).sort((a, b) => b.cosine - a.cosine).slice(0, finalK);
1444
+ const getChunk = db.prepare(
1445
+ `SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
1446
+ FROM chunks WHERE id = ?`
1447
+ );
1448
+ for (const r of reranked) {
1449
+ const row = getChunk.get(BigInt(r.chunkId));
1450
+ if (!row) continue;
1451
+ hits.push({
1452
+ chunkUid: row.chunk_uid,
1453
+ documentId: row.document_id,
1454
+ title: row.title,
1455
+ headingPath: parseHeadingPath(row.heading_path),
1456
+ lineStart: Number(row.line_start),
1457
+ lineEnd: Number(row.line_end),
1458
+ text: row.text,
1459
+ cosine: r.cosine,
1460
+ source: "vector",
1461
+ shardId
1462
+ });
1463
+ }
1464
+ }
1465
+ hits.sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0));
1466
+ return hits;
1467
+ }
1468
+ /** Chunk-body FTS over the routed shards (explicit search only). */
1469
+ searchChunksFts(query, shardIds, limitPerShard = 12) {
1470
+ const match = ftsQuery(query);
1471
+ if (!match) return [];
1472
+ const hits = [];
1473
+ for (const shardId of shardIds) {
1474
+ const shard = this.shards.find((s) => s.id === shardId);
1475
+ if (!shard) continue;
1476
+ const db = this.shardDb(shard);
1477
+ try {
1478
+ const rows = db.prepare(
1479
+ `SELECT c.chunk_uid, c.document_id, c.title, c.heading_path, c.line_start, c.line_end, c.text
1480
+ FROM fts_chunks f JOIN chunks c ON c.id = f.rowid
1481
+ WHERE fts_chunks MATCH ? ORDER BY f.rank LIMIT ?`
1482
+ ).all(match, limitPerShard);
1483
+ for (const row of rows) {
1484
+ hits.push({
1485
+ chunkUid: row.chunk_uid,
1486
+ documentId: row.document_id,
1487
+ title: row.title,
1488
+ headingPath: parseHeadingPath(row.heading_path),
1489
+ lineStart: Number(row.line_start),
1490
+ lineEnd: Number(row.line_end),
1491
+ text: row.text,
1492
+ source: "fts",
1493
+ shardId
1494
+ });
1495
+ }
1496
+ } catch {
1497
+ }
1498
+ }
1499
+ return hits;
1500
+ }
1501
+ /**
1502
+ * Embedder-free integrity smoke (§6.3): chunk 1's own bit vector must be
1503
+ * its own nearest neighbor — proves the extension parses this file's vec
1504
+ * blobs. Returns false instead of throwing (callers report a reason).
1505
+ */
1506
+ selfKnnSmoke(shardId) {
1507
+ const shard = this.shards.find((s) => s.id === shardId);
1508
+ if (!shard) return false;
1509
+ try {
1510
+ const db = this.shardDb(shard);
1511
+ const first = db.prepare("SELECT embedding FROM vec_chunks WHERE chunk_id = 1").get();
1512
+ if (!first) return false;
1513
+ const nearest = db.prepare(
1514
+ "SELECT chunk_id FROM vec_chunks WHERE embedding MATCH vec_bit(?) AND k = 1 ORDER BY distance"
1515
+ ).get(Buffer.from(first.embedding));
1516
+ return nearest !== void 0 && Number(nearest.chunk_id) === 1;
1517
+ } catch {
1518
+ return false;
1519
+ }
1520
+ }
1521
+ };
1522
+ function rowToDocumentMeta(row) {
1523
+ let attribution = null;
1524
+ if (typeof row.attribution_json === "string") {
1525
+ try {
1526
+ attribution = JSON.parse(row.attribution_json);
1527
+ } catch {
1528
+ attribution = null;
1529
+ }
1530
+ }
1531
+ return {
1532
+ id: row.id,
1533
+ title: row.title,
1534
+ slug: row.slug,
1535
+ summary: row.summary ?? null,
1536
+ language: row.language,
1537
+ topicId: row.topic_id,
1538
+ sourceUrl: row.source_url ?? null,
1539
+ sourceRevision: row.source_revision ?? null,
1540
+ sourceUpdatedAt: row.source_updated_at ?? null,
1541
+ attribution
1542
+ };
1543
+ }
1544
+ function parseHeadingPath(raw) {
1545
+ try {
1546
+ const parsed = JSON.parse(raw);
1547
+ return Array.isArray(parsed) ? parsed.filter((p) => typeof p === "string") : [];
1548
+ } catch {
1549
+ return [];
1550
+ }
1551
+ }
1552
+ function ftsQuery(query) {
1553
+ const tokens = [
1554
+ ...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
1555
+ ];
1556
+ if (tokens.length === 0) return null;
1557
+ return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
1558
+ }
1559
+
1560
+ // src/reader/validate.ts
1561
+ import { readFile } from "fs/promises";
1562
+ import { join as join3 } from "path";
1563
+ import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/gezel";
1564
+ async function validateExtractedCatalog(rootDir, opts = {}) {
1565
+ const checks = [];
1566
+ const check = (name, ok, detail) => {
1567
+ checks.push({ name, ok, ...detail ? { detail } : {} });
1568
+ return ok;
1569
+ };
1570
+ const report = (manifest2) => ({
1571
+ ok: checks.every((c) => c.ok),
1572
+ manifest: manifest2,
1573
+ checks
1574
+ });
1575
+ let manifest;
1576
+ try {
1577
+ manifest = KnowledgeCatalogManifestSchema3.parse(
1578
+ JSON.parse(await readFile(join3(rootDir, MANIFEST_PATH), "utf8"))
1579
+ );
1580
+ check("manifest-parse", true);
1581
+ } catch (err) {
1582
+ check("manifest-parse", false, err instanceof Error ? err.message : String(err));
1583
+ return report(null);
1584
+ }
1585
+ for (const file of manifest.files) {
1586
+ try {
1587
+ const actual = await hashFileStreaming(
1588
+ join3(rootDir, file.path),
1589
+ GEZK_ARCHIVE_LIMITS.maxEntryBytes
1590
+ );
1591
+ if (actual.sha256 !== file.sha256) {
1592
+ check(
1593
+ `file:${file.path}`,
1594
+ false,
1595
+ `sha256 mismatch (expected ${file.sha256}, got ${actual.sha256})`
1596
+ );
1597
+ } else if (actual.sizeBytes !== file.sizeBytes) {
1598
+ check(
1599
+ `file:${file.path}`,
1600
+ false,
1601
+ `size mismatch (expected ${file.sizeBytes}, got ${actual.sizeBytes})`
1602
+ );
1603
+ } else {
1604
+ check(`file:${file.path}`, true);
1605
+ }
1606
+ } catch (err) {
1607
+ check(`file:${file.path}`, false, err instanceof Error ? err.message : String(err));
1608
+ }
1609
+ }
1610
+ if (!checks.every((c) => c.ok)) return report(manifest);
1611
+ let handle;
1612
+ try {
1613
+ handle = CatalogHandle.open(rootDir);
1614
+ check("router-open", true);
1615
+ } catch (err) {
1616
+ check("router-open", false, err instanceof Error ? err.message : String(err));
1617
+ return report(manifest);
1618
+ }
1619
+ try {
1620
+ check(
1621
+ "meta-echo",
1622
+ handle.meta.catalog_id === manifest.id && handle.meta.catalog_version === manifest.version,
1623
+ `router meta says ${handle.meta.catalog_id}@${handle.meta.catalog_version}`
1624
+ );
1625
+ check(
1626
+ "meta-profile",
1627
+ handle.meta.embedding_profile_id === manifest.embedding.id,
1628
+ `router meta profile ${handle.meta.embedding_profile_id}`
1629
+ );
1630
+ const topics = handle.topics();
1631
+ check("toc-present", topics.length >= 1, `${topics.length} topics`);
1632
+ const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
1633
+ check(
1634
+ "counts-documents",
1635
+ handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum === manifest.counts.documents,
1636
+ `documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, manifest ${manifest.counts.documents}`
1637
+ );
1638
+ check(
1639
+ "counts-shards",
1640
+ handle.shards.length === manifest.counts.shards,
1641
+ `shards table ${handle.shards.length}, manifest ${manifest.counts.shards}`
1642
+ );
1643
+ const shardChunkSum = handle.shards.reduce((sum, s) => sum + s.chunkCount, 0);
1644
+ check(
1645
+ "counts-chunks",
1646
+ shardChunkSum === manifest.counts.chunks,
1647
+ `shard rows sum ${shardChunkSum}, manifest ${manifest.counts.chunks}`
1648
+ );
1649
+ if (opts.deep) {
1650
+ const routerQuick = handle.routerQuickCheck();
1651
+ check("quick-check:index/router.db", routerQuick === "ok", routerQuick);
1652
+ const bodies = handle.documentBodyProfile();
1653
+ check(
1654
+ "document-body-codecs",
1655
+ bodies.unknownCodecs === 0,
1656
+ `${bodies.unknownCodecs} documents use an unknown body codec`
1657
+ );
1658
+ check(
1659
+ "document-body-raw-limit",
1660
+ bodies.maxRawBytes <= MAX_KNOWLEDGE_DOCUMENT_BYTES,
1661
+ `largest raw body is ${bodies.maxRawBytes} bytes`
1662
+ );
1663
+ check(
1664
+ "document-body-compressed-limit",
1665
+ bodies.maxCompressedBytes <= MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024,
1666
+ `largest compressed body is ${bodies.maxCompressedBytes} bytes`
1667
+ );
1668
+ for (const shard of handle.shards) {
1669
+ const conn = openCatalogDatabase(handle.resolveCatalogPath(shard.path));
1670
+ const db = conn.db;
1671
+ try {
1672
+ const quick = db.prepare("PRAGMA quick_check").get();
1673
+ check(
1674
+ `quick-check:${shard.path}`,
1675
+ quick.quick_check === "ok",
1676
+ quick.quick_check ?? "no result"
1677
+ );
1678
+ const chunkCount = Number(
1679
+ db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n
1680
+ );
1681
+ const vecCount = Number(
1682
+ db.prepare("SELECT COUNT(*) AS n FROM vec_chunks").get().n
1683
+ );
1684
+ const int8Count = Number(
1685
+ db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8").get().n
1686
+ );
1687
+ check(
1688
+ `vectors-aligned:${shard.path}`,
1689
+ chunkCount === shard.chunkCount && vecCount === chunkCount && int8Count === chunkCount,
1690
+ `chunks ${chunkCount}, vec ${vecCount}, int8 ${int8Count}, shards row ${shard.chunkCount}`
1691
+ );
1692
+ } finally {
1693
+ conn.close();
1694
+ }
1695
+ check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
1696
+ }
1697
+ for (const smoke of manifest.smokeQueries ?? []) {
1698
+ const hits = handle.searchDocumentsFts(smoke.query, 10).map((h) => h.documentId);
1699
+ const missing = smoke.expectedDocumentIds.filter((id) => !hits.includes(id));
1700
+ check(
1701
+ `smoke:${smoke.query}`,
1702
+ missing.length === 0,
1703
+ missing.length > 0 ? `missing ${missing.join(", ")}` : void 0
1704
+ );
1705
+ }
1706
+ }
1707
+ } finally {
1708
+ handle.close();
1709
+ }
1710
+ return report(manifest);
1711
+ }
1712
+
1713
+ // src/profiles/registry.ts
1714
+ var SHARED_ENCODING = {
1715
+ pooling: "mean",
1716
+ normalized: true,
1717
+ dimensions: 384,
1718
+ maxTokens: 512,
1719
+ vectorEncoding: "bit384+int8",
1720
+ distance: { stage1: "hamming", stage2: "cosine" },
1721
+ quantization: {
1722
+ int8: { method: "symmetric-linear", scale: 127 },
1723
+ binary: { method: "sign", threshold: 0, packing: "lsb-first" }
1724
+ }
1725
+ };
1726
+ var GEZEL_MULTILINGUAL_E5_SMALL_1 = {
1727
+ id: "gezel-multilingual-e5-small@1",
1728
+ model: {
1729
+ repo: "Xenova/multilingual-e5-small",
1730
+ revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78"
1731
+ },
1732
+ tokenizer: { kind: "sentencepiece-xlmr" },
1733
+ queryInstruction: "query: ",
1734
+ passageInstruction: "passage: ",
1735
+ ...SHARED_ENCODING
1736
+ };
1737
+ var GEZEL_BGE_SMALL_EN_V15_1 = {
1738
+ id: "gezel-bge-small-en-v1.5@1",
1739
+ model: {
1740
+ repo: "Xenova/bge-small-en-v1.5",
1741
+ revision: "ea104dacec62c0de699686887e3f920caeb4f3e3"
1742
+ },
1743
+ tokenizer: { kind: "wordpiece-bert" },
1744
+ queryInstruction: "Represent this sentence for searching relevant passages: ",
1745
+ passageInstruction: "",
1746
+ ...SHARED_ENCODING
1747
+ };
1748
+ var KNOWLEDGE_EMBEDDING_PROFILES = [
1749
+ GEZEL_MULTILINGUAL_E5_SMALL_1,
1750
+ GEZEL_BGE_SMALL_EN_V15_1
1751
+ ];
1752
+ function knowledgeEmbeddingProfile(id) {
1753
+ return KNOWLEDGE_EMBEDDING_PROFILES.find((p) => p.id === id) ?? null;
1754
+ }
1755
+ var GEZEL_MARKDOWN_CHUNKS_2 = {
1756
+ id: "gezel-markdown-chunks@2",
1757
+ unit: "tokens",
1758
+ tokenizer: "profile",
1759
+ targetTokens: 420,
1760
+ overlapTokens: 64,
1761
+ contextHeader: { maxTokens: 64 }
1762
+ };
1763
+
1764
+ // src/embedding/profile-embedder.ts
1765
+ var MAX_BATCH = 8;
1766
+ var MAX_CHARS = 8e3;
1767
+ var EmbedderUnavailableError = class extends Error {
1768
+ isActionable = true;
1769
+ constructor(message) {
1770
+ super(message);
1771
+ this.name = "EmbedderUnavailableError";
1772
+ }
1773
+ };
1774
+ async function loadTransformers() {
1775
+ try {
1776
+ const specifier = "@huggingface/transformers";
1777
+ return await import(specifier);
1778
+ } catch (err) {
1779
+ const message = err instanceof Error ? err.message : String(err);
1780
+ if (/Cannot find (module|package)/i.test(message)) {
1781
+ throw new EmbedderUnavailableError(
1782
+ "the embedding runtime (@huggingface/transformers) is not installed \u2014 knowledge builds and semantic search need it; install it or run inside a gezel install that ships it"
1783
+ );
1784
+ }
1785
+ throw err;
1786
+ }
1787
+ }
1788
+ async function createProfileEmbedder(profile, opts = {}) {
1789
+ const transformers = await loadTransformers();
1790
+ const cacheDir = opts.cacheDir ?? process.env.GEZEL_HF_CACHE_DIR;
1791
+ if (cacheDir) {
1792
+ transformers.env.cacheDir = cacheDir;
1793
+ transformers.env.useFSCache = true;
1794
+ transformers.env.allowRemoteModels = true;
1795
+ }
1796
+ const modelOpts = { revision: profile.model.revision };
1797
+ const [pipe, tokenizer] = await Promise.all([
1798
+ transformers.pipeline("feature-extraction", profile.model.repo, modelOpts),
1799
+ transformers.AutoTokenizer.from_pretrained(profile.model.repo, modelOpts)
1800
+ ]);
1801
+ const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
1802
+ const embed = async (texts) => {
1803
+ const out = [];
1804
+ for (let i = 0; i < texts.length; i += MAX_BATCH) {
1805
+ const slice = texts.slice(i, i + MAX_BATCH).map(cap);
1806
+ const result = await pipe(slice, { pooling: "mean", normalize: true });
1807
+ out.push(...result.tolist());
1808
+ }
1809
+ return out;
1810
+ };
1811
+ return {
1812
+ profile,
1813
+ embed,
1814
+ embedQuery: async (text) => {
1815
+ const [vector] = await embed([`${profile.queryInstruction}${text}`]);
1816
+ return Float32Array.from(vector ?? []);
1817
+ },
1818
+ countTokens: (text) => tokenizer.encode(text).length,
1819
+ dispose: async () => {
1820
+ await pipe.dispose?.();
1821
+ }
1822
+ };
1823
+ }
1824
+
1825
+ // src/markdown-adapter/load.ts
1826
+ import { readFile as readFile2, readdir } from "fs/promises";
1827
+ import { join as join4, relative as relative3, sep } from "path";
1828
+ import { CatalogDocumentSchema } from "@bendyline/gezel";
1829
+ var ROOT_TOPIC_ID = "general";
1830
+ var SUMMARY_MAX_CHARS = 280;
1831
+ function parseFrontMatter(markdown) {
1832
+ if (!markdown.startsWith("---\n") && !markdown.startsWith("---\r\n")) return { body: markdown };
1833
+ const end = markdown.indexOf("\n---", 3);
1834
+ if (end === -1) return { body: markdown };
1835
+ const head = markdown.slice(markdown.indexOf("\n") + 1, end);
1836
+ const body = markdown.slice(markdown.indexOf("\n", end + 1) + 1);
1837
+ const out = { body };
1838
+ for (const line of head.split("\n")) {
1839
+ const at = line.indexOf(":");
1840
+ if (at === -1) continue;
1841
+ const key = line.slice(0, at).trim();
1842
+ const value = line.slice(at + 1).trim();
1843
+ if (!value) continue;
1844
+ if (key === "title") out.title = value;
1845
+ else if (key === "summary") out.summary = value;
1846
+ else if (key === "aliases") {
1847
+ out.aliases = value.split(",").map((a) => a.trim()).filter(Boolean);
1848
+ }
1849
+ }
1850
+ return out;
1851
+ }
1852
+ function firstHeading(markdown) {
1853
+ const match = /^#\s+(.+)$/m.exec(markdown);
1854
+ return match?.[1]?.trim() || void 0;
1855
+ }
1856
+ function firstParagraph(markdown) {
1857
+ for (const block of markdown.split(/\n\s*\n/)) {
1858
+ const text = block.trim();
1859
+ if (!text || text.startsWith("#") || text.startsWith("```") || text.startsWith("|")) continue;
1860
+ const flat = text.replace(/\s+/g, " ");
1861
+ return flat.length > SUMMARY_MAX_CHARS ? `${flat.slice(0, SUMMARY_MAX_CHARS - 1)}\u2026` : flat;
1862
+ }
1863
+ return void 0;
1864
+ }
1865
+ function topicIdFor(name, taken) {
1866
+ let base = documentSlug(name).slice(0, 60).replace(/-+$/, "");
1867
+ if (!base) base = "topic";
1868
+ let id = base;
1869
+ let n = 2;
1870
+ while (taken.has(id)) id = `${base}-${n++}`;
1871
+ taken.add(id);
1872
+ return id;
1873
+ }
1874
+ async function walkMarkdownFiles(root) {
1875
+ const out = [];
1876
+ const visit = async (dir) => {
1877
+ const entries = (await readdir(dir, { withFileTypes: true })).sort(
1878
+ (a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0
1879
+ );
1880
+ for (const entry of entries) {
1881
+ if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
1882
+ const abs = join4(dir, entry.name);
1883
+ if (entry.isDirectory()) await visit(abs);
1884
+ else if (entry.isFile() && /\.(md|markdown)$/i.test(entry.name)) out.push(abs);
1885
+ }
1886
+ };
1887
+ await visit(root);
1888
+ return out;
1889
+ }
1890
+ async function loadMarkdownCatalog(rootDir, opts) {
1891
+ const files = await walkMarkdownFiles(rootDir);
1892
+ if (files.length === 0) {
1893
+ throw new Error(`no Markdown files found under ${rootDir}`);
1894
+ }
1895
+ const takenTopicIds = /* @__PURE__ */ new Set();
1896
+ const topicIdByDir = /* @__PURE__ */ new Map();
1897
+ const topics = [];
1898
+ const rootTopicId = opts.rootTopicId ?? ROOT_TOPIC_ID;
1899
+ let rootTopicUsed = false;
1900
+ const ensureTopicChain = (relDirPosix) => {
1901
+ if (!relDirPosix) {
1902
+ rootTopicUsed = true;
1903
+ return [rootTopicId];
1904
+ }
1905
+ const segments = relDirPosix.split("/");
1906
+ const path = [];
1907
+ let prefix = "";
1908
+ for (const segment of segments) {
1909
+ prefix = prefix ? `${prefix}/${segment}` : segment;
1910
+ let id = topicIdByDir.get(prefix);
1911
+ if (!id) {
1912
+ id = topicIdFor(segment, takenTopicIds);
1913
+ topicIdByDir.set(prefix, id);
1914
+ topics.push({
1915
+ id,
1916
+ name: segment,
1917
+ ...path.length > 0 ? { parentId: path[path.length - 1] } : {}
1918
+ });
1919
+ }
1920
+ path.push(id);
1921
+ }
1922
+ return path;
1923
+ };
1924
+ const documents = [];
1925
+ const seenIds = /* @__PURE__ */ new Set();
1926
+ for (const abs of files) {
1927
+ const rel = relative3(rootDir, abs).split(sep).join("/");
1928
+ const relDir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
1929
+ const topicPath = ensureTopicChain(relDir);
1930
+ const id = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
1931
+ if (seenIds.has(id)) throw new Error(`duplicate document id from path: ${rel}`);
1932
+ seenIds.add(id);
1933
+ const raw = (await readFile2(abs, "utf8")).replace(/^\uFEFF/, "");
1934
+ const fm = parseFrontMatter(raw);
1935
+ const stem = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
1936
+ const title = fm.title ?? firstHeading(fm.body) ?? stem;
1937
+ const summary = fm.summary ?? firstParagraph(fm.body);
1938
+ documents.push(
1939
+ CatalogDocumentSchema.parse({
1940
+ id,
1941
+ title,
1942
+ slug: documentSlug(title),
1943
+ ...summary ? { summary } : {},
1944
+ language: opts.language,
1945
+ topicPath,
1946
+ markdown: fm.body,
1947
+ ...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {}
1948
+ })
1949
+ );
1950
+ }
1951
+ if (rootTopicUsed) {
1952
+ if (takenTopicIds.has(rootTopicId)) {
1953
+ throw new Error(
1954
+ `root topic id '${rootTopicId}' collides with a folder-derived topic; pass rootTopicId`
1955
+ );
1956
+ }
1957
+ topics.unshift({ id: rootTopicId, name: opts.rootTopicName ?? "General" });
1958
+ }
1959
+ return { topics, documents };
1960
+ }
1961
+
1962
+ // src/signatures/jcs.ts
1963
+ function canonicalizeJson(value) {
1964
+ return serialize(value, "$");
1965
+ }
1966
+ function serialize(value, path) {
1967
+ if (value === null) return "null";
1968
+ switch (typeof value) {
1969
+ case "boolean":
1970
+ return value ? "true" : "false";
1971
+ case "number":
1972
+ if (!Number.isFinite(value)) throw new Error(`non-finite number at ${path}`);
1973
+ return JSON.stringify(value);
1974
+ case "string":
1975
+ return JSON.stringify(value);
1976
+ case "object":
1977
+ break;
1978
+ default:
1979
+ throw new Error(`value of type ${typeof value} at ${path} has no JSON identity`);
1980
+ }
1981
+ if (Array.isArray(value)) {
1982
+ return `[${value.map((item, i) => serialize(item ?? null, `${path}[${i}]`)).join(",")}]`;
1983
+ }
1984
+ const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => compareUtf16(a, b));
1985
+ return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${serialize(v, `${path}.${k}`)}`).join(",")}}`;
1986
+ }
1987
+ function compareUtf16(a, b) {
1988
+ const n = Math.min(a.length, b.length);
1989
+ for (let i = 0; i < n; i++) {
1990
+ const d = a.charCodeAt(i) - b.charCodeAt(i);
1991
+ if (d !== 0) return d;
1992
+ }
1993
+ return a.length - b.length;
1994
+ }
1995
+
1996
+ // src/signatures/signing.ts
1997
+ import {
1998
+ createHash as createHash5,
1999
+ createPrivateKey,
2000
+ createPublicKey,
2001
+ sign as cryptoSign,
2002
+ verify as cryptoVerify,
2003
+ generateKeyPairSync
2004
+ } from "crypto";
2005
+ function knowledgeKeyId(publicKeyPem) {
2006
+ const der = createPublicKey(publicKeyPem).export({ type: "spki", format: "der" });
2007
+ return createHash5("sha256").update(der).digest("hex").slice(0, 16);
2008
+ }
2009
+ function generateKnowledgeSigningKeyPair() {
2010
+ const { publicKey, privateKey } = generateKeyPairSync("ed25519");
2011
+ const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }).toString();
2012
+ return {
2013
+ publicKeyPem,
2014
+ privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
2015
+ keyId: knowledgeKeyId(publicKeyPem)
2016
+ };
2017
+ }
2018
+ function manifestSigningPayload(manifest) {
2019
+ const { signature: _omitted, ...unsigned } = manifest;
2020
+ return Buffer.from(canonicalizeJson(unsigned), "utf8");
2021
+ }
2022
+ function signManifest(manifest, privateKeyPem) {
2023
+ const key = createPrivateKey(privateKeyPem);
2024
+ const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
2025
+ const value = cryptoSign(null, manifestSigningPayload(manifest), key).toString("base64");
2026
+ return {
2027
+ ...manifest,
2028
+ signature: {
2029
+ algorithm: "ed25519",
2030
+ keyId: knowledgeKeyId(publicKeyPem),
2031
+ canonicalization: "rfc8785",
2032
+ value
2033
+ }
2034
+ };
2035
+ }
2036
+ function verifyManifestSignature(manifest, anchors) {
2037
+ const signature = manifest.signature;
2038
+ if (!signature) return { ok: false, reason: "unsigned" };
2039
+ const anchor = anchors.find((a) => a.keyId === signature.keyId);
2040
+ if (!anchor) {
2041
+ return { ok: false, reason: "unknown-key", detail: `no trust anchor for ${signature.keyId}` };
2042
+ }
2043
+ try {
2044
+ if (knowledgeKeyId(anchor.publicKeyPem) !== signature.keyId) {
2045
+ return { ok: false, reason: "unknown-key", detail: "anchor keyId does not match its key" };
2046
+ }
2047
+ const ok = cryptoVerify(
2048
+ null,
2049
+ manifestSigningPayload(manifest),
2050
+ createPublicKey(anchor.publicKeyPem),
2051
+ Buffer.from(signature.value, "base64")
2052
+ );
2053
+ return ok ? { ok: true, keyId: signature.keyId } : { ok: false, reason: "bad-signature" };
2054
+ } catch (err) {
2055
+ return { ok: false, reason: "error", detail: err instanceof Error ? err.message : String(err) };
2056
+ }
2057
+ }
2058
+ export {
2059
+ BODY_CODEC_MIN_BYTES,
2060
+ CENTROID_CHUNKS_PER,
2061
+ CENTROID_KMEANS_MAX_ITER,
2062
+ CENTROID_MAX_PER_SHARD,
2063
+ CENTROID_SAMPLE_MAX,
2064
+ CatalogHandle,
2065
+ CatalogOpenError,
2066
+ EmbedderUnavailableError,
2067
+ GEZEL_BGE_SMALL_EN_V15_1,
2068
+ GEZEL_MARKDOWN_CHUNKS_2,
2069
+ GEZEL_MULTILINGUAL_E5_SMALL_1,
2070
+ GEZK_APPLICATION_ID,
2071
+ GEZK_ARCHIVE_LIMITS,
2072
+ GEZK_FORMAT_VERSION,
2073
+ GEZK_INDEX_SCHEMA_VERSION,
2074
+ GezkArchiveError,
2075
+ KNOWLEDGE_EMBEDDING_PROFILES,
2076
+ MANIFEST_PATH,
2077
+ MAX_KNOWLEDGE_DOCUMENT_BYTES,
2078
+ RERANK_FINAL_K,
2079
+ ROUTER_DB_PATH,
2080
+ ROUTER_DDL,
2081
+ ROUTE_SHARDS_EXPLICIT,
2082
+ ROUTE_SHARDS_PROACTIVE,
2083
+ SHARD_DDL,
2084
+ SHARD_MAX_CHUNKS,
2085
+ SHARD_TARGET_CHUNKS,
2086
+ ZIP_FIXED_MTIME,
2087
+ canonicalizeJson,
2088
+ chunkContentHash,
2089
+ chunkMarkdownProfile,
2090
+ chunkUid,
2091
+ compileKnowledgeCatalog,
2092
+ createProfileEmbedder,
2093
+ documentSlug,
2094
+ extractGezkVerified,
2095
+ generateKnowledgeSigningKeyPair,
2096
+ inspectGezkArchive,
2097
+ knowledgeEmbeddingProfile,
2098
+ knowledgeKeyId,
2099
+ l2Normalize,
2100
+ loadMarkdownCatalog,
2101
+ manifestSigningPayload,
2102
+ openCatalogDatabase,
2103
+ quantizeBinary,
2104
+ quantizeInt8,
2105
+ readGezkManifest,
2106
+ rerankK,
2107
+ rerankScore,
2108
+ signManifest,
2109
+ validateExtractedCatalog,
2110
+ vecChunksDdl,
2111
+ verifyManifestSignature,
2112
+ writeGezkArchive
2113
+ };