@bendyline/gezel-knowledge 1.0.6 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -17
- package/dist/index.d.ts +482 -137
- package/dist/index.js +1233 -397
- package/package.json +6 -5
package/dist/index.js
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
1
|
// src/format/constants.ts
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
import {
|
|
3
|
+
BODY_CODEC_MIN_BYTES,
|
|
4
|
+
GEZK_APPLICATION_ID,
|
|
5
|
+
GEZK_FORMAT_VERSION,
|
|
6
|
+
GEZK_INDEX_SCHEMA_VERSION,
|
|
7
|
+
GEZK_MIME_TYPE,
|
|
8
|
+
LICENSE_NOTICE_PATH,
|
|
9
|
+
MANIFEST_PATH,
|
|
10
|
+
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
11
|
+
MIMETYPE_PATH,
|
|
12
|
+
README_PATH,
|
|
13
|
+
ROUTER_DB_PATH,
|
|
14
|
+
SOURCE_NOTICES_PATH,
|
|
15
|
+
ZIP_FIXED_MTIME
|
|
16
|
+
} from "@bendyline/gezk";
|
|
7
17
|
var SHARD_TARGET_CHUNKS = 2e5;
|
|
8
18
|
var SHARD_MAX_CHUNKS = 25e4;
|
|
9
|
-
var BODY_CODEC_MIN_BYTES = 512;
|
|
10
|
-
var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
|
|
11
19
|
var CENTROID_CHUNKS_PER = 12500;
|
|
12
20
|
var CENTROID_MAX_PER_SHARD = 32;
|
|
13
21
|
var CENTROID_KMEANS_MAX_ITER = 25;
|
|
@@ -18,144 +26,16 @@ var RERANK_FINAL_K = 24;
|
|
|
18
26
|
function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
|
|
19
27
|
return Math.min(512, Math.max(128, 8 * finalK), chunkCount);
|
|
20
28
|
}
|
|
21
|
-
var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
|
|
22
29
|
|
|
23
30
|
// src/format/ddl.ts
|
|
24
|
-
|
|
25
|
-
CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
26
|
-
|
|
27
|
-
CREATE TABLE topics(
|
|
28
|
-
id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,
|
|
29
|
-
description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL
|
|
30
|
-
);
|
|
31
|
-
|
|
32
|
-
CREATE TABLE documents(
|
|
33
|
-
id TEXT PRIMARY KEY,
|
|
34
|
-
title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
|
|
35
|
-
language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
|
|
36
|
-
shard_id INTEGER NOT NULL,
|
|
37
|
-
chunk_count INTEGER NOT NULL,
|
|
38
|
-
source_url TEXT, source_revision TEXT, source_updated_at TEXT,
|
|
39
|
-
attribution_json TEXT,
|
|
40
|
-
body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
|
|
41
|
-
body_blob BLOB NOT NULL
|
|
42
|
-
);
|
|
43
|
-
CREATE INDEX documents_topic ON documents(topic_id, slug);
|
|
44
|
-
CREATE INDEX documents_shard ON documents(shard_id);
|
|
45
|
-
|
|
46
|
-
CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
|
|
47
|
-
PRIMARY KEY (alias, document_id)) WITHOUT ROWID;
|
|
48
|
-
|
|
49
|
-
CREATE TABLE shards(
|
|
50
|
-
id INTEGER PRIMARY KEY,
|
|
51
|
-
path TEXT NOT NULL,
|
|
52
|
-
chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,
|
|
53
|
-
topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,
|
|
54
|
-
bytes INTEGER NOT NULL
|
|
55
|
-
);
|
|
56
|
-
|
|
57
|
-
CREATE TABLE route_centroids(
|
|
58
|
-
id INTEGER PRIMARY KEY,
|
|
59
|
-
shard_id INTEGER NOT NULL REFERENCES shards(id),
|
|
60
|
-
embedding BLOB NOT NULL,
|
|
61
|
-
weight INTEGER NOT NULL
|
|
62
|
-
);
|
|
63
|
-
CREATE INDEX route_centroids_shard ON route_centroids(shard_id);
|
|
64
|
-
|
|
65
|
-
CREATE VIRTUAL TABLE fts_documents USING fts5(
|
|
66
|
-
title, summary, aliases, document_id UNINDEXED,
|
|
67
|
-
tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
|
|
68
|
-
);
|
|
69
|
-
`;
|
|
70
|
-
var SHARD_DDL = `
|
|
71
|
-
CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
72
|
-
|
|
73
|
-
CREATE TABLE chunks(
|
|
74
|
-
id INTEGER PRIMARY KEY,
|
|
75
|
-
chunk_uid TEXT NOT NULL,
|
|
76
|
-
document_id TEXT NOT NULL,
|
|
77
|
-
ordinal INTEGER NOT NULL,
|
|
78
|
-
title TEXT NOT NULL,
|
|
79
|
-
heading_path TEXT NOT NULL,
|
|
80
|
-
heading_text TEXT NOT NULL,
|
|
81
|
-
line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,
|
|
82
|
-
token_count INTEGER NOT NULL,
|
|
83
|
-
content_hash TEXT NOT NULL,
|
|
84
|
-
text TEXT NOT NULL
|
|
85
|
-
);
|
|
86
|
-
CREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);
|
|
87
|
-
CREATE INDEX chunks_document ON chunks(document_id, ordinal);
|
|
88
|
-
|
|
89
|
-
CREATE VIRTUAL TABLE fts_chunks USING fts5(
|
|
90
|
-
title, heading_text, text,
|
|
91
|
-
content = 'chunks', content_rowid = 'id',
|
|
92
|
-
tokenize = 'unicode61 remove_diacritics 2'
|
|
93
|
-
);
|
|
94
|
-
|
|
95
|
-
CREATE TABLE chunk_vectors_int8(
|
|
96
|
-
chunk_id INTEGER PRIMARY KEY,
|
|
97
|
-
v BLOB NOT NULL
|
|
98
|
-
);
|
|
99
|
-
`;
|
|
100
|
-
function vecChunksDdl(dim) {
|
|
101
|
-
return `CREATE VIRTUAL TABLE vec_chunks USING vec0(
|
|
102
|
-
chunk_id INTEGER PRIMARY KEY,
|
|
103
|
-
embedding bit[${dim}],
|
|
104
|
-
chunk_size=1024
|
|
105
|
-
);`;
|
|
106
|
-
}
|
|
31
|
+
import { ROUTER_DDL, SHARD_DDL } from "@bendyline/gezk";
|
|
107
32
|
|
|
108
33
|
// src/format/ids.ts
|
|
109
|
-
import {
|
|
110
|
-
|
|
111
|
-
const textHash = createHash("sha256").update(chunkText, "utf8").digest();
|
|
112
|
-
return createHash("sha256").update(documentId, "utf8").update(Buffer.from([0])).update(String(ordinal), "utf8").update(Buffer.from([0])).update(textHash).digest("hex").slice(0, 32);
|
|
113
|
-
}
|
|
114
|
-
function chunkContentHash(chunkText) {
|
|
115
|
-
return createHash("sha256").update(chunkText, "utf8").digest("hex");
|
|
116
|
-
}
|
|
117
|
-
function documentSlug(title) {
|
|
118
|
-
const slug = title.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
119
|
-
return slug || "untitled";
|
|
120
|
-
}
|
|
34
|
+
import { documentSlug } from "@bendyline/gezk";
|
|
35
|
+
import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
|
|
121
36
|
|
|
122
37
|
// src/format/quantize.ts
|
|
123
|
-
|
|
124
|
-
const out = new Int8Array(unitVector.length);
|
|
125
|
-
for (let i = 0; i < unitVector.length; i++) {
|
|
126
|
-
const q = Math.round(unitVector[i] * 127);
|
|
127
|
-
out[i] = q > 127 ? 127 : q < -127 ? -127 : q;
|
|
128
|
-
}
|
|
129
|
-
return out;
|
|
130
|
-
}
|
|
131
|
-
function quantizeBinary(unitVector) {
|
|
132
|
-
const bytes = Math.ceil(unitVector.length / 8);
|
|
133
|
-
const out = new Uint8Array(bytes);
|
|
134
|
-
for (let i = 0; i < unitVector.length; i++) {
|
|
135
|
-
if (unitVector[i] > 0) {
|
|
136
|
-
out[i >> 3] = out[i >> 3] | 1 << (i & 7);
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
return out;
|
|
140
|
-
}
|
|
141
|
-
function rerankScore(query, passageInt8) {
|
|
142
|
-
let dot = 0;
|
|
143
|
-
const n = Math.min(query.length, passageInt8.length);
|
|
144
|
-
for (let i = 0; i < n; i++) dot += query[i] * (passageInt8[i] / 127);
|
|
145
|
-
return dot;
|
|
146
|
-
}
|
|
147
|
-
function l2Normalize(vector) {
|
|
148
|
-
let sum = 0;
|
|
149
|
-
for (let i = 0; i < vector.length; i++) {
|
|
150
|
-
const v = vector[i];
|
|
151
|
-
sum += v * v;
|
|
152
|
-
}
|
|
153
|
-
const norm = Math.sqrt(sum);
|
|
154
|
-
if (!Number.isFinite(norm) || norm === 0) throw new Error("degenerate vector (zero norm)");
|
|
155
|
-
const out = new Float32Array(vector.length);
|
|
156
|
-
for (let i = 0; i < vector.length; i++) out[i] = vector[i] / norm;
|
|
157
|
-
return out;
|
|
158
|
-
}
|
|
38
|
+
import { l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from "@bendyline/gezk";
|
|
159
39
|
|
|
160
40
|
// src/chunking/markdown-chunker.ts
|
|
161
41
|
var HEADING = /^(#{1,6})\s+(.*)$/;
|
|
@@ -284,12 +164,12 @@ function splitOversizeLine(line, opts) {
|
|
|
284
164
|
}
|
|
285
165
|
|
|
286
166
|
// src/compiler/compile.ts
|
|
287
|
-
import { createHash
|
|
288
|
-
import { mkdirSync, rmSync } from "fs";
|
|
167
|
+
import { createHash } from "crypto";
|
|
168
|
+
import { mkdirSync, rmSync, statSync } from "fs";
|
|
289
169
|
import { stat } from "fs/promises";
|
|
290
170
|
import { join } from "path";
|
|
291
171
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
292
|
-
import { KnowledgeCatalogManifestSchema } from "@bendyline/
|
|
172
|
+
import { GEZK_MANIFEST_KIND, KnowledgeCatalogManifestSchema } from "@bendyline/gezk";
|
|
293
173
|
|
|
294
174
|
// src/archive/write.ts
|
|
295
175
|
import { createWriteStream } from "fs";
|
|
@@ -301,9 +181,13 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
301
181
|
const yazl = nodeRequire("yazl");
|
|
302
182
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
303
183
|
const zip = new yazl.ZipFile();
|
|
304
|
-
const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress:
|
|
184
|
+
const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress: false };
|
|
185
|
+
zip.addBuffer(Buffer.from(GEZK_MIME_TYPE, "utf8"), MIMETYPE_PATH, options);
|
|
305
186
|
const sorted = [...entries].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
|
306
187
|
for (const entry of sorted) {
|
|
188
|
+
if (entry.path === MIMETYPE_PATH) {
|
|
189
|
+
throw new Error(`the writer owns the ${MIMETYPE_PATH} entry`);
|
|
190
|
+
}
|
|
307
191
|
if (entry.path.includes("\\") || entry.path.startsWith("/") || entry.path.includes("..")) {
|
|
308
192
|
throw new Error(`unsafe archive path: ${entry.path}`);
|
|
309
193
|
}
|
|
@@ -311,10 +195,10 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
311
195
|
else if (entry.absPath) zip.addFile(entry.absPath, entry.path, options);
|
|
312
196
|
else throw new Error(`archive entry ${entry.path} has neither content nor absPath`);
|
|
313
197
|
}
|
|
314
|
-
await new Promise((
|
|
198
|
+
await new Promise((resolve4, reject) => {
|
|
315
199
|
const out = createWriteStream(outputPath);
|
|
316
200
|
out.on("error", reject);
|
|
317
|
-
out.on("close", () =>
|
|
201
|
+
out.on("close", () => resolve4());
|
|
318
202
|
zip.outputStream.on("error", reject);
|
|
319
203
|
zip.outputStream.pipe(out);
|
|
320
204
|
zip.end();
|
|
@@ -322,27 +206,7 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
322
206
|
}
|
|
323
207
|
|
|
324
208
|
// src/format/file-hash.ts
|
|
325
|
-
import {
|
|
326
|
-
import { createReadStream } from "fs";
|
|
327
|
-
async function hashFileStreaming(path, maxBytes = Number.MAX_SAFE_INTEGER) {
|
|
328
|
-
const hasher = createHash2("sha256");
|
|
329
|
-
let sizeBytes = 0;
|
|
330
|
-
await new Promise((resolve3, reject) => {
|
|
331
|
-
const stream = createReadStream(path, { highWaterMark: 4 * 1024 * 1024 });
|
|
332
|
-
stream.on("data", (chunk) => {
|
|
333
|
-
const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
|
|
334
|
-
sizeBytes += bytes.byteLength;
|
|
335
|
-
if (sizeBytes > maxBytes) {
|
|
336
|
-
stream.destroy(new Error(`file exceeds ${maxBytes} byte limit`));
|
|
337
|
-
return;
|
|
338
|
-
}
|
|
339
|
-
hasher.update(bytes);
|
|
340
|
-
});
|
|
341
|
-
stream.on("error", reject);
|
|
342
|
-
stream.on("end", resolve3);
|
|
343
|
-
});
|
|
344
|
-
return { sha256: hasher.digest("hex"), sizeBytes };
|
|
345
|
-
}
|
|
209
|
+
import { hashFileStreaming } from "@bendyline/gezk/node";
|
|
346
210
|
|
|
347
211
|
// src/format/node-sqlite.ts
|
|
348
212
|
import { createRequire as createRequire2 } from "module";
|
|
@@ -350,25 +214,34 @@ var nodeRequire2 = createRequire2(import.meta.url);
|
|
|
350
214
|
var sqlite = nodeRequire2("node:sqlite");
|
|
351
215
|
var DatabaseSync = sqlite.DatabaseSync;
|
|
352
216
|
|
|
353
|
-
// src/
|
|
217
|
+
// src/reader/fts-query.ts
|
|
218
|
+
function sanitizeFtsQuery(query) {
|
|
219
|
+
const tokens = [
|
|
220
|
+
...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
|
|
221
|
+
];
|
|
222
|
+
if (tokens.length === 0) return null;
|
|
223
|
+
return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
|
|
224
|
+
}
|
|
225
|
+
function documentFtsTopIds(db, match, limit) {
|
|
226
|
+
return db.prepare(
|
|
227
|
+
"SELECT document_id FROM fts_documents WHERE fts_documents MATCH ? ORDER BY rank LIMIT ?"
|
|
228
|
+
).all(match, limit).map((row) => row.document_id);
|
|
229
|
+
}
|
|
230
|
+
var SMOKE_QUERY_TOP_N = 10;
|
|
231
|
+
function documentSmokeQueryMisses(db, smoke) {
|
|
232
|
+
const match = sanitizeFtsQuery(smoke.query);
|
|
233
|
+
if (!match) return [...smoke.expectedDocumentIds];
|
|
234
|
+
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N);
|
|
235
|
+
return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// src/toolchain.ts
|
|
354
239
|
import { createRequire as createRequire3 } from "module";
|
|
355
240
|
var nodeRequire3 = createRequire3(import.meta.url);
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
db.loadExtension(sqliteVec.getLoadablePath());
|
|
361
|
-
} catch (err) {
|
|
362
|
-
throw new Error(
|
|
363
|
-
`sqlite-vec extension failed to load \u2014 the compiler cannot build vector tables: ${err instanceof Error ? err.message : String(err)}`
|
|
364
|
-
);
|
|
365
|
-
} finally {
|
|
366
|
-
try {
|
|
367
|
-
db.enableLoadExtension(false);
|
|
368
|
-
} catch {
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
}
|
|
241
|
+
var KNOWLEDGE_TOOLCHAIN = {
|
|
242
|
+
name: "@bendyline/gezel-knowledge",
|
|
243
|
+
version: nodeRequire3("../package.json").version
|
|
244
|
+
};
|
|
372
245
|
|
|
373
246
|
// src/compiler/compile.ts
|
|
374
247
|
async function compileKnowledgeCatalog(opts) {
|
|
@@ -380,9 +253,9 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
380
253
|
const topicIds = new Set(opts.topics.map((t) => t.id));
|
|
381
254
|
const profile = opts.embeddingProfile;
|
|
382
255
|
const chunkerOpts = {
|
|
383
|
-
unit:
|
|
384
|
-
target: opts.chunkingProfile.
|
|
385
|
-
overlap: opts.chunkingProfile.
|
|
256
|
+
unit: opts.chunkingProfile.unit,
|
|
257
|
+
target: opts.chunkingProfile.target,
|
|
258
|
+
overlap: opts.chunkingProfile.overlap,
|
|
386
259
|
countTokens: opts.countTokens
|
|
387
260
|
};
|
|
388
261
|
const prepared = [];
|
|
@@ -404,7 +277,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
404
277
|
(a, b) => a.topicPathKey === b.topicPathKey ? a.doc.id < b.doc.id ? -1 : a.doc.id > b.doc.id ? 1 : 0 : a.topicPathKey < b.topicPathKey ? -1 : 1
|
|
405
278
|
);
|
|
406
279
|
const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
|
|
407
|
-
const
|
|
280
|
+
const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
|
|
281
|
+
const embedded = totalChunks <= shardTarget;
|
|
408
282
|
const shardOf = /* @__PURE__ */ new Map();
|
|
409
283
|
let shardCount = 1;
|
|
410
284
|
if (embedded) {
|
|
@@ -413,7 +287,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
413
287
|
let current = 0;
|
|
414
288
|
let filled = 0;
|
|
415
289
|
for (const p of prepared) {
|
|
416
|
-
if (filled > 0 && filled + p.chunks.length >
|
|
290
|
+
if (filled > 0 && filled + p.chunks.length > shardTarget) {
|
|
417
291
|
current++;
|
|
418
292
|
filled = 0;
|
|
419
293
|
}
|
|
@@ -441,7 +315,6 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
441
315
|
router.exec("BEGIN");
|
|
442
316
|
if (embedded) {
|
|
443
317
|
router.exec(SHARD_DDL.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
|
|
444
|
-
router.exec(vecChunksDdl(profile.dimensions));
|
|
445
318
|
}
|
|
446
319
|
const commonMeta = {
|
|
447
320
|
format_version: String(GEZK_FORMAT_VERSION),
|
|
@@ -538,7 +411,6 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
538
411
|
if (!db) {
|
|
539
412
|
db = createCatalogDb(join(opts.workDir, shardPath(id)));
|
|
540
413
|
db.exec(SHARD_DDL);
|
|
541
|
-
db.exec(vecChunksDdl(profile.dimensions));
|
|
542
414
|
db.exec("BEGIN");
|
|
543
415
|
shardDbs.set(id, db);
|
|
544
416
|
}
|
|
@@ -549,7 +421,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
549
421
|
let stmts = vecStmts.get(db);
|
|
550
422
|
if (!stmts) {
|
|
551
423
|
stmts = {
|
|
552
|
-
vec: db.prepare("INSERT INTO
|
|
424
|
+
vec: db.prepare("INSERT INTO chunk_vectors_bit (chunk_id, v) VALUES (?, ?)"),
|
|
553
425
|
int8: db.prepare("INSERT INTO chunk_vectors_int8 (chunk_id, v) VALUES (?, ?)")
|
|
554
426
|
};
|
|
555
427
|
vecStmts.set(db, stmts);
|
|
@@ -628,7 +500,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
628
500
|
const header = buildContextHeader(
|
|
629
501
|
p.doc.title,
|
|
630
502
|
chunk.headingPath,
|
|
631
|
-
opts.chunkingProfile.contextHeader.
|
|
503
|
+
opts.chunkingProfile.contextHeader.max,
|
|
632
504
|
opts.countTokens
|
|
633
505
|
);
|
|
634
506
|
pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
|
|
@@ -637,6 +509,30 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
637
509
|
}
|
|
638
510
|
}
|
|
639
511
|
await flushEmbeds();
|
|
512
|
+
for (let id = 0; id < shardCount; id++) {
|
|
513
|
+
if (!embedded)
|
|
514
|
+
putMeta(shardDb(id), {
|
|
515
|
+
shard_id: String(id),
|
|
516
|
+
chunk_count: String(shardChunkCounts.get(id) ?? 0)
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
|
|
520
|
+
const finalDir = join(opts.workDir, "final");
|
|
521
|
+
mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
|
|
522
|
+
const vacuumed = [];
|
|
523
|
+
const vacuumInto = (db, archivePath) => {
|
|
524
|
+
db.exec("COMMIT");
|
|
525
|
+
const dest = join(finalDir, archivePath);
|
|
526
|
+
db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
|
|
527
|
+
vacuumed.push({ archivePath, absPath: dest });
|
|
528
|
+
return dest;
|
|
529
|
+
};
|
|
530
|
+
const shardBytes = /* @__PURE__ */ new Map();
|
|
531
|
+
if (!embedded) {
|
|
532
|
+
for (let id = 0; id < shardCount; id++) {
|
|
533
|
+
shardBytes.set(id, statSync(vacuumInto(shardDb(id), shardPath(id))).size);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
640
536
|
{
|
|
641
537
|
const insertShard = router.prepare(
|
|
642
538
|
"INSERT INTO shards (id, path, chunk_count, document_count, topic_ids_json, centroid_count, bytes) VALUES (?, ?, ?, ?, ?, ?, ?)"
|
|
@@ -659,37 +555,45 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
659
555
|
BigInt(shardDocCounts.get(id) ?? 0),
|
|
660
556
|
JSON.stringify([...shardTopicIds.get(id) ?? []].sort()),
|
|
661
557
|
BigInt(centroids.length),
|
|
662
|
-
BigInt(0)
|
|
663
|
-
// authoritative sizes live in the manifest (post-VACUUM)
|
|
558
|
+
BigInt(shardBytes.get(id) ?? 0)
|
|
664
559
|
);
|
|
665
560
|
for (const c of centroids) {
|
|
666
561
|
insertCentroid.run(BigInt(id), Buffer.from(c.centroid.buffer.slice(0)), BigInt(c.weight));
|
|
667
562
|
}
|
|
668
563
|
}
|
|
669
564
|
}
|
|
670
|
-
for (let id = 0; id < shardCount; id++) {
|
|
671
|
-
if (!embedded)
|
|
672
|
-
putMeta(shardDb(id), {
|
|
673
|
-
shard_id: String(id),
|
|
674
|
-
chunk_count: String(shardChunkCounts.get(id) ?? 0)
|
|
675
|
-
});
|
|
676
|
-
}
|
|
677
|
-
if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
|
|
678
|
-
const finalDir = join(opts.workDir, "final");
|
|
679
|
-
mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
|
|
680
|
-
const vacuumed = [];
|
|
681
|
-
const vacuumInto = (db, archivePath) => {
|
|
682
|
-
db.exec("COMMIT");
|
|
683
|
-
const dest = join(finalDir, archivePath);
|
|
684
|
-
db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
|
|
685
|
-
vacuumed.push({ archivePath, absPath: dest });
|
|
686
|
-
};
|
|
687
565
|
vacuumInto(router, ROUTER_DB_PATH);
|
|
688
|
-
if (!embedded) {
|
|
689
|
-
for (let id = 0; id < shardCount; id++) vacuumInto(shardDb(id), shardPath(id));
|
|
690
|
-
}
|
|
691
566
|
router.close();
|
|
692
567
|
for (const db of shardDbs.values()) db.close();
|
|
568
|
+
let smokeQueries = opts.smokeQueries;
|
|
569
|
+
if (smokeQueries && smokeQueries.length > 0) {
|
|
570
|
+
const routerFinal = new DatabaseSync(join(finalDir, ROUTER_DB_PATH), { readOnly: true });
|
|
571
|
+
try {
|
|
572
|
+
const verdicts = smokeQueries.map((smoke) => ({
|
|
573
|
+
smoke,
|
|
574
|
+
missing: documentSmokeQueryMisses(routerFinal, smoke)
|
|
575
|
+
}));
|
|
576
|
+
if ((opts.smokeQueryPolicy ?? "require") === "select") {
|
|
577
|
+
smokeQueries = verdicts.filter((v) => v.missing.length === 0).slice(0, 3).map((v) => v.smoke);
|
|
578
|
+
if (smokeQueries.length === 0) {
|
|
579
|
+
throw new Error(
|
|
580
|
+
`no smoke-query candidate passes against the built index (tried ${verdicts.length}: ${verdicts.map((v) => `"${v.smoke.query}"`).join(", ")}) \u2014 the fts_documents index cannot surface any candidate's expected document in the top ${SMOKE_QUERY_TOP_N}`
|
|
581
|
+
);
|
|
582
|
+
}
|
|
583
|
+
} else {
|
|
584
|
+
const failed = verdicts.filter((v) => v.missing.length > 0);
|
|
585
|
+
if (failed.length > 0) {
|
|
586
|
+
throw new Error(
|
|
587
|
+
`smoke queries failed against the built index: ${failed.map((v) => `"${v.smoke.query}" (missing ${v.missing.join(", ")})`).join(
|
|
588
|
+
"; "
|
|
589
|
+
)} \u2014 every recorded sanity query must return its expected documents in the top ${SMOKE_QUERY_TOP_N} fts_documents results; pick more specific queries (distinctive titles beat common words in the rank race) or pass smokeQueryPolicy: 'select' with candidates`
|
|
590
|
+
);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
} finally {
|
|
594
|
+
routerFinal.close();
|
|
595
|
+
}
|
|
596
|
+
}
|
|
693
597
|
const files = [];
|
|
694
598
|
const routerShardStats = [];
|
|
695
599
|
for (const f of vacuumed) {
|
|
@@ -714,17 +618,21 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
714
618
|
});
|
|
715
619
|
}
|
|
716
620
|
}
|
|
717
|
-
const extraFiles =
|
|
621
|
+
const extraFiles = {
|
|
622
|
+
[README_PATH]: defaultReadme(opts),
|
|
623
|
+
[LICENSE_NOTICE_PATH]: defaultLicenseNotice(opts),
|
|
624
|
+
...opts.extraFiles ?? {}
|
|
625
|
+
};
|
|
718
626
|
for (const [path, content] of Object.entries(extraFiles)) {
|
|
719
627
|
const bytes = Buffer.from(content, "utf8");
|
|
720
628
|
files.push({
|
|
721
629
|
path,
|
|
722
630
|
sizeBytes: bytes.length,
|
|
723
|
-
sha256:
|
|
631
|
+
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
724
632
|
});
|
|
725
633
|
}
|
|
726
634
|
const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
|
|
727
|
-
kind:
|
|
635
|
+
kind: GEZK_MANIFEST_KIND,
|
|
728
636
|
formatVersion: GEZK_FORMAT_VERSION,
|
|
729
637
|
indexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION,
|
|
730
638
|
id: opts.catalog.id,
|
|
@@ -735,20 +643,23 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
735
643
|
publisher: opts.catalog.publisher,
|
|
736
644
|
createdAt: opts.catalog.createdAt,
|
|
737
645
|
...opts.catalog.sourceSnapshot ? { sourceSnapshot: opts.catalog.sourceSnapshot } : {},
|
|
738
|
-
license:
|
|
646
|
+
license: {
|
|
647
|
+
...opts.catalog.license,
|
|
648
|
+
noticePath: opts.catalog.license.noticePath ?? LICENSE_NOTICE_PATH
|
|
649
|
+
},
|
|
739
650
|
embedding: profile,
|
|
740
651
|
chunking: opts.chunkingProfile,
|
|
741
652
|
topics: opts.topics.map((t) => ({ id: t.id, name: t.name })),
|
|
742
653
|
router: {
|
|
743
|
-
shardTargetChunks:
|
|
654
|
+
shardTargetChunks: shardTarget,
|
|
744
655
|
shards: routerShardStats.sort((a, b) => a.id - b.id),
|
|
745
656
|
totalCentroids: routerShardStats.reduce((sum, s) => sum + s.centroids, 0)
|
|
746
657
|
},
|
|
747
658
|
counts: { documents: prepared.length, chunks: totalChunks, shards: shardCount },
|
|
748
659
|
files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
749
|
-
|
|
750
|
-
...
|
|
751
|
-
|
|
660
|
+
requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
|
|
661
|
+
...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
|
|
662
|
+
toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
|
|
752
663
|
});
|
|
753
664
|
const manifest = opts.finalizeManifest ? KnowledgeCatalogManifestSchema.parse(opts.finalizeManifest(unsignedManifest)) : unsignedManifest;
|
|
754
665
|
const archiveEntries = [
|
|
@@ -778,14 +689,37 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
778
689
|
}
|
|
779
690
|
}
|
|
780
691
|
function createCatalogDb(absPath) {
|
|
781
|
-
const db = new DatabaseSync(absPath
|
|
782
|
-
loadVecExtension(db);
|
|
692
|
+
const db = new DatabaseSync(absPath);
|
|
783
693
|
db.exec("PRAGMA page_size=8192");
|
|
784
694
|
db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
|
|
785
695
|
db.exec(`PRAGMA user_version=${GEZK_INDEX_SCHEMA_VERSION}`);
|
|
786
696
|
db.exec("PRAGMA journal_mode=DELETE");
|
|
787
697
|
return db;
|
|
788
698
|
}
|
|
699
|
+
function defaultReadme(opts) {
|
|
700
|
+
const c = opts.catalog;
|
|
701
|
+
return [
|
|
702
|
+
`# ${c.name}`,
|
|
703
|
+
"",
|
|
704
|
+
...c.description ? [c.description, ""] : [],
|
|
705
|
+
`A gezk knowledge catalog (${c.id}@${c.version}) published by ${c.publisher.name}.`,
|
|
706
|
+
`Language: ${c.language}. License: ${c.license.name} (see ${c.license.noticePath ?? LICENSE_NOTICE_PATH}).`,
|
|
707
|
+
...c.sourceSnapshot ? [`Source snapshot: ${c.sourceSnapshot.name} (${c.sourceSnapshot.date}).`] : [],
|
|
708
|
+
""
|
|
709
|
+
].join("\n");
|
|
710
|
+
}
|
|
711
|
+
function defaultLicenseNotice(opts) {
|
|
712
|
+
const c = opts.catalog;
|
|
713
|
+
const publisher = c.publisher.url ? `${c.publisher.name} <${c.publisher.url}>` : c.publisher.name;
|
|
714
|
+
const license = c.license.spdx ? `${c.license.name} (SPDX: ${c.license.spdx})` : c.license.name;
|
|
715
|
+
return [
|
|
716
|
+
`${c.name} (${c.id}@${c.version})`,
|
|
717
|
+
`Publisher: ${publisher}`,
|
|
718
|
+
`License: ${license}`,
|
|
719
|
+
...c.license.attributionRequired ? ["Attribution is required when redistributing this content."] : [],
|
|
720
|
+
""
|
|
721
|
+
].join("\n");
|
|
722
|
+
}
|
|
789
723
|
function normalizeMarkdown(md) {
|
|
790
724
|
return md.replace(/\r\n/g, "\n").normalize("NFC");
|
|
791
725
|
}
|
|
@@ -803,7 +737,7 @@ ${path.join(" > ")}
|
|
|
803
737
|
return header;
|
|
804
738
|
}
|
|
805
739
|
function seedFor(catalogId, shardId) {
|
|
806
|
-
const digest =
|
|
740
|
+
const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
|
|
807
741
|
return digest.readBigUInt64LE(24);
|
|
808
742
|
}
|
|
809
743
|
function makePrng(seed) {
|
|
@@ -888,14 +822,14 @@ function kMeans(samples, k, seed) {
|
|
|
888
822
|
}
|
|
889
823
|
|
|
890
824
|
// src/archive/read.ts
|
|
891
|
-
import { createHash as
|
|
825
|
+
import { createHash as createHash2 } from "crypto";
|
|
892
826
|
import { createWriteStream as createWriteStream2 } from "fs";
|
|
893
827
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
894
828
|
import { createRequire as createRequire4 } from "module";
|
|
895
829
|
import { dirname as dirname2, isAbsolute, normalize, relative, resolve } from "path";
|
|
896
830
|
import { Transform } from "stream";
|
|
897
831
|
import { pipeline } from "stream/promises";
|
|
898
|
-
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/
|
|
832
|
+
import { GEZK_MANIFEST_KIND as GEZK_MANIFEST_KIND2, KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/gezk";
|
|
899
833
|
var nodeRequire4 = createRequire4(import.meta.url);
|
|
900
834
|
var GEZK_ARCHIVE_LIMITS = {
|
|
901
835
|
maxEntries: 16384,
|
|
@@ -985,7 +919,7 @@ async function inspectEntries(archivePath) {
|
|
|
985
919
|
function reconcileEntries(manifest, entries) {
|
|
986
920
|
const declared = new Map(manifest.files.map((file) => [file.path, file]));
|
|
987
921
|
for (const path of entries.keys()) {
|
|
988
|
-
if (path === MANIFEST_PATH) continue;
|
|
922
|
+
if (path === MANIFEST_PATH || path === MIMETYPE_PATH) continue;
|
|
989
923
|
if (!declared.has(path)) {
|
|
990
924
|
throw new GezkArchiveError(`archive contains undeclared file: ${path}`, "undeclared-file");
|
|
991
925
|
}
|
|
@@ -1015,14 +949,88 @@ async function inspectGezkArchive(archivePath) {
|
|
|
1015
949
|
totalUncompressedBytes: inspected.totalUncompressedBytes
|
|
1016
950
|
};
|
|
1017
951
|
}
|
|
952
|
+
function assertMimetypeEntry(entry, position) {
|
|
953
|
+
if (position !== 0) {
|
|
954
|
+
throw new GezkArchiveError("the mimetype entry must be the first archive entry", "mimetype");
|
|
955
|
+
}
|
|
956
|
+
if (entry.compressionMethod !== 0 || entry.uncompressedSize !== GEZK_MIME_TYPE.length) {
|
|
957
|
+
throw new GezkArchiveError("the mimetype entry must be stored, not compressed", "mimetype");
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
function assertSupportedFormat(raw) {
|
|
961
|
+
const kind = raw?.kind;
|
|
962
|
+
const version = raw?.formatVersion;
|
|
963
|
+
if (kind !== GEZK_MANIFEST_KIND2 || version !== GEZK_FORMAT_VERSION) {
|
|
964
|
+
throw new GezkArchiveError(
|
|
965
|
+
`unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${GEZK_FORMAT_VERSION} \u2014 catalogs built for an earlier version must be rebuilt`,
|
|
966
|
+
"format-version"
|
|
967
|
+
);
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
function readEntryText(zip, entry) {
|
|
971
|
+
return new Promise((resolveText, rejectText) => {
|
|
972
|
+
zip.openReadStream(entry, (err, stream) => {
|
|
973
|
+
if (err || !stream) {
|
|
974
|
+
rejectText(new GezkArchiveError(`${entry.fileName} read failed: ${err?.message}`, "io"));
|
|
975
|
+
return;
|
|
976
|
+
}
|
|
977
|
+
const parts = [];
|
|
978
|
+
stream.on("data", (d) => parts.push(d));
|
|
979
|
+
stream.on("end", () => resolveText(Buffer.concat(parts).toString("utf8")));
|
|
980
|
+
stream.on(
|
|
981
|
+
"error",
|
|
982
|
+
(streamErr) => rejectText(
|
|
983
|
+
new GezkArchiveError(`${entry.fileName} read failed: ${streamErr.message}`, "io")
|
|
984
|
+
)
|
|
985
|
+
);
|
|
986
|
+
});
|
|
987
|
+
});
|
|
988
|
+
}
|
|
1018
989
|
async function readGezkManifest(archivePath) {
|
|
1019
990
|
const zip = await openZip(archivePath);
|
|
1020
991
|
return new Promise((resolvePromise, reject) => {
|
|
992
|
+
let position = 0;
|
|
993
|
+
let mimetypeSeen = false;
|
|
994
|
+
const fail = (err) => {
|
|
995
|
+
zip.close();
|
|
996
|
+
reject(
|
|
997
|
+
err instanceof GezkArchiveError ? err : new GezkArchiveError(err instanceof Error ? err.message : String(err), "io")
|
|
998
|
+
);
|
|
999
|
+
};
|
|
1021
1000
|
zip.on("entry", (entry) => {
|
|
1001
|
+
const index = position++;
|
|
1002
|
+
if (entry.fileName === MIMETYPE_PATH) {
|
|
1003
|
+
try {
|
|
1004
|
+
assertMimetypeEntry(entry, index);
|
|
1005
|
+
} catch (err) {
|
|
1006
|
+
fail(err);
|
|
1007
|
+
return;
|
|
1008
|
+
}
|
|
1009
|
+
readEntryText(zip, entry).then((text) => {
|
|
1010
|
+
if (text !== GEZK_MIME_TYPE) {
|
|
1011
|
+
fail(
|
|
1012
|
+
new GezkArchiveError(
|
|
1013
|
+
`not a gezk archive (mimetype ${JSON.stringify(text)})`,
|
|
1014
|
+
"mimetype"
|
|
1015
|
+
)
|
|
1016
|
+
);
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
1019
|
+
mimetypeSeen = true;
|
|
1020
|
+
zip.readEntry();
|
|
1021
|
+
}, fail);
|
|
1022
|
+
return;
|
|
1023
|
+
}
|
|
1022
1024
|
if (entry.fileName !== MANIFEST_PATH) {
|
|
1023
1025
|
zip.readEntry();
|
|
1024
1026
|
return;
|
|
1025
1027
|
}
|
|
1028
|
+
if (!mimetypeSeen) {
|
|
1029
|
+
fail(
|
|
1030
|
+
new GezkArchiveError("archive does not start with the gezk mimetype entry", "mimetype")
|
|
1031
|
+
);
|
|
1032
|
+
return;
|
|
1033
|
+
}
|
|
1026
1034
|
if (entry.uncompressedSize > GEZK_ARCHIVE_LIMITS.maxManifestBytes) {
|
|
1027
1035
|
zip.close();
|
|
1028
1036
|
reject(new GezkArchiveError("manifest exceeds size limit", "limits"));
|
|
@@ -1039,12 +1047,14 @@ async function readGezkManifest(archivePath) {
|
|
|
1039
1047
|
stream.on("end", () => {
|
|
1040
1048
|
zip.close();
|
|
1041
1049
|
try {
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
)
|
|
1046
|
-
);
|
|
1050
|
+
const raw = JSON.parse(Buffer.concat(parts).toString("utf8"));
|
|
1051
|
+
assertSupportedFormat(raw);
|
|
1052
|
+
resolvePromise(KnowledgeCatalogManifestSchema2.parse(raw));
|
|
1047
1053
|
} catch (parseErr) {
|
|
1054
|
+
if (parseErr instanceof GezkArchiveError) {
|
|
1055
|
+
reject(parseErr);
|
|
1056
|
+
return;
|
|
1057
|
+
}
|
|
1048
1058
|
reject(
|
|
1049
1059
|
new GezkArchiveError(
|
|
1050
1060
|
`invalid manifest: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,
|
|
@@ -1102,7 +1112,7 @@ async function extractGezkVerified(archivePath, destDir) {
|
|
|
1102
1112
|
pipeline(stream, createWriteStream2(target)).then(res, rej);
|
|
1103
1113
|
return;
|
|
1104
1114
|
}
|
|
1105
|
-
const hasher =
|
|
1115
|
+
const hasher = createHash2("sha256");
|
|
1106
1116
|
let sizeBytes = 0;
|
|
1107
1117
|
const verify = new Transform({
|
|
1108
1118
|
transform(chunk, _encoding, callback) {
|
|
@@ -1163,9 +1173,7 @@ async function extractGezkVerified(archivePath, destDir) {
|
|
|
1163
1173
|
}
|
|
1164
1174
|
|
|
1165
1175
|
// src/reader/open.ts
|
|
1166
|
-
import { createRequire as createRequire5 } from "module";
|
|
1167
1176
|
import { pathToFileURL } from "url";
|
|
1168
|
-
var nodeRequire5 = createRequire5(import.meta.url);
|
|
1169
1177
|
var CatalogOpenError = class extends Error {
|
|
1170
1178
|
constructor(message, reason) {
|
|
1171
1179
|
super(message);
|
|
@@ -1178,7 +1186,7 @@ function openCatalogDatabase(absPath) {
|
|
|
1178
1186
|
const uri = `${pathToFileURL(absPath).href}?immutable=1`;
|
|
1179
1187
|
let db;
|
|
1180
1188
|
try {
|
|
1181
|
-
db = new DatabaseSync(uri, { readOnly: true
|
|
1189
|
+
db = new DatabaseSync(uri, { readOnly: true });
|
|
1182
1190
|
} catch (err) {
|
|
1183
1191
|
throw new CatalogOpenError(
|
|
1184
1192
|
`cannot open catalog database ${absPath}: ${err instanceof Error ? err.message : String(err)}`,
|
|
@@ -1186,20 +1194,6 @@ function openCatalogDatabase(absPath) {
|
|
|
1186
1194
|
);
|
|
1187
1195
|
}
|
|
1188
1196
|
try {
|
|
1189
|
-
let vecLoaded = false;
|
|
1190
|
-
try {
|
|
1191
|
-
db.enableLoadExtension(true);
|
|
1192
|
-
const sqliteVec = nodeRequire5("sqlite-vec");
|
|
1193
|
-
db.loadExtension(sqliteVec.getLoadablePath());
|
|
1194
|
-
vecLoaded = true;
|
|
1195
|
-
} catch {
|
|
1196
|
-
vecLoaded = false;
|
|
1197
|
-
} finally {
|
|
1198
|
-
try {
|
|
1199
|
-
db.enableLoadExtension(false);
|
|
1200
|
-
} catch {
|
|
1201
|
-
}
|
|
1202
|
-
}
|
|
1203
1197
|
const appId = readPragmaNumber(db, "application_id");
|
|
1204
1198
|
if (appId !== GEZK_APPLICATION_ID) {
|
|
1205
1199
|
throw new CatalogOpenError(
|
|
@@ -1210,13 +1204,10 @@ function openCatalogDatabase(absPath) {
|
|
|
1210
1204
|
const schemaVersion = readPragmaNumber(db, "user_version");
|
|
1211
1205
|
if (schemaVersion !== GEZK_INDEX_SCHEMA_VERSION) {
|
|
1212
1206
|
throw new CatalogOpenError(
|
|
1213
|
-
`unsupported index schema version ${schemaVersion} (reader supports ${GEZK_INDEX_SCHEMA_VERSION})`,
|
|
1207
|
+
`unsupported index schema version ${schemaVersion} (this reader supports ${GEZK_INDEX_SCHEMA_VERSION}; catalogs built for gezk 0.4 and earlier must be rebuilt)`,
|
|
1214
1208
|
"schema-version"
|
|
1215
1209
|
);
|
|
1216
1210
|
}
|
|
1217
|
-
if (!vecLoaded) {
|
|
1218
|
-
throw new CatalogOpenError("sqlite-vec extension failed to load", "vec-unavailable");
|
|
1219
|
-
}
|
|
1220
1211
|
db.exec("PRAGMA mmap_size=536870912");
|
|
1221
1212
|
db.exec("PRAGMA cache_size=-8192");
|
|
1222
1213
|
return {
|
|
@@ -1245,6 +1236,80 @@ function readPragmaNumber(db, pragma) {
|
|
|
1245
1236
|
// src/reader/catalog-handle.ts
|
|
1246
1237
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1247
1238
|
import { brotliDecompressSync } from "zlib";
|
|
1239
|
+
|
|
1240
|
+
// src/reader/bit-scan.ts
|
|
1241
|
+
var POPCOUNT8 = new Uint8Array(256);
|
|
1242
|
+
for (let i = 1; i < 256; i++) POPCOUNT8[i] = (i & 1) + POPCOUNT8[i >> 1];
|
|
1243
|
+
function popcount32(x) {
|
|
1244
|
+
let v = x - (x >>> 1 & 1431655765);
|
|
1245
|
+
v = (v & 858993459) + (v >>> 2 & 858993459);
|
|
1246
|
+
return Math.imul(v + (v >>> 4) & 252645135, 16843009) >>> 24;
|
|
1247
|
+
}
|
|
1248
|
+
function hammingAll(index, query, out) {
|
|
1249
|
+
const { bits, bytesPerRow, rows } = index;
|
|
1250
|
+
if (bytesPerRow % 4 === 0 && bits.byteOffset % 4 === 0) {
|
|
1251
|
+
const perRow = bytesPerRow / 4;
|
|
1252
|
+
const words = new Uint32Array(bits.buffer, bits.byteOffset, rows * perRow);
|
|
1253
|
+
const aligned = new Uint8Array(bytesPerRow);
|
|
1254
|
+
aligned.set(query);
|
|
1255
|
+
const qwords = new Uint32Array(aligned.buffer, 0, perRow);
|
|
1256
|
+
let offset2 = 0;
|
|
1257
|
+
for (let r = 0; r < rows; r++) {
|
|
1258
|
+
let d = 0;
|
|
1259
|
+
for (let w = 0; w < perRow; w++) {
|
|
1260
|
+
d += popcount32((words[offset2 + w] ^ qwords[w]) >>> 0);
|
|
1261
|
+
}
|
|
1262
|
+
out[r] = d;
|
|
1263
|
+
offset2 += perRow;
|
|
1264
|
+
}
|
|
1265
|
+
return;
|
|
1266
|
+
}
|
|
1267
|
+
let offset = 0;
|
|
1268
|
+
for (let r = 0; r < rows; r++) {
|
|
1269
|
+
let d = 0;
|
|
1270
|
+
for (let b = 0; b < bytesPerRow; b++) {
|
|
1271
|
+
d += POPCOUNT8[bits[offset + b] ^ query[b]];
|
|
1272
|
+
}
|
|
1273
|
+
out[r] = d;
|
|
1274
|
+
offset += bytesPerRow;
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
function hammingTopK(index, query, k) {
|
|
1278
|
+
if (query.length !== index.bytesPerRow) {
|
|
1279
|
+
throw new Error(`query has ${query.length} bytes, shard rows have ${index.bytesPerRow}`);
|
|
1280
|
+
}
|
|
1281
|
+
const limit = Math.min(k, index.rows);
|
|
1282
|
+
if (limit <= 0) return [];
|
|
1283
|
+
const distances = new Uint16Array(index.rows);
|
|
1284
|
+
hammingAll(index, query, distances);
|
|
1285
|
+
const maxDistance = index.bytesPerRow * 8;
|
|
1286
|
+
const histogram = new Uint32Array(maxDistance + 1);
|
|
1287
|
+
for (let r = 0; r < index.rows; r++) {
|
|
1288
|
+
const d = distances[r];
|
|
1289
|
+
histogram[d] = histogram[d] + 1;
|
|
1290
|
+
}
|
|
1291
|
+
let threshold = 0;
|
|
1292
|
+
let seen = 0;
|
|
1293
|
+
for (; threshold <= maxDistance; threshold++) {
|
|
1294
|
+
seen += histogram[threshold];
|
|
1295
|
+
if (seen >= limit) break;
|
|
1296
|
+
}
|
|
1297
|
+
let atThreshold = limit - (seen - histogram[threshold]);
|
|
1298
|
+
const hits = [];
|
|
1299
|
+
for (let r = 0; r < index.rows && hits.length < limit; r++) {
|
|
1300
|
+
const d = distances[r];
|
|
1301
|
+
if (d < threshold) hits.push({ chunkId: r + 1, distance: d });
|
|
1302
|
+
else if (d === threshold && atThreshold > 0) {
|
|
1303
|
+
hits.push({ chunkId: r + 1, distance: d });
|
|
1304
|
+
atThreshold--;
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
|
|
1308
|
+
return hits;
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
// src/reader/catalog-handle.ts
|
|
1312
|
+
var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
|
|
1248
1313
|
var CatalogHandle = class _CatalogHandle {
|
|
1249
1314
|
constructor(rootDir, router) {
|
|
1250
1315
|
this.rootDir = rootDir;
|
|
@@ -1259,6 +1324,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1259
1324
|
rootDir;
|
|
1260
1325
|
router;
|
|
1261
1326
|
connections = /* @__PURE__ */ new Map();
|
|
1327
|
+
bitIndexes = /* @__PURE__ */ new Map();
|
|
1262
1328
|
meta;
|
|
1263
1329
|
shards;
|
|
1264
1330
|
/** Open a catalog's extracted version directory (the dir holding manifest.json). */
|
|
@@ -1269,6 +1335,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1269
1335
|
close() {
|
|
1270
1336
|
for (const conn of this.connections.values()) conn.close();
|
|
1271
1337
|
this.connections.clear();
|
|
1338
|
+
this.bitIndexes.clear();
|
|
1272
1339
|
}
|
|
1273
1340
|
/** Integrity check for the router connection, used by deep validation. */
|
|
1274
1341
|
routerQuickCheck() {
|
|
@@ -1299,6 +1366,57 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1299
1366
|
}
|
|
1300
1367
|
return conn.db;
|
|
1301
1368
|
}
|
|
1369
|
+
/**
|
|
1370
|
+
* A shard's sign-bit rows as one contiguous array, loaded on first use and
|
|
1371
|
+
* kept while the resident budget allows. Chunk ids must be dense from 1 —
|
|
1372
|
+
* the row index IS the id — so a gap is a corrupt shard, not a sparse one.
|
|
1373
|
+
*/
|
|
1374
|
+
shardBits(shard) {
|
|
1375
|
+
const cached = this.bitIndexes.get(shard.path);
|
|
1376
|
+
if (cached) return cached;
|
|
1377
|
+
const db = this.shardDb(shard);
|
|
1378
|
+
const span = db.prepare(
|
|
1379
|
+
"SELECT COUNT(*) AS n, MIN(chunk_id) AS lo, MAX(chunk_id) AS hi FROM chunk_vectors_bit"
|
|
1380
|
+
).get();
|
|
1381
|
+
const rows = Number(span.n);
|
|
1382
|
+
if (rows > 0 && (Number(span.lo) !== 1 || Number(span.hi) !== rows)) {
|
|
1383
|
+
throw new CatalogOpenError(`chunk ids are not dense in ${shard.path}`, "corrupt");
|
|
1384
|
+
}
|
|
1385
|
+
const bytesPerRow = Math.ceil(this.dimensions() / 8);
|
|
1386
|
+
const bits = new Uint8Array(rows * bytesPerRow);
|
|
1387
|
+
const rowsIter = db.prepare("SELECT chunk_id, v FROM chunk_vectors_bit ORDER BY chunk_id").iterate();
|
|
1388
|
+
for (const row of rowsIter) {
|
|
1389
|
+
if (row.v.byteLength !== bytesPerRow) {
|
|
1390
|
+
throw new CatalogOpenError(
|
|
1391
|
+
`bit vector width ${row.v.byteLength} != ${bytesPerRow} in ${shard.path}`,
|
|
1392
|
+
"corrupt"
|
|
1393
|
+
);
|
|
1394
|
+
}
|
|
1395
|
+
bits.set(row.v, (Number(row.chunk_id) - 1) * bytesPerRow);
|
|
1396
|
+
}
|
|
1397
|
+
const index = { bits, bytesPerRow, rows };
|
|
1398
|
+
let resident = bits.byteLength;
|
|
1399
|
+
for (const other of this.bitIndexes.values()) resident += other.bits.byteLength;
|
|
1400
|
+
for (const [key, other] of this.bitIndexes) {
|
|
1401
|
+
if (resident <= BIT_INDEX_BUDGET_BYTES) break;
|
|
1402
|
+
this.bitIndexes.delete(key);
|
|
1403
|
+
resident -= other.bits.byteLength;
|
|
1404
|
+
}
|
|
1405
|
+
this.bitIndexes.set(shard.path, index);
|
|
1406
|
+
return index;
|
|
1407
|
+
}
|
|
1408
|
+
/** The embedding dimension from the router's profile echo. */
|
|
1409
|
+
dimensions() {
|
|
1410
|
+
const raw = this.meta.embedding_profile_json;
|
|
1411
|
+
if (raw) {
|
|
1412
|
+
try {
|
|
1413
|
+
const dims = JSON.parse(raw).dimensions;
|
|
1414
|
+
if (typeof dims === "number" && dims > 0) return dims;
|
|
1415
|
+
} catch {
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1418
|
+
throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
|
|
1419
|
+
}
|
|
1302
1420
|
/** Resolve a manifest/router path without allowing it to escape the catalog. */
|
|
1303
1421
|
resolveCatalogPath(path) {
|
|
1304
1422
|
const root = resolve2(this.rootDir);
|
|
@@ -1375,13 +1493,26 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1375
1493
|
const match = ftsQuery(query);
|
|
1376
1494
|
if (!match) return [];
|
|
1377
1495
|
try {
|
|
1378
|
-
return this.router.db.
|
|
1379
|
-
|
|
1380
|
-
|
|
1496
|
+
return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
|
|
1497
|
+
documentId,
|
|
1498
|
+
rank: i
|
|
1499
|
+
}));
|
|
1381
1500
|
} catch {
|
|
1382
1501
|
return [];
|
|
1383
1502
|
}
|
|
1384
1503
|
}
|
|
1504
|
+
/**
|
|
1505
|
+
* Router-level shape stats: topic-tree size and total route centroids.
|
|
1506
|
+
* Read-only counts for inspectors and recall/routing diagnostics — a
|
|
1507
|
+
* single-topic catalog fills shards in documentId order, which is the
|
|
1508
|
+
* first thing to check when centroid routing underperforms.
|
|
1509
|
+
*/
|
|
1510
|
+
routerStats() {
|
|
1511
|
+
const count = (table) => Number(
|
|
1512
|
+
this.router.db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get().c
|
|
1513
|
+
);
|
|
1514
|
+
return { topics: count("topics"), routeCentroids: count("route_centroids") };
|
|
1515
|
+
}
|
|
1385
1516
|
/**
|
|
1386
1517
|
* Score every shard against the (unit float32) query: shard score = max
|
|
1387
1518
|
* cosine over its centroids (§3.4). This is the cross-catalog routing
|
|
@@ -1423,16 +1554,16 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1423
1554
|
* after global cross-catalog routing has already spent the S budget.
|
|
1424
1555
|
*/
|
|
1425
1556
|
searchShards(queryVector, shardIds, finalK) {
|
|
1426
|
-
const queryBits =
|
|
1557
|
+
const queryBits = quantizeBinary(queryVector);
|
|
1427
1558
|
const hits = [];
|
|
1428
1559
|
for (const shardId of shardIds) {
|
|
1429
1560
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
1430
1561
|
if (!shard) continue;
|
|
1431
1562
|
const db = this.shardDb(shard);
|
|
1432
1563
|
const k = rerankK(finalK, shard.chunkCount);
|
|
1433
|
-
const candidates =
|
|
1434
|
-
|
|
1435
|
-
)
|
|
1564
|
+
const candidates = hammingTopK(this.shardBits(shard), queryBits, k).map((hit) => ({
|
|
1565
|
+
chunk_id: hit.chunkId
|
|
1566
|
+
}));
|
|
1436
1567
|
if (candidates.length === 0) continue;
|
|
1437
1568
|
const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
|
|
1438
1569
|
const reranked = candidates.map((c) => {
|
|
@@ -1499,21 +1630,19 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1499
1630
|
return hits;
|
|
1500
1631
|
}
|
|
1501
1632
|
/**
|
|
1502
|
-
* Embedder-free integrity smoke
|
|
1503
|
-
*
|
|
1504
|
-
*
|
|
1633
|
+
* Embedder-free integrity smoke: chunk 1's own bit vector must be its own
|
|
1634
|
+
* nearest neighbor — proves the shard's vectors load and scan. Returns
|
|
1635
|
+
* false instead of throwing (callers report a reason).
|
|
1505
1636
|
*/
|
|
1506
1637
|
selfKnnSmoke(shardId) {
|
|
1507
1638
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
1508
1639
|
if (!shard) return false;
|
|
1509
1640
|
try {
|
|
1510
|
-
const
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
const nearest =
|
|
1514
|
-
|
|
1515
|
-
).get(Buffer.from(first.embedding));
|
|
1516
|
-
return nearest !== void 0 && Number(nearest.chunk_id) === 1;
|
|
1641
|
+
const index = this.shardBits(shard);
|
|
1642
|
+
if (index.rows === 0) return false;
|
|
1643
|
+
const first = index.bits.subarray(0, index.bytesPerRow);
|
|
1644
|
+
const nearest = hammingTopK(index, first, 1)[0];
|
|
1645
|
+
return nearest !== void 0 && nearest.chunkId === 1 && nearest.distance === 0;
|
|
1517
1646
|
} catch {
|
|
1518
1647
|
return false;
|
|
1519
1648
|
}
|
|
@@ -1549,18 +1678,12 @@ function parseHeadingPath(raw) {
|
|
|
1549
1678
|
return [];
|
|
1550
1679
|
}
|
|
1551
1680
|
}
|
|
1552
|
-
|
|
1553
|
-
const tokens = [
|
|
1554
|
-
...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
|
|
1555
|
-
];
|
|
1556
|
-
if (tokens.length === 0) return null;
|
|
1557
|
-
return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
|
|
1558
|
-
}
|
|
1681
|
+
var ftsQuery = sanitizeFtsQuery;
|
|
1559
1682
|
|
|
1560
1683
|
// src/reader/validate.ts
|
|
1561
1684
|
import { readFile } from "fs/promises";
|
|
1562
1685
|
import { join as join3 } from "path";
|
|
1563
|
-
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/
|
|
1686
|
+
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/gezk";
|
|
1564
1687
|
async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
1565
1688
|
const checks = [];
|
|
1566
1689
|
const check = (name, ok, detail) => {
|
|
@@ -1629,6 +1752,11 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1629
1752
|
);
|
|
1630
1753
|
const topics = handle.topics();
|
|
1631
1754
|
check("toc-present", topics.length >= 1, `${topics.length} topics`);
|
|
1755
|
+
check(
|
|
1756
|
+
"license-notice",
|
|
1757
|
+
manifest.files.some((f) => f.path === manifest.license.noticePath),
|
|
1758
|
+
`manifest.files lacks the declared notice ${manifest.license.noticePath}`
|
|
1759
|
+
);
|
|
1632
1760
|
const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
|
|
1633
1761
|
check(
|
|
1634
1762
|
"counts-documents",
|
|
@@ -1679,7 +1807,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1679
1807
|
db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n
|
|
1680
1808
|
);
|
|
1681
1809
|
const vecCount = Number(
|
|
1682
|
-
db.prepare("SELECT COUNT(*) AS n FROM
|
|
1810
|
+
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit").get().n
|
|
1683
1811
|
);
|
|
1684
1812
|
const int8Count = Number(
|
|
1685
1813
|
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8").get().n
|
|
@@ -1689,18 +1817,36 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1689
1817
|
chunkCount === shard.chunkCount && vecCount === chunkCount && int8Count === chunkCount,
|
|
1690
1818
|
`chunks ${chunkCount}, vec ${vecCount}, int8 ${int8Count}, shards row ${shard.chunkCount}`
|
|
1691
1819
|
);
|
|
1820
|
+
const dims = manifest.embedding.dimensions;
|
|
1821
|
+
const badBit = Number(
|
|
1822
|
+
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit WHERE length(v) != ?").get(Math.ceil(dims / 8)).n
|
|
1823
|
+
);
|
|
1824
|
+
const badInt8 = Number(
|
|
1825
|
+
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8 WHERE length(v) != ?").get(dims).n
|
|
1826
|
+
);
|
|
1827
|
+
check(
|
|
1828
|
+
`vector-widths:${shard.path}`,
|
|
1829
|
+
badBit === 0 && badInt8 === 0,
|
|
1830
|
+
`${badBit} bit rows and ${badInt8} int8 rows have the wrong width for ${dims} dimensions`
|
|
1831
|
+
);
|
|
1832
|
+
const span = db.prepare("SELECT MIN(id) AS lo, MAX(id) AS hi FROM chunks").get();
|
|
1833
|
+
check(
|
|
1834
|
+
`chunk-ids-dense:${shard.path}`,
|
|
1835
|
+
chunkCount === 0 || Number(span.lo) === 1 && Number(span.hi) === chunkCount,
|
|
1836
|
+
`ids span ${String(span.lo)}..${String(span.hi)} for ${chunkCount} chunks`
|
|
1837
|
+
);
|
|
1692
1838
|
} finally {
|
|
1693
1839
|
conn.close();
|
|
1694
1840
|
}
|
|
1695
1841
|
check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
|
|
1696
1842
|
}
|
|
1697
1843
|
for (const smoke of manifest.smokeQueries ?? []) {
|
|
1698
|
-
const hits = handle.searchDocumentsFts(smoke.query,
|
|
1844
|
+
const hits = handle.searchDocumentsFts(smoke.query, SMOKE_QUERY_TOP_N).map((h) => h.documentId);
|
|
1699
1845
|
const missing = smoke.expectedDocumentIds.filter((id) => !hits.includes(id));
|
|
1700
1846
|
check(
|
|
1701
1847
|
`smoke:${smoke.query}`,
|
|
1702
1848
|
missing.length === 0,
|
|
1703
|
-
missing.length > 0 ? `missing ${missing.join(", ")}` : void 0
|
|
1849
|
+
missing.length > 0 ? `the catalog's built-in sanity query "${smoke.query}" did not return its expected document${missing.length > 1 ? "s" : ""} (${missing.join(", ")}) in the top ${SMOKE_QUERY_TOP_N} results \u2014 the search index cannot answer queries the publisher recorded as guaranteed, so the archive is corrupt or was built by a broken toolchain; re-download it or report it to the publisher` : void 0
|
|
1704
1850
|
);
|
|
1705
1851
|
}
|
|
1706
1852
|
}
|
|
@@ -1710,56 +1856,642 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1710
1856
|
return report(manifest);
|
|
1711
1857
|
}
|
|
1712
1858
|
|
|
1859
|
+
// src/export/duckdb.ts
|
|
1860
|
+
import { spawn } from "child_process";
|
|
1861
|
+
import { access } from "fs/promises";
|
|
1862
|
+
var DuckdbError = class extends Error {
|
|
1863
|
+
constructor(message, stderr) {
|
|
1864
|
+
super(message);
|
|
1865
|
+
this.stderr = stderr;
|
|
1866
|
+
this.name = "DuckdbError";
|
|
1867
|
+
}
|
|
1868
|
+
stderr;
|
|
1869
|
+
};
|
|
1870
|
+
async function duckdbVersion(binaryPath) {
|
|
1871
|
+
const { stdout } = await runDuckdbProcess(binaryPath, ["--version"], "");
|
|
1872
|
+
const match = /v?(\d+\.\d+\.\d+)/.exec(stdout);
|
|
1873
|
+
if (!match?.[1])
|
|
1874
|
+
throw new DuckdbError(`could not read the DuckDB version from: ${stdout.trim()}`, "");
|
|
1875
|
+
return match[1];
|
|
1876
|
+
}
|
|
1877
|
+
async function assertDuckdbCli(cli) {
|
|
1878
|
+
await access(cli.binaryPath).catch(() => {
|
|
1879
|
+
throw new DuckdbError(`DuckDB CLI not found at ${cli.binaryPath}`, "");
|
|
1880
|
+
});
|
|
1881
|
+
const version = await duckdbVersion(cli.binaryPath);
|
|
1882
|
+
if (cli.expectedVersion && version !== cli.expectedVersion) {
|
|
1883
|
+
throw new DuckdbError(
|
|
1884
|
+
`DuckDB ${cli.expectedVersion} expected, but ${cli.binaryPath} is ${version}; Parquet output is only reproducible against the pinned release`,
|
|
1885
|
+
""
|
|
1886
|
+
);
|
|
1887
|
+
}
|
|
1888
|
+
return version;
|
|
1889
|
+
}
|
|
1890
|
+
async function runDuckdbScript(cli, sql) {
|
|
1891
|
+
const { stdout } = await runDuckdbProcess(cli.binaryPath, ["-batch", ":memory:"], sql);
|
|
1892
|
+
return stdout;
|
|
1893
|
+
}
|
|
1894
|
+
function runDuckdbProcess(binaryPath, args, stdin) {
|
|
1895
|
+
return new Promise((resolve4, reject) => {
|
|
1896
|
+
const child = spawn(binaryPath, args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
|
|
1897
|
+
let stdout = "";
|
|
1898
|
+
let stderr = "";
|
|
1899
|
+
child.stdout.setEncoding("utf8");
|
|
1900
|
+
child.stderr.setEncoding("utf8");
|
|
1901
|
+
child.stdout.on("data", (chunk) => {
|
|
1902
|
+
stdout += chunk;
|
|
1903
|
+
});
|
|
1904
|
+
child.stderr.on("data", (chunk) => {
|
|
1905
|
+
stderr += chunk;
|
|
1906
|
+
});
|
|
1907
|
+
child.on(
|
|
1908
|
+
"error",
|
|
1909
|
+
(err) => reject(new DuckdbError(`could not start DuckDB: ${err.message}`, stderr))
|
|
1910
|
+
);
|
|
1911
|
+
child.on("close", (code) => {
|
|
1912
|
+
if (code === 0) resolve4({ stdout });
|
|
1913
|
+
else
|
|
1914
|
+
reject(
|
|
1915
|
+
new DuckdbError(
|
|
1916
|
+
`DuckDB exited with code ${code}: ${stderr.trim() || stdout.trim()}`,
|
|
1917
|
+
stderr
|
|
1918
|
+
)
|
|
1919
|
+
);
|
|
1920
|
+
});
|
|
1921
|
+
child.stdin.on("error", () => {
|
|
1922
|
+
});
|
|
1923
|
+
child.stdin.end(stdin);
|
|
1924
|
+
});
|
|
1925
|
+
}
|
|
1926
|
+
|
|
1927
|
+
// src/export/parquet.ts
|
|
1928
|
+
import { createHash as createHash3 } from "crypto";
|
|
1929
|
+
import { createWriteStream as createWriteStream3 } from "fs";
|
|
1930
|
+
import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
|
|
1931
|
+
import { join as join4, resolve as resolve3 } from "path";
|
|
1932
|
+
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
1933
|
+
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
|
|
1934
|
+
var PARQUET_EXPORT_VERSION = 1;
|
|
1935
|
+
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
1936
|
+
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
1937
|
+
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
1938
|
+
async function exportCatalogParquet(opts) {
|
|
1939
|
+
const duckdbVersion2 = await assertDuckdbCli(opts.duckdb);
|
|
1940
|
+
const outDir = resolve3(opts.outDir);
|
|
1941
|
+
await mkdir3(outDir, { recursive: true });
|
|
1942
|
+
const work = join4(outDir, `.export-${process.pid}-${Date.now().toString(36)}`);
|
|
1943
|
+
await mkdir3(work, { recursive: true });
|
|
1944
|
+
try {
|
|
1945
|
+
let rootDir;
|
|
1946
|
+
if ("archivePath" in opts.source) {
|
|
1947
|
+
opts.onProgress?.({ phase: "extract" });
|
|
1948
|
+
rootDir = join4(work, "catalog");
|
|
1949
|
+
await extractGezkVerified(opts.source.archivePath, rootDir);
|
|
1950
|
+
} else {
|
|
1951
|
+
rootDir = resolve3(opts.source.rootDir);
|
|
1952
|
+
}
|
|
1953
|
+
const manifest = KnowledgeCatalogManifestSchema4.parse(
|
|
1954
|
+
JSON.parse(await readFile2(join4(rootDir, MANIFEST_PATH), "utf8"))
|
|
1955
|
+
);
|
|
1956
|
+
const staging = join4(work, "ndjson");
|
|
1957
|
+
await mkdir3(staging, { recursive: true });
|
|
1958
|
+
const files = [];
|
|
1959
|
+
const metadata = kvMetadata(manifest);
|
|
1960
|
+
const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
|
|
1961
|
+
const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
|
|
1962
|
+
try {
|
|
1963
|
+
const topicPaths = topicPathsOf(router);
|
|
1964
|
+
const topicRows = await stageTopics(router, join4(staging, "topics.ndjson"));
|
|
1965
|
+
opts.onProgress?.({ phase: "stage", table: "topics", rows: topicRows });
|
|
1966
|
+
files.push(
|
|
1967
|
+
await writeParquet(opts.duckdb, {
|
|
1968
|
+
outDir,
|
|
1969
|
+
file: "topics.parquet",
|
|
1970
|
+
table: "topics",
|
|
1971
|
+
rows: topicRows,
|
|
1972
|
+
sql: topicsCopy(
|
|
1973
|
+
join4(staging, "topics.ndjson"),
|
|
1974
|
+
join4(outDir, "topics.parquet"),
|
|
1975
|
+
rowGroupSize,
|
|
1976
|
+
metadata
|
|
1977
|
+
),
|
|
1978
|
+
onProgress: opts.onProgress
|
|
1979
|
+
})
|
|
1980
|
+
);
|
|
1981
|
+
for (const shard of manifest.router.shards) {
|
|
1982
|
+
const tag = String(shard.id).padStart(3, "0");
|
|
1983
|
+
const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
|
|
1984
|
+
const documentRows = await stageDocuments(router, shard.id, topicPaths, documentsNdjson);
|
|
1985
|
+
opts.onProgress?.({
|
|
1986
|
+
phase: "stage",
|
|
1987
|
+
table: "documents",
|
|
1988
|
+
shardId: shard.id,
|
|
1989
|
+
rows: documentRows
|
|
1990
|
+
});
|
|
1991
|
+
files.push(
|
|
1992
|
+
await writeParquet(opts.duckdb, {
|
|
1993
|
+
outDir,
|
|
1994
|
+
file: `documents-${tag}.parquet`,
|
|
1995
|
+
table: "documents",
|
|
1996
|
+
shardId: shard.id,
|
|
1997
|
+
rows: documentRows,
|
|
1998
|
+
sql: documentsCopy(
|
|
1999
|
+
documentsNdjson,
|
|
2000
|
+
join4(outDir, `documents-${tag}.parquet`),
|
|
2001
|
+
rowGroupSize,
|
|
2002
|
+
metadata
|
|
2003
|
+
),
|
|
2004
|
+
onProgress: opts.onProgress
|
|
2005
|
+
})
|
|
2006
|
+
);
|
|
2007
|
+
const shardDb = openCatalogDatabase(join4(rootDir, shard.path));
|
|
2008
|
+
const chunksNdjson = join4(staging, `chunks-${tag}.ndjson`);
|
|
2009
|
+
let chunkRows;
|
|
2010
|
+
try {
|
|
2011
|
+
chunkRows = await stageChunks(
|
|
2012
|
+
shardDb,
|
|
2013
|
+
shard.id,
|
|
2014
|
+
manifest.embedding.dimensions,
|
|
2015
|
+
chunksNdjson
|
|
2016
|
+
);
|
|
2017
|
+
} finally {
|
|
2018
|
+
shardDb.close();
|
|
2019
|
+
}
|
|
2020
|
+
opts.onProgress?.({ phase: "stage", table: "chunks", shardId: shard.id, rows: chunkRows });
|
|
2021
|
+
files.push(
|
|
2022
|
+
await writeParquet(opts.duckdb, {
|
|
2023
|
+
outDir,
|
|
2024
|
+
file: `chunks-${tag}.parquet`,
|
|
2025
|
+
table: "chunks",
|
|
2026
|
+
shardId: shard.id,
|
|
2027
|
+
rows: chunkRows,
|
|
2028
|
+
sql: chunksCopy(
|
|
2029
|
+
chunksNdjson,
|
|
2030
|
+
join4(outDir, `chunks-${tag}.parquet`),
|
|
2031
|
+
manifest.embedding.dimensions,
|
|
2032
|
+
rowGroupSize,
|
|
2033
|
+
metadata
|
|
2034
|
+
),
|
|
2035
|
+
onProgress: opts.onProgress
|
|
2036
|
+
})
|
|
2037
|
+
);
|
|
2038
|
+
}
|
|
2039
|
+
} finally {
|
|
2040
|
+
router.close();
|
|
2041
|
+
}
|
|
2042
|
+
const report = {
|
|
2043
|
+
exportVersion: PARQUET_EXPORT_VERSION,
|
|
2044
|
+
catalogId: manifest.id,
|
|
2045
|
+
version: manifest.version,
|
|
2046
|
+
publisherId: manifest.publisher.id,
|
|
2047
|
+
formatVersion: manifest.formatVersion,
|
|
2048
|
+
embeddingProfileId: manifest.embedding.id,
|
|
2049
|
+
dimensions: manifest.embedding.dimensions,
|
|
2050
|
+
files,
|
|
2051
|
+
duckdbVersion: duckdbVersion2,
|
|
2052
|
+
exporter: { name: KNOWLEDGE_TOOLCHAIN.name, version: KNOWLEDGE_TOOLCHAIN.version }
|
|
2053
|
+
};
|
|
2054
|
+
const { writeFile } = await import("fs/promises");
|
|
2055
|
+
await writeFile(
|
|
2056
|
+
join4(outDir, PARQUET_REPORT_PATH),
|
|
2057
|
+
`${JSON.stringify(report, null, 2)}
|
|
2058
|
+
`,
|
|
2059
|
+
"utf8"
|
|
2060
|
+
);
|
|
2061
|
+
return report;
|
|
2062
|
+
} finally {
|
|
2063
|
+
await rm(work, { recursive: true, force: true }).catch(() => {
|
|
2064
|
+
});
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
function ndjsonWriter(path) {
|
|
2068
|
+
const stream = createWriteStream3(path, { encoding: "utf8" });
|
|
2069
|
+
return {
|
|
2070
|
+
write(row) {
|
|
2071
|
+
const line = `${JSON.stringify(row)}
|
|
2072
|
+
`;
|
|
2073
|
+
if (stream.write(line)) return Promise.resolve();
|
|
2074
|
+
return new Promise((resolveDrain) => stream.once("drain", () => resolveDrain()));
|
|
2075
|
+
},
|
|
2076
|
+
close() {
|
|
2077
|
+
return new Promise((resolveClose, reject) => {
|
|
2078
|
+
stream.on("error", reject);
|
|
2079
|
+
stream.end(() => resolveClose());
|
|
2080
|
+
});
|
|
2081
|
+
}
|
|
2082
|
+
};
|
|
2083
|
+
}
|
|
2084
|
+
function topicPathsOf(router) {
|
|
2085
|
+
const parents = /* @__PURE__ */ new Map();
|
|
2086
|
+
for (const row of router.db.prepare("SELECT id, parent_id FROM topics").all()) {
|
|
2087
|
+
parents.set(row.id, row.parent_id);
|
|
2088
|
+
}
|
|
2089
|
+
const paths = /* @__PURE__ */ new Map();
|
|
2090
|
+
for (const id of parents.keys()) {
|
|
2091
|
+
const path = [];
|
|
2092
|
+
let cursor = id;
|
|
2093
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2094
|
+
while (cursor && !seen.has(cursor)) {
|
|
2095
|
+
seen.add(cursor);
|
|
2096
|
+
path.unshift(cursor);
|
|
2097
|
+
cursor = parents.get(cursor) ?? null;
|
|
2098
|
+
}
|
|
2099
|
+
paths.set(id, path);
|
|
2100
|
+
}
|
|
2101
|
+
return paths;
|
|
2102
|
+
}
|
|
2103
|
+
async function stageTopics(router, path) {
|
|
2104
|
+
const out = ndjsonWriter(path);
|
|
2105
|
+
let rows = 0;
|
|
2106
|
+
const statement = router.db.prepare(
|
|
2107
|
+
"SELECT id, parent_id, name, description, sort_key, document_count FROM topics ORDER BY sort_key, id"
|
|
2108
|
+
);
|
|
2109
|
+
for (const row of statement.iterate()) {
|
|
2110
|
+
await out.write({
|
|
2111
|
+
id: row.id,
|
|
2112
|
+
parent_id: row.parent_id ?? null,
|
|
2113
|
+
name: row.name,
|
|
2114
|
+
description: row.description ?? null,
|
|
2115
|
+
sort_key: row.sort_key,
|
|
2116
|
+
document_count: Number(row.document_count)
|
|
2117
|
+
});
|
|
2118
|
+
rows++;
|
|
2119
|
+
}
|
|
2120
|
+
await out.close();
|
|
2121
|
+
return rows;
|
|
2122
|
+
}
|
|
2123
|
+
async function stageDocuments(router, shardId, topicPaths, path) {
|
|
2124
|
+
const out = ndjsonWriter(path);
|
|
2125
|
+
let rows = 0;
|
|
2126
|
+
const aliasesOf = router.db.prepare(
|
|
2127
|
+
"SELECT alias FROM aliases WHERE document_id = ? ORDER BY alias"
|
|
2128
|
+
);
|
|
2129
|
+
const statement = router.db.prepare(
|
|
2130
|
+
`SELECT id, title, slug, summary, language, topic_id, shard_id, chunk_count,
|
|
2131
|
+
source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob
|
|
2132
|
+
FROM documents WHERE shard_id = ? ORDER BY id`
|
|
2133
|
+
);
|
|
2134
|
+
for (const row of statement.iterate(shardId)) {
|
|
2135
|
+
const aliases = aliasesOf.all(row.id).map(
|
|
2136
|
+
(a) => a.alias
|
|
2137
|
+
);
|
|
2138
|
+
await out.write({
|
|
2139
|
+
id: row.id,
|
|
2140
|
+
title: row.title,
|
|
2141
|
+
slug: row.slug,
|
|
2142
|
+
summary: row.summary ?? null,
|
|
2143
|
+
language: row.language,
|
|
2144
|
+
topic_id: row.topic_id,
|
|
2145
|
+
topic_path: topicPaths.get(row.topic_id) ?? [row.topic_id],
|
|
2146
|
+
shard_id: Number(row.shard_id),
|
|
2147
|
+
chunk_count: Number(row.chunk_count),
|
|
2148
|
+
markdown: decodeBody(row.body_codec, row.body_blob),
|
|
2149
|
+
source_url: row.source_url ?? null,
|
|
2150
|
+
source_revision: row.source_revision ?? null,
|
|
2151
|
+
source_updated_at: row.source_updated_at ?? null,
|
|
2152
|
+
attribution: row.attribution_json ?? null,
|
|
2153
|
+
aliases
|
|
2154
|
+
});
|
|
2155
|
+
rows++;
|
|
2156
|
+
}
|
|
2157
|
+
await out.close();
|
|
2158
|
+
return rows;
|
|
2159
|
+
}
|
|
2160
|
+
function decodeBody(codec, blob) {
|
|
2161
|
+
const bytes = Buffer.from(blob);
|
|
2162
|
+
if (codec === "br") {
|
|
2163
|
+
return brotliDecompressSync2(bytes, { maxOutputLength: MAX_KNOWLEDGE_DOCUMENT_BYTES }).toString(
|
|
2164
|
+
"utf8"
|
|
2165
|
+
);
|
|
2166
|
+
}
|
|
2167
|
+
if (codec === "none") return bytes.toString("utf8");
|
|
2168
|
+
throw new Error(`unknown body codec ${codec}`);
|
|
2169
|
+
}
|
|
2170
|
+
async function stageChunks(shard, shardId, dimensions, path) {
|
|
2171
|
+
const out = ndjsonWriter(path);
|
|
2172
|
+
const bitBytes = Math.ceil(dimensions / 8);
|
|
2173
|
+
let rows = 0;
|
|
2174
|
+
const statement = shard.db.prepare(
|
|
2175
|
+
`SELECT c.id, c.chunk_uid, c.document_id, c.ordinal, c.title, c.heading_path, c.heading_text,
|
|
2176
|
+
c.text, c.token_count, c.content_hash, c.line_start, c.line_end,
|
|
2177
|
+
i.v AS int8, b.v AS bits
|
|
2178
|
+
FROM chunks c
|
|
2179
|
+
JOIN chunk_vectors_int8 i ON i.chunk_id = c.id
|
|
2180
|
+
JOIN chunk_vectors_bit b ON b.chunk_id = c.id
|
|
2181
|
+
ORDER BY c.id`
|
|
2182
|
+
);
|
|
2183
|
+
for (const row of statement.iterate()) {
|
|
2184
|
+
const int8 = Buffer.from(row.int8);
|
|
2185
|
+
const bits = Buffer.from(row.bits);
|
|
2186
|
+
if (int8.length !== dimensions || bits.length !== bitBytes) {
|
|
2187
|
+
throw new Error(
|
|
2188
|
+
`chunk ${row.chunk_uid}: vector widths ${int8.length}/${bits.length} do not match the ${dimensions}-dimension profile`
|
|
2189
|
+
);
|
|
2190
|
+
}
|
|
2191
|
+
await out.write({
|
|
2192
|
+
id: Number(row.id),
|
|
2193
|
+
chunk_uid: row.chunk_uid,
|
|
2194
|
+
document_id: row.document_id,
|
|
2195
|
+
ordinal: Number(row.ordinal),
|
|
2196
|
+
title: row.title,
|
|
2197
|
+
heading_path: parseJsonStrings(row.heading_path),
|
|
2198
|
+
heading_text: row.heading_text,
|
|
2199
|
+
text: row.text,
|
|
2200
|
+
token_count: Number(row.token_count),
|
|
2201
|
+
content_hash: row.content_hash,
|
|
2202
|
+
line_start: Number(row.line_start),
|
|
2203
|
+
line_end: Number(row.line_end),
|
|
2204
|
+
shard_id: shardId,
|
|
2205
|
+
embedding: Array.from(new Int8Array(int8.buffer, int8.byteOffset, int8.length)),
|
|
2206
|
+
embedding_bit: bits.toString("base64")
|
|
2207
|
+
});
|
|
2208
|
+
rows++;
|
|
2209
|
+
}
|
|
2210
|
+
await out.close();
|
|
2211
|
+
return rows;
|
|
2212
|
+
}
|
|
2213
|
+
function parseJsonStrings(raw) {
|
|
2214
|
+
try {
|
|
2215
|
+
const parsed = JSON.parse(raw);
|
|
2216
|
+
return Array.isArray(parsed) ? parsed.filter((p) => typeof p === "string") : [];
|
|
2217
|
+
} catch {
|
|
2218
|
+
return [];
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
function sqlString(value) {
|
|
2222
|
+
return `'${value.replace(/'/g, "''")}'`;
|
|
2223
|
+
}
|
|
2224
|
+
function kvMetadata(manifest) {
|
|
2225
|
+
return {
|
|
2226
|
+
gezk_catalog_id: manifest.id,
|
|
2227
|
+
gezk_catalog_version: manifest.version,
|
|
2228
|
+
gezk_publisher_id: manifest.publisher.id,
|
|
2229
|
+
gezk_format_version: manifest.formatVersion,
|
|
2230
|
+
gezk_embedding_profile: manifest.embedding.id,
|
|
2231
|
+
gezk_embedding_model: manifest.embedding.model.repo,
|
|
2232
|
+
gezk_embedding_dimensions: String(manifest.embedding.dimensions),
|
|
2233
|
+
gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
|
|
2234
|
+
gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
|
|
2235
|
+
gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
|
|
2236
|
+
gezk_license: manifest.license.name
|
|
2237
|
+
};
|
|
2238
|
+
}
|
|
2239
|
+
function copyOptions(rowGroupSize, metadata) {
|
|
2240
|
+
const kv = Object.entries(metadata).map(([key, value]) => `${key}: ${sqlString(value)}`).join(", ");
|
|
2241
|
+
return `(FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 3, ROW_GROUP_SIZE ${rowGroupSize}, KV_METADATA {${kv}})`;
|
|
2242
|
+
}
|
|
2243
|
+
var PRELUDE = "SET threads = 1;\nSET preserve_insertion_order = true;\n";
|
|
2244
|
+
function readNdjson(path, columns) {
|
|
2245
|
+
const spec = Object.entries(columns).map(([name, type]) => `${name}: ${sqlString(type)}`).join(", ");
|
|
2246
|
+
return `read_ndjson(${sqlString(path)}, columns = {${spec}}, maximum_object_size = ${NDJSON_MAX_OBJECT_BYTES})`;
|
|
2247
|
+
}
|
|
2248
|
+
function topicsCopy(ndjson, out, rowGroupSize, metadata) {
|
|
2249
|
+
const source = readNdjson(ndjson, {
|
|
2250
|
+
id: "VARCHAR",
|
|
2251
|
+
parent_id: "VARCHAR",
|
|
2252
|
+
name: "VARCHAR",
|
|
2253
|
+
description: "VARCHAR",
|
|
2254
|
+
sort_key: "VARCHAR",
|
|
2255
|
+
document_count: "BIGINT"
|
|
2256
|
+
});
|
|
2257
|
+
return `${PRELUDE}COPY (SELECT id, parent_id, name, description, sort_key, document_count FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2258
|
+
`;
|
|
2259
|
+
}
|
|
2260
|
+
function documentsCopy(ndjson, out, rowGroupSize, metadata) {
|
|
2261
|
+
const source = readNdjson(ndjson, {
|
|
2262
|
+
id: "VARCHAR",
|
|
2263
|
+
title: "VARCHAR",
|
|
2264
|
+
slug: "VARCHAR",
|
|
2265
|
+
summary: "VARCHAR",
|
|
2266
|
+
language: "VARCHAR",
|
|
2267
|
+
topic_id: "VARCHAR",
|
|
2268
|
+
topic_path: "VARCHAR[]",
|
|
2269
|
+
shard_id: "INTEGER",
|
|
2270
|
+
chunk_count: "INTEGER",
|
|
2271
|
+
markdown: "VARCHAR",
|
|
2272
|
+
source_url: "VARCHAR",
|
|
2273
|
+
source_revision: "VARCHAR",
|
|
2274
|
+
source_updated_at: "VARCHAR",
|
|
2275
|
+
attribution: "VARCHAR",
|
|
2276
|
+
aliases: "VARCHAR[]"
|
|
2277
|
+
});
|
|
2278
|
+
return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2279
|
+
`;
|
|
2280
|
+
}
|
|
2281
|
+
function chunksCopy(ndjson, out, dimensions, rowGroupSize, metadata) {
|
|
2282
|
+
const source = readNdjson(ndjson, {
|
|
2283
|
+
id: "BIGINT",
|
|
2284
|
+
chunk_uid: "VARCHAR",
|
|
2285
|
+
document_id: "VARCHAR",
|
|
2286
|
+
ordinal: "INTEGER",
|
|
2287
|
+
title: "VARCHAR",
|
|
2288
|
+
heading_path: "VARCHAR[]",
|
|
2289
|
+
heading_text: "VARCHAR",
|
|
2290
|
+
text: "VARCHAR",
|
|
2291
|
+
token_count: "INTEGER",
|
|
2292
|
+
content_hash: "VARCHAR",
|
|
2293
|
+
line_start: "INTEGER",
|
|
2294
|
+
line_end: "INTEGER",
|
|
2295
|
+
shard_id: "INTEGER",
|
|
2296
|
+
embedding: "TINYINT[]",
|
|
2297
|
+
embedding_bit: "VARCHAR"
|
|
2298
|
+
});
|
|
2299
|
+
return `${PRELUDE}COPY (SELECT id, chunk_uid, document_id, ordinal, title, heading_path, heading_text, text, token_count, content_hash, line_start, line_end, shard_id, embedding::TINYINT[${dimensions}] AS embedding, from_base64(embedding_bit) AS embedding_bit FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2300
|
+
`;
|
|
2301
|
+
}
|
|
2302
|
+
async function writeParquet(cli, spec) {
|
|
2303
|
+
spec.onProgress?.({ phase: "write", file: spec.file });
|
|
2304
|
+
const target = join4(spec.outDir, spec.file);
|
|
2305
|
+
await rm(target, { force: true });
|
|
2306
|
+
await runDuckdbScript(cli, spec.sql);
|
|
2307
|
+
const bytes = await readFile2(target);
|
|
2308
|
+
return {
|
|
2309
|
+
path: spec.file,
|
|
2310
|
+
table: spec.table,
|
|
2311
|
+
...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
|
|
2312
|
+
rows: spec.rows,
|
|
2313
|
+
sizeBytes: (await stat2(target)).size,
|
|
2314
|
+
sha256: createHash3("sha256").update(bytes).digest("hex")
|
|
2315
|
+
};
|
|
2316
|
+
}
|
|
2317
|
+
async function findDuckdbBinary(env = process.env) {
|
|
2318
|
+
const configured = env.GEZEL_DUCKDB_BIN?.trim();
|
|
2319
|
+
if (configured) return configured;
|
|
2320
|
+
const { readdir: readdir2 } = await import("fs/promises");
|
|
2321
|
+
const { homedir } = await import("os");
|
|
2322
|
+
const name = process.platform === "win32" ? "duckdb.exe" : "duckdb";
|
|
2323
|
+
const homes = [env.GEZEL_HOME, join4(homedir(), ".gezel-dev"), join4(homedir(), ".gezel")].filter(
|
|
2324
|
+
(h) => Boolean(h)
|
|
2325
|
+
);
|
|
2326
|
+
for (const home of homes) {
|
|
2327
|
+
const engines = join4(home, "engines", "duckdb");
|
|
2328
|
+
const versions = (await readdir2(engines).catch(() => [])).sort().reverse();
|
|
2329
|
+
for (const version of versions) {
|
|
2330
|
+
const candidate = join4(engines, version, name);
|
|
2331
|
+
if (await stat2(candidate).catch(() => null)) return candidate;
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
const pathDirs = (env.PATH ?? "").split(process.platform === "win32" ? ";" : ":");
|
|
2335
|
+
for (const dir of pathDirs) {
|
|
2336
|
+
if (!dir) continue;
|
|
2337
|
+
const candidate = join4(dir, name);
|
|
2338
|
+
if (await stat2(candidate).catch(() => null)) return candidate;
|
|
2339
|
+
}
|
|
2340
|
+
return null;
|
|
2341
|
+
}
|
|
2342
|
+
|
|
1713
2343
|
// src/profiles/registry.ts
|
|
1714
2344
|
var SHARED_ENCODING = {
|
|
1715
2345
|
pooling: "mean",
|
|
1716
2346
|
normalized: true,
|
|
1717
2347
|
dimensions: 384,
|
|
1718
2348
|
maxTokens: 512,
|
|
1719
|
-
vectorEncoding: "
|
|
2349
|
+
vectorEncoding: "bit+int8",
|
|
1720
2350
|
distance: { stage1: "hamming", stage2: "cosine" },
|
|
1721
2351
|
quantization: {
|
|
1722
2352
|
int8: { method: "symmetric-linear", scale: 127 },
|
|
1723
2353
|
binary: { method: "sign", threshold: 0, packing: "lsb-first" }
|
|
1724
2354
|
}
|
|
1725
2355
|
};
|
|
1726
|
-
var
|
|
1727
|
-
id: "
|
|
2356
|
+
var MULTILINGUAL_E5_SMALL_1 = {
|
|
2357
|
+
id: "multilingual-e5-small@1",
|
|
1728
2358
|
model: {
|
|
1729
2359
|
repo: "Xenova/multilingual-e5-small",
|
|
1730
|
-
revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78"
|
|
2360
|
+
revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
|
|
2361
|
+
onnxFile: "onnx/model.onnx",
|
|
2362
|
+
onnxDigest: "sha256:4aa845c27760e06e9a686b9d8b5d440eae4b6612cd09e5b522b716d3941f77ff"
|
|
2363
|
+
},
|
|
2364
|
+
tokenizer: {
|
|
2365
|
+
kind: "sentencepiece-xlmr",
|
|
2366
|
+
file: "tokenizer.json",
|
|
2367
|
+
digest: "sha256:0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39"
|
|
1731
2368
|
},
|
|
1732
|
-
tokenizer: { kind: "sentencepiece-xlmr" },
|
|
1733
2369
|
queryInstruction: "query: ",
|
|
1734
2370
|
passageInstruction: "passage: ",
|
|
1735
2371
|
...SHARED_ENCODING
|
|
1736
2372
|
};
|
|
1737
|
-
var
|
|
1738
|
-
id: "
|
|
2373
|
+
var BGE_SMALL_EN_V15_1 = {
|
|
2374
|
+
id: "bge-small-en-v1.5@1",
|
|
1739
2375
|
model: {
|
|
1740
2376
|
repo: "Xenova/bge-small-en-v1.5",
|
|
1741
|
-
revision: "ea104dacec62c0de699686887e3f920caeb4f3e3"
|
|
2377
|
+
revision: "ea104dacec62c0de699686887e3f920caeb4f3e3",
|
|
2378
|
+
onnxFile: "onnx/model.onnx",
|
|
2379
|
+
onnxDigest: "sha256:828e1496d7fabb79cfa4dcd84fa38625c0d3d21da474a00f08db0f559940cf35"
|
|
2380
|
+
},
|
|
2381
|
+
tokenizer: {
|
|
2382
|
+
kind: "wordpiece-bert",
|
|
2383
|
+
file: "tokenizer.json",
|
|
2384
|
+
digest: "sha256:d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66"
|
|
1742
2385
|
},
|
|
1743
|
-
tokenizer: { kind: "wordpiece-bert" },
|
|
1744
2386
|
queryInstruction: "Represent this sentence for searching relevant passages: ",
|
|
1745
2387
|
passageInstruction: "",
|
|
1746
2388
|
...SHARED_ENCODING
|
|
1747
2389
|
};
|
|
1748
2390
|
var KNOWLEDGE_EMBEDDING_PROFILES = [
|
|
1749
|
-
|
|
1750
|
-
|
|
2391
|
+
MULTILINGUAL_E5_SMALL_1,
|
|
2392
|
+
BGE_SMALL_EN_V15_1
|
|
1751
2393
|
];
|
|
1752
2394
|
function knowledgeEmbeddingProfile(id) {
|
|
1753
2395
|
return KNOWLEDGE_EMBEDDING_PROFILES.find((p) => p.id === id) ?? null;
|
|
1754
2396
|
}
|
|
1755
|
-
var
|
|
1756
|
-
id: "
|
|
2397
|
+
var MARKDOWN_CHUNKS_2 = {
|
|
2398
|
+
id: "markdown-chunks@2",
|
|
1757
2399
|
unit: "tokens",
|
|
1758
2400
|
tokenizer: "profile",
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
contextHeader: {
|
|
2401
|
+
target: 420,
|
|
2402
|
+
overlap: 64,
|
|
2403
|
+
contextHeader: { max: 64 }
|
|
1762
2404
|
};
|
|
2405
|
+
var GEZEL_MULTILINGUAL_E5_SMALL_1 = MULTILINGUAL_E5_SMALL_1;
|
|
2406
|
+
var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
|
|
2407
|
+
var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
|
|
2408
|
+
|
|
2409
|
+
// src/embedding/profile-embedder.ts
|
|
2410
|
+
import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
|
|
2411
|
+
|
|
2412
|
+
// src/embedding/artifact-verify.ts
|
|
2413
|
+
import { createHash as createHash4 } from "crypto";
|
|
2414
|
+
import { createReadStream } from "fs";
|
|
2415
|
+
import { stat as stat3 } from "fs/promises";
|
|
2416
|
+
import { join as join5 } from "path";
|
|
2417
|
+
import { embeddingProfileArtifacts } from "@bendyline/gezk";
|
|
2418
|
+
var EmbedderArtifactError = class extends Error {
|
|
2419
|
+
constructor(reason, profileId, checks) {
|
|
2420
|
+
super(describeFailure(reason, profileId, checks));
|
|
2421
|
+
this.reason = reason;
|
|
2422
|
+
this.profileId = profileId;
|
|
2423
|
+
this.checks = checks;
|
|
2424
|
+
this.name = "EmbedderArtifactError";
|
|
2425
|
+
}
|
|
2426
|
+
reason;
|
|
2427
|
+
profileId;
|
|
2428
|
+
checks;
|
|
2429
|
+
isActionable = true;
|
|
2430
|
+
};
|
|
2431
|
+
function describeFailure(reason, profileId, checks) {
|
|
2432
|
+
const detail = checks.filter((c) => c.actual !== c.expected).map(
|
|
2433
|
+
(c) => c.actual === null ? `${c.file} not found at ${c.path}` : `${c.file} is ${c.actual}, profile pins ${c.expected}`
|
|
2434
|
+
).join("; ");
|
|
2435
|
+
return reason === "missing" ? `cannot verify the model files for profile ${profileId}: ${detail}` : `the model files loaded for profile ${profileId} differ from the ones it pins: ${detail}`;
|
|
2436
|
+
}
|
|
2437
|
+
function transformersCachePath(cacheDir, repo, revision, file) {
|
|
2438
|
+
return revision === "main" ? join5(cacheDir, repo, file) : join5(cacheDir, repo, revision, file);
|
|
2439
|
+
}
|
|
2440
|
+
var digestMemo = /* @__PURE__ */ new Map();
|
|
2441
|
+
var DIGEST_MEMO_MAX = 32;
|
|
2442
|
+
async function fileDigest(path) {
|
|
2443
|
+
let size;
|
|
2444
|
+
let mtimeMs;
|
|
2445
|
+
try {
|
|
2446
|
+
const info = await stat3(path);
|
|
2447
|
+
if (!info.isFile()) return null;
|
|
2448
|
+
size = info.size;
|
|
2449
|
+
mtimeMs = info.mtimeMs;
|
|
2450
|
+
} catch {
|
|
2451
|
+
return null;
|
|
2452
|
+
}
|
|
2453
|
+
const key = `${path} ${size} ${mtimeMs}`;
|
|
2454
|
+
const memo = digestMemo.get(key);
|
|
2455
|
+
if (memo) return memo;
|
|
2456
|
+
const digest = await new Promise((resolve4, reject) => {
|
|
2457
|
+
const hash = createHash4("sha256");
|
|
2458
|
+
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve4(`sha256:${hash.digest("hex")}`));
|
|
2459
|
+
});
|
|
2460
|
+
if (digestMemo.size >= DIGEST_MEMO_MAX) {
|
|
2461
|
+
const oldest = digestMemo.keys().next().value;
|
|
2462
|
+
if (oldest !== void 0) digestMemo.delete(oldest);
|
|
2463
|
+
}
|
|
2464
|
+
digestMemo.set(key, digest);
|
|
2465
|
+
return digest;
|
|
2466
|
+
}
|
|
2467
|
+
async function verifyProfileArtifacts(profile, opts) {
|
|
2468
|
+
const artifacts = embeddingProfileArtifacts(profile);
|
|
2469
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
2470
|
+
const pinned = [];
|
|
2471
|
+
if (artifacts.onnxDigest) {
|
|
2472
|
+
pinned.push({ role: "onnx", file: artifacts.onnxFile, expected: artifacts.onnxDigest });
|
|
2473
|
+
}
|
|
2474
|
+
if (artifacts.tokenizerDigest) {
|
|
2475
|
+
pinned.push({
|
|
2476
|
+
role: "tokenizer",
|
|
2477
|
+
file: artifacts.tokenizerFile,
|
|
2478
|
+
expected: artifacts.tokenizerDigest
|
|
2479
|
+
});
|
|
2480
|
+
}
|
|
2481
|
+
if (pinned.length === 0) return { status: "unpinned", checks: [] };
|
|
2482
|
+
const checks = [];
|
|
2483
|
+
for (const { role, file, expected } of pinned) {
|
|
2484
|
+
const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file);
|
|
2485
|
+
checks.push({ role, file, path, expected, actual: await fileDigest(path) });
|
|
2486
|
+
}
|
|
2487
|
+
if (checks.some((c) => c.actual === null)) {
|
|
2488
|
+
throw new EmbedderArtifactError("missing", profile.id, checks);
|
|
2489
|
+
}
|
|
2490
|
+
if (checks.some((c) => c.actual !== c.expected)) {
|
|
2491
|
+
throw new EmbedderArtifactError("mismatch", profile.id, checks);
|
|
2492
|
+
}
|
|
2493
|
+
return { status: "verified", checks };
|
|
2494
|
+
}
|
|
1763
2495
|
|
|
1764
2496
|
// src/embedding/profile-embedder.ts
|
|
1765
2497
|
var MAX_BATCH = 8;
|
|
@@ -1771,6 +2503,42 @@ var EmbedderUnavailableError = class extends Error {
|
|
|
1771
2503
|
this.name = "EmbedderUnavailableError";
|
|
1772
2504
|
}
|
|
1773
2505
|
};
|
|
2506
|
+
var ONNX_DTYPE_BY_SUFFIX = [
|
|
2507
|
+
["_q4f16", "q4f16"],
|
|
2508
|
+
["_quantized", "q8"],
|
|
2509
|
+
["_uint8", "uint8"],
|
|
2510
|
+
["_int8", "int8"],
|
|
2511
|
+
["_bnb4", "bnb4"],
|
|
2512
|
+
["_fp16", "fp16"],
|
|
2513
|
+
["_q4", "q4"]
|
|
2514
|
+
];
|
|
2515
|
+
var POOLING_BY_PROFILE = {
|
|
2516
|
+
mean: "mean",
|
|
2517
|
+
cls: "cls"
|
|
2518
|
+
};
|
|
2519
|
+
function resolveTransformersModelOptions(profile) {
|
|
2520
|
+
const { onnxFile, tokenizerFile } = embeddingProfileArtifacts2(profile);
|
|
2521
|
+
if (tokenizerFile !== "tokenizer.json") {
|
|
2522
|
+
throw new EmbedderUnavailableError(
|
|
2523
|
+
`profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
|
|
2524
|
+
);
|
|
2525
|
+
}
|
|
2526
|
+
if (!onnxFile.endsWith(".onnx")) {
|
|
2527
|
+
throw new EmbedderUnavailableError(
|
|
2528
|
+
`profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
|
|
2529
|
+
);
|
|
2530
|
+
}
|
|
2531
|
+
const slash = onnxFile.lastIndexOf("/");
|
|
2532
|
+
const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
|
|
2533
|
+
const base = onnxFile.slice(slash + 1, -".onnx".length);
|
|
2534
|
+
const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
|
|
2535
|
+
const [suffix, dtype] = variant ?? ["", "fp32"];
|
|
2536
|
+
const modelFileName = base.slice(0, base.length - suffix.length);
|
|
2537
|
+
if (modelFileName === "") {
|
|
2538
|
+
throw new EmbedderUnavailableError(`profile ${profile.id} pins an unnamed graph ${onnxFile}`);
|
|
2539
|
+
}
|
|
2540
|
+
return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
|
|
2541
|
+
}
|
|
1774
2542
|
async function loadTransformers() {
|
|
1775
2543
|
try {
|
|
1776
2544
|
const specifier = "@huggingface/transformers";
|
|
@@ -1786,30 +2554,52 @@ async function loadTransformers() {
|
|
|
1786
2554
|
}
|
|
1787
2555
|
}
|
|
1788
2556
|
async function createProfileEmbedder(profile, opts = {}) {
|
|
1789
|
-
const
|
|
2557
|
+
const pooling = POOLING_BY_PROFILE[profile.pooling];
|
|
2558
|
+
if (!pooling) {
|
|
2559
|
+
throw new EmbedderUnavailableError(
|
|
2560
|
+
`profile ${profile.id} uses ${profile.pooling} pooling, which this runtime does not implement`
|
|
2561
|
+
);
|
|
2562
|
+
}
|
|
2563
|
+
const modelOptions = resolveTransformersModelOptions(profile);
|
|
2564
|
+
const transformers = opts.transformers ?? await loadTransformers();
|
|
1790
2565
|
const cacheDir = opts.cacheDir ?? process.env.GEZEL_HF_CACHE_DIR;
|
|
1791
2566
|
if (cacheDir) {
|
|
1792
2567
|
transformers.env.cacheDir = cacheDir;
|
|
1793
2568
|
transformers.env.useFSCache = true;
|
|
1794
2569
|
transformers.env.allowRemoteModels = true;
|
|
1795
2570
|
}
|
|
1796
|
-
const modelOpts = { revision: profile.model.revision };
|
|
1797
2571
|
const [pipe, tokenizer] = await Promise.all([
|
|
1798
|
-
transformers.pipeline("feature-extraction", profile.model.repo,
|
|
1799
|
-
|
|
2572
|
+
transformers.pipeline("feature-extraction", profile.model.repo, {
|
|
2573
|
+
...modelOptions,
|
|
2574
|
+
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
|
|
2575
|
+
}),
|
|
2576
|
+
transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
|
|
2577
|
+
revision: modelOptions.revision
|
|
2578
|
+
})
|
|
1800
2579
|
]);
|
|
2580
|
+
let verification;
|
|
2581
|
+
try {
|
|
2582
|
+
verification = await verifyProfileArtifacts(profile, {
|
|
2583
|
+
cacheDir: cacheDir ?? transformers.env.cacheDir ?? ""
|
|
2584
|
+
});
|
|
2585
|
+
} catch (err) {
|
|
2586
|
+
await pipe.dispose?.().catch(() => {
|
|
2587
|
+
});
|
|
2588
|
+
throw err;
|
|
2589
|
+
}
|
|
1801
2590
|
const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
|
|
1802
2591
|
const embed = async (texts) => {
|
|
1803
2592
|
const out = [];
|
|
1804
2593
|
for (let i = 0; i < texts.length; i += MAX_BATCH) {
|
|
1805
2594
|
const slice = texts.slice(i, i + MAX_BATCH).map(cap);
|
|
1806
|
-
const result = await pipe(slice, { pooling
|
|
2595
|
+
const result = await pipe(slice, { pooling, normalize: true });
|
|
1807
2596
|
out.push(...result.tolist());
|
|
1808
2597
|
}
|
|
1809
2598
|
return out;
|
|
1810
2599
|
};
|
|
1811
2600
|
return {
|
|
1812
2601
|
profile,
|
|
2602
|
+
verification,
|
|
1813
2603
|
embed,
|
|
1814
2604
|
embedQuery: async (text) => {
|
|
1815
2605
|
const [vector] = await embed([`${profile.queryInstruction}${text}`]);
|
|
@@ -1823,9 +2613,9 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
1823
2613
|
}
|
|
1824
2614
|
|
|
1825
2615
|
// src/markdown-adapter/load.ts
|
|
1826
|
-
import { readFile as
|
|
1827
|
-
import { join as
|
|
1828
|
-
import { CatalogDocumentSchema } from "@bendyline/
|
|
2616
|
+
import { readFile as readFile3, readdir } from "fs/promises";
|
|
2617
|
+
import { join as join6, relative as relative3, sep } from "path";
|
|
2618
|
+
import { CatalogDocumentSchema } from "@bendyline/gezk";
|
|
1829
2619
|
var ROOT_TOPIC_ID = "general";
|
|
1830
2620
|
var SUMMARY_MAX_CHARS = 280;
|
|
1831
2621
|
function parseFrontMatter(markdown) {
|
|
@@ -1879,7 +2669,7 @@ async function walkMarkdownFiles(root) {
|
|
|
1879
2669
|
);
|
|
1880
2670
|
for (const entry of entries) {
|
|
1881
2671
|
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
1882
|
-
const abs =
|
|
2672
|
+
const abs = join6(dir, entry.name);
|
|
1883
2673
|
if (entry.isDirectory()) await visit(abs);
|
|
1884
2674
|
else if (entry.isFile() && /\.(md|markdown)$/i.test(entry.name)) out.push(abs);
|
|
1885
2675
|
}
|
|
@@ -1930,7 +2720,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
1930
2720
|
const id = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
|
|
1931
2721
|
if (seenIds.has(id)) throw new Error(`duplicate document id from path: ${rel}`);
|
|
1932
2722
|
seenIds.add(id);
|
|
1933
|
-
const raw = (await
|
|
2723
|
+
const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
|
|
1934
2724
|
const fm = parseFrontMatter(raw);
|
|
1935
2725
|
const stem = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
|
|
1936
2726
|
const title = fm.title ?? firstHeading(fm.body) ?? stem;
|
|
@@ -1959,110 +2749,131 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
1959
2749
|
return { topics, documents };
|
|
1960
2750
|
}
|
|
1961
2751
|
|
|
1962
|
-
// src/
|
|
1963
|
-
|
|
1964
|
-
return serialize(value, "$");
|
|
1965
|
-
}
|
|
1966
|
-
function serialize(value, path) {
|
|
1967
|
-
if (value === null) return "null";
|
|
1968
|
-
switch (typeof value) {
|
|
1969
|
-
case "boolean":
|
|
1970
|
-
return value ? "true" : "false";
|
|
1971
|
-
case "number":
|
|
1972
|
-
if (!Number.isFinite(value)) throw new Error(`non-finite number at ${path}`);
|
|
1973
|
-
return JSON.stringify(value);
|
|
1974
|
-
case "string":
|
|
1975
|
-
return JSON.stringify(value);
|
|
1976
|
-
case "object":
|
|
1977
|
-
break;
|
|
1978
|
-
default:
|
|
1979
|
-
throw new Error(`value of type ${typeof value} at ${path} has no JSON identity`);
|
|
1980
|
-
}
|
|
1981
|
-
if (Array.isArray(value)) {
|
|
1982
|
-
return `[${value.map((item, i) => serialize(item ?? null, `${path}[${i}]`)).join(",")}]`;
|
|
1983
|
-
}
|
|
1984
|
-
const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => compareUtf16(a, b));
|
|
1985
|
-
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${serialize(v, `${path}.${k}`)}`).join(",")}}`;
|
|
1986
|
-
}
|
|
1987
|
-
function compareUtf16(a, b) {
|
|
1988
|
-
const n = Math.min(a.length, b.length);
|
|
1989
|
-
for (let i = 0; i < n; i++) {
|
|
1990
|
-
const d = a.charCodeAt(i) - b.charCodeAt(i);
|
|
1991
|
-
if (d !== 0) return d;
|
|
1992
|
-
}
|
|
1993
|
-
return a.length - b.length;
|
|
1994
|
-
}
|
|
2752
|
+
// src/registry-client/fetch.ts
|
|
2753
|
+
import { KnowledgeRegistryIndexSchema } from "@bendyline/gezk";
|
|
1995
2754
|
|
|
1996
2755
|
// src/signatures/signing.ts
|
|
1997
2756
|
import {
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
};
|
|
2017
|
-
}
|
|
2018
|
-
function manifestSigningPayload(manifest) {
|
|
2019
|
-
const { signature: _omitted, ...unsigned } = manifest;
|
|
2020
|
-
return Buffer.from(canonicalizeJson(unsigned), "utf8");
|
|
2021
|
-
}
|
|
2022
|
-
function signManifest(manifest, privateKeyPem) {
|
|
2023
|
-
const key = createPrivateKey(privateKeyPem);
|
|
2024
|
-
const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
|
|
2025
|
-
const value = cryptoSign(null, manifestSigningPayload(manifest), key).toString("base64");
|
|
2026
|
-
return {
|
|
2027
|
-
...manifest,
|
|
2028
|
-
signature: {
|
|
2029
|
-
algorithm: "ed25519",
|
|
2030
|
-
keyId: knowledgeKeyId(publicKeyPem),
|
|
2031
|
-
canonicalization: "rfc8785",
|
|
2032
|
-
value
|
|
2033
|
-
}
|
|
2034
|
-
};
|
|
2035
|
-
}
|
|
2036
|
-
function verifyManifestSignature(manifest, anchors) {
|
|
2037
|
-
const signature = manifest.signature;
|
|
2038
|
-
if (!signature) return { ok: false, reason: "unsigned" };
|
|
2039
|
-
const anchor = anchors.find((a) => a.keyId === signature.keyId);
|
|
2040
|
-
if (!anchor) {
|
|
2041
|
-
return { ok: false, reason: "unknown-key", detail: `no trust anchor for ${signature.keyId}` };
|
|
2757
|
+
generateKnowledgeSigningKeyPair,
|
|
2758
|
+
knowledgeKeyId,
|
|
2759
|
+
manifestSigningPayload,
|
|
2760
|
+
signManifest,
|
|
2761
|
+
signRegistryIndex,
|
|
2762
|
+
verifyManifestSignature,
|
|
2763
|
+
verifyRegistryIndex
|
|
2764
|
+
} from "@bendyline/gezk/node";
|
|
2765
|
+
|
|
2766
|
+
// src/registry-client/fetch.ts
|
|
2767
|
+
var DEFAULT_MAX_REGISTRY_BYTES = 4 * 1024 * 1024;
|
|
2768
|
+
var DEFAULT_TIMEOUT_MS = 3e4;
|
|
2769
|
+
var KnowledgeRegistryFetchError = class extends Error {
|
|
2770
|
+
code;
|
|
2771
|
+
constructor(code, message) {
|
|
2772
|
+
super(message);
|
|
2773
|
+
this.name = "KnowledgeRegistryFetchError";
|
|
2774
|
+
this.code = code;
|
|
2042
2775
|
}
|
|
2776
|
+
};
|
|
2777
|
+
async function fetchKnowledgeRegistry(url, opts) {
|
|
2778
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
2779
|
+
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_REGISTRY_BYTES;
|
|
2780
|
+
let response;
|
|
2043
2781
|
try {
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2782
|
+
response = await fetchImpl(url, {
|
|
2783
|
+
signal: opts.signal ?? AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
|
2784
|
+
redirect: "follow",
|
|
2785
|
+
headers: { accept: "application/json" }
|
|
2786
|
+
});
|
|
2787
|
+
} catch (err) {
|
|
2788
|
+
throw new KnowledgeRegistryFetchError(
|
|
2789
|
+
"http",
|
|
2790
|
+
`registry fetch failed: ${err instanceof Error ? err.message : String(err)}`
|
|
2791
|
+
);
|
|
2792
|
+
}
|
|
2793
|
+
if (!response.ok) {
|
|
2794
|
+
throw new KnowledgeRegistryFetchError("http", `registry fetch failed: HTTP ${response.status}`);
|
|
2795
|
+
}
|
|
2796
|
+
const declared = Number(response.headers.get("content-length") ?? "0");
|
|
2797
|
+
if (declared > maxBytes) {
|
|
2798
|
+
throw new KnowledgeRegistryFetchError(
|
|
2799
|
+
"too-large",
|
|
2800
|
+
`registry declares ${declared} bytes (cap ${maxBytes})`
|
|
2801
|
+
);
|
|
2802
|
+
}
|
|
2803
|
+
const body = Buffer.from(await response.arrayBuffer());
|
|
2804
|
+
if (body.byteLength > maxBytes) {
|
|
2805
|
+
throw new KnowledgeRegistryFetchError(
|
|
2806
|
+
"too-large",
|
|
2807
|
+
`registry body is ${body.byteLength} bytes (cap ${maxBytes})`
|
|
2052
2808
|
);
|
|
2053
|
-
|
|
2809
|
+
}
|
|
2810
|
+
let registry;
|
|
2811
|
+
try {
|
|
2812
|
+
registry = KnowledgeRegistryIndexSchema.parse(JSON.parse(body.toString("utf8")));
|
|
2054
2813
|
} catch (err) {
|
|
2055
|
-
|
|
2814
|
+
throw new KnowledgeRegistryFetchError(
|
|
2815
|
+
"invalid",
|
|
2816
|
+
`registry is not a valid gezk-registry document: ${err instanceof Error ? err.message : String(err)}`
|
|
2817
|
+
);
|
|
2818
|
+
}
|
|
2819
|
+
const verdict = verifyRegistryIndex(registry, opts.anchors);
|
|
2820
|
+
if (!verdict.ok) {
|
|
2821
|
+
throw new KnowledgeRegistryFetchError(
|
|
2822
|
+
"unsigned-or-untrusted",
|
|
2823
|
+
`registry signature rejected (${verdict.reason}${verdict.detail ? `: ${verdict.detail}` : ""})`
|
|
2824
|
+
);
|
|
2825
|
+
}
|
|
2826
|
+
return { registry, keyId: verdict.keyId };
|
|
2827
|
+
}
|
|
2828
|
+
function findRegistryEntry(registry, ref) {
|
|
2829
|
+
if (registry.publisher.id !== ref.publisherId) return null;
|
|
2830
|
+
const entry = registry.catalogs.find(
|
|
2831
|
+
(c) => c.catalogId === ref.catalogId && c.version === ref.version
|
|
2832
|
+
);
|
|
2833
|
+
if (!entry) return null;
|
|
2834
|
+
if (ref.contentDigest && entry.contentDigest !== ref.contentDigest.toLowerCase()) return null;
|
|
2835
|
+
return entry;
|
|
2836
|
+
}
|
|
2837
|
+
function compareCatalogVersions(a, b) {
|
|
2838
|
+
const as = a.split(".");
|
|
2839
|
+
const bs = b.split(".");
|
|
2840
|
+
const len = Math.max(as.length, bs.length);
|
|
2841
|
+
for (let i = 0; i < len; i++) {
|
|
2842
|
+
const av = as[i] ?? "";
|
|
2843
|
+
const bv = bs[i] ?? "";
|
|
2844
|
+
const an = /^\d+$/.test(av) ? Number(av) : null;
|
|
2845
|
+
const bn = /^\d+$/.test(bv) ? Number(bv) : null;
|
|
2846
|
+
if (an !== null && bn !== null) {
|
|
2847
|
+
if (an !== bn) return an < bn ? -1 : 1;
|
|
2848
|
+
} else if (av !== bv) {
|
|
2849
|
+
return av < bv ? -1 : 1;
|
|
2850
|
+
}
|
|
2056
2851
|
}
|
|
2852
|
+
return 0;
|
|
2853
|
+
}
|
|
2854
|
+
function newerRegistryEntries(registry, installed) {
|
|
2855
|
+
return registry.catalogs.filter(
|
|
2856
|
+
(c) => c.catalogId === installed.catalogId && compareCatalogVersions(c.version, installed.version) > 0
|
|
2857
|
+
).sort((x, y) => compareCatalogVersions(y.version, x.version));
|
|
2057
2858
|
}
|
|
2859
|
+
|
|
2860
|
+
// src/signatures/anchors.ts
|
|
2861
|
+
var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
|
|
2862
|
+
|
|
2863
|
+
// src/signatures/jcs.ts
|
|
2864
|
+
import { canonicalizeJson } from "@bendyline/gezk";
|
|
2058
2865
|
export {
|
|
2866
|
+
BGE_SMALL_EN_V15_1,
|
|
2059
2867
|
BODY_CODEC_MIN_BYTES,
|
|
2868
|
+
BUILTIN_KNOWLEDGE_TRUST_ANCHORS,
|
|
2060
2869
|
CENTROID_CHUNKS_PER,
|
|
2061
2870
|
CENTROID_KMEANS_MAX_ITER,
|
|
2062
2871
|
CENTROID_MAX_PER_SHARD,
|
|
2063
2872
|
CENTROID_SAMPLE_MAX,
|
|
2064
2873
|
CatalogHandle,
|
|
2065
2874
|
CatalogOpenError,
|
|
2875
|
+
DuckdbError,
|
|
2876
|
+
EmbedderArtifactError,
|
|
2066
2877
|
EmbedderUnavailableError,
|
|
2067
2878
|
GEZEL_BGE_SMALL_EN_V15_1,
|
|
2068
2879
|
GEZEL_MARKDOWN_CHUNKS_2,
|
|
@@ -2071,10 +2882,20 @@ export {
|
|
|
2071
2882
|
GEZK_ARCHIVE_LIMITS,
|
|
2072
2883
|
GEZK_FORMAT_VERSION,
|
|
2073
2884
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
2885
|
+
GEZK_MIME_TYPE,
|
|
2074
2886
|
GezkArchiveError,
|
|
2075
2887
|
KNOWLEDGE_EMBEDDING_PROFILES,
|
|
2888
|
+
KNOWLEDGE_TOOLCHAIN,
|
|
2889
|
+
KnowledgeRegistryFetchError,
|
|
2890
|
+
LICENSE_NOTICE_PATH,
|
|
2076
2891
|
MANIFEST_PATH,
|
|
2892
|
+
MARKDOWN_CHUNKS_2,
|
|
2077
2893
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
2894
|
+
MIMETYPE_PATH,
|
|
2895
|
+
MULTILINGUAL_E5_SMALL_1,
|
|
2896
|
+
PARQUET_EXPORT_VERSION,
|
|
2897
|
+
PARQUET_REPORT_PATH,
|
|
2898
|
+
README_PATH,
|
|
2078
2899
|
RERANK_FINAL_K,
|
|
2079
2900
|
ROUTER_DB_PATH,
|
|
2080
2901
|
ROUTER_DDL,
|
|
@@ -2083,31 +2904,46 @@ export {
|
|
|
2083
2904
|
SHARD_DDL,
|
|
2084
2905
|
SHARD_MAX_CHUNKS,
|
|
2085
2906
|
SHARD_TARGET_CHUNKS,
|
|
2907
|
+
SOURCE_NOTICES_PATH,
|
|
2086
2908
|
ZIP_FIXED_MTIME,
|
|
2909
|
+
assertDuckdbCli,
|
|
2087
2910
|
canonicalizeJson,
|
|
2088
2911
|
chunkContentHash,
|
|
2089
2912
|
chunkMarkdownProfile,
|
|
2090
2913
|
chunkUid,
|
|
2914
|
+
compareCatalogVersions,
|
|
2091
2915
|
compileKnowledgeCatalog,
|
|
2092
2916
|
createProfileEmbedder,
|
|
2093
2917
|
documentSlug,
|
|
2918
|
+
duckdbVersion,
|
|
2919
|
+
exportCatalogParquet,
|
|
2094
2920
|
extractGezkVerified,
|
|
2921
|
+
fetchKnowledgeRegistry,
|
|
2922
|
+
findDuckdbBinary,
|
|
2923
|
+
findRegistryEntry,
|
|
2095
2924
|
generateKnowledgeSigningKeyPair,
|
|
2925
|
+
hammingTopK,
|
|
2096
2926
|
inspectGezkArchive,
|
|
2097
2927
|
knowledgeEmbeddingProfile,
|
|
2098
2928
|
knowledgeKeyId,
|
|
2099
2929
|
l2Normalize,
|
|
2100
2930
|
loadMarkdownCatalog,
|
|
2101
2931
|
manifestSigningPayload,
|
|
2932
|
+
newerRegistryEntries,
|
|
2102
2933
|
openCatalogDatabase,
|
|
2103
2934
|
quantizeBinary,
|
|
2104
2935
|
quantizeInt8,
|
|
2105
2936
|
readGezkManifest,
|
|
2106
2937
|
rerankK,
|
|
2107
2938
|
rerankScore,
|
|
2939
|
+
resolveTransformersModelOptions,
|
|
2940
|
+
runDuckdbScript,
|
|
2108
2941
|
signManifest,
|
|
2942
|
+
signRegistryIndex,
|
|
2943
|
+
transformersCachePath,
|
|
2109
2944
|
validateExtractedCatalog,
|
|
2110
|
-
vecChunksDdl,
|
|
2111
2945
|
verifyManifestSignature,
|
|
2946
|
+
verifyProfileArtifacts,
|
|
2947
|
+
verifyRegistryIndex,
|
|
2112
2948
|
writeGezkArchive
|
|
2113
2949
|
};
|