@bendyline/gezel-knowledge 1.0.7 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -17
- package/dist/index.d.ts +376 -157
- package/dist/index.js +1061 -421
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
1
|
// src/format/constants.ts
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
import {
|
|
3
|
+
BODY_CODEC_MIN_BYTES,
|
|
4
|
+
GEZK_APPLICATION_ID,
|
|
5
|
+
GEZK_FORMAT_VERSION,
|
|
6
|
+
GEZK_INDEX_SCHEMA_VERSION,
|
|
7
|
+
GEZK_MIME_TYPE,
|
|
8
|
+
LICENSE_NOTICE_PATH,
|
|
9
|
+
MANIFEST_PATH,
|
|
10
|
+
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
11
|
+
MIMETYPE_PATH,
|
|
12
|
+
README_PATH,
|
|
13
|
+
ROUTER_DB_PATH,
|
|
14
|
+
SOURCE_NOTICES_PATH,
|
|
15
|
+
ZIP_FIXED_MTIME
|
|
16
|
+
} from "@bendyline/gezk";
|
|
7
17
|
var SHARD_TARGET_CHUNKS = 2e5;
|
|
8
18
|
var SHARD_MAX_CHUNKS = 25e4;
|
|
9
|
-
var BODY_CODEC_MIN_BYTES = 512;
|
|
10
|
-
var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
|
|
11
19
|
var CENTROID_CHUNKS_PER = 12500;
|
|
12
20
|
var CENTROID_MAX_PER_SHARD = 32;
|
|
13
21
|
var CENTROID_KMEANS_MAX_ITER = 25;
|
|
@@ -18,144 +26,16 @@ var RERANK_FINAL_K = 24;
|
|
|
18
26
|
function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
|
|
19
27
|
return Math.min(512, Math.max(128, 8 * finalK), chunkCount);
|
|
20
28
|
}
|
|
21
|
-
var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
|
|
22
29
|
|
|
23
30
|
// src/format/ddl.ts
|
|
24
|
-
|
|
25
|
-
CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
26
|
-
|
|
27
|
-
CREATE TABLE topics(
|
|
28
|
-
id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,
|
|
29
|
-
description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL
|
|
30
|
-
);
|
|
31
|
-
|
|
32
|
-
CREATE TABLE documents(
|
|
33
|
-
id TEXT PRIMARY KEY,
|
|
34
|
-
title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
|
|
35
|
-
language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
|
|
36
|
-
shard_id INTEGER NOT NULL,
|
|
37
|
-
chunk_count INTEGER NOT NULL,
|
|
38
|
-
source_url TEXT, source_revision TEXT, source_updated_at TEXT,
|
|
39
|
-
attribution_json TEXT,
|
|
40
|
-
body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
|
|
41
|
-
body_blob BLOB NOT NULL
|
|
42
|
-
);
|
|
43
|
-
CREATE INDEX documents_topic ON documents(topic_id, slug);
|
|
44
|
-
CREATE INDEX documents_shard ON documents(shard_id);
|
|
45
|
-
|
|
46
|
-
CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
|
|
47
|
-
PRIMARY KEY (alias, document_id)) WITHOUT ROWID;
|
|
48
|
-
|
|
49
|
-
CREATE TABLE shards(
|
|
50
|
-
id INTEGER PRIMARY KEY,
|
|
51
|
-
path TEXT NOT NULL,
|
|
52
|
-
chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,
|
|
53
|
-
topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,
|
|
54
|
-
bytes INTEGER NOT NULL
|
|
55
|
-
);
|
|
56
|
-
|
|
57
|
-
CREATE TABLE route_centroids(
|
|
58
|
-
id INTEGER PRIMARY KEY,
|
|
59
|
-
shard_id INTEGER NOT NULL REFERENCES shards(id),
|
|
60
|
-
embedding BLOB NOT NULL,
|
|
61
|
-
weight INTEGER NOT NULL
|
|
62
|
-
);
|
|
63
|
-
CREATE INDEX route_centroids_shard ON route_centroids(shard_id);
|
|
64
|
-
|
|
65
|
-
CREATE VIRTUAL TABLE fts_documents USING fts5(
|
|
66
|
-
title, summary, aliases, document_id UNINDEXED,
|
|
67
|
-
tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
|
|
68
|
-
);
|
|
69
|
-
`;
|
|
70
|
-
var SHARD_DDL = `
|
|
71
|
-
CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
72
|
-
|
|
73
|
-
CREATE TABLE chunks(
|
|
74
|
-
id INTEGER PRIMARY KEY,
|
|
75
|
-
chunk_uid TEXT NOT NULL,
|
|
76
|
-
document_id TEXT NOT NULL,
|
|
77
|
-
ordinal INTEGER NOT NULL,
|
|
78
|
-
title TEXT NOT NULL,
|
|
79
|
-
heading_path TEXT NOT NULL,
|
|
80
|
-
heading_text TEXT NOT NULL,
|
|
81
|
-
line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,
|
|
82
|
-
token_count INTEGER NOT NULL,
|
|
83
|
-
content_hash TEXT NOT NULL,
|
|
84
|
-
text TEXT NOT NULL
|
|
85
|
-
);
|
|
86
|
-
CREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);
|
|
87
|
-
CREATE INDEX chunks_document ON chunks(document_id, ordinal);
|
|
88
|
-
|
|
89
|
-
CREATE VIRTUAL TABLE fts_chunks USING fts5(
|
|
90
|
-
title, heading_text, text,
|
|
91
|
-
content = 'chunks', content_rowid = 'id',
|
|
92
|
-
tokenize = 'unicode61 remove_diacritics 2'
|
|
93
|
-
);
|
|
94
|
-
|
|
95
|
-
CREATE TABLE chunk_vectors_int8(
|
|
96
|
-
chunk_id INTEGER PRIMARY KEY,
|
|
97
|
-
v BLOB NOT NULL
|
|
98
|
-
);
|
|
99
|
-
`;
|
|
100
|
-
function vecChunksDdl(dim) {
|
|
101
|
-
return `CREATE VIRTUAL TABLE vec_chunks USING vec0(
|
|
102
|
-
chunk_id INTEGER PRIMARY KEY,
|
|
103
|
-
embedding bit[${dim}],
|
|
104
|
-
chunk_size=1024
|
|
105
|
-
);`;
|
|
106
|
-
}
|
|
31
|
+
import { ROUTER_DDL, SHARD_DDL } from "@bendyline/gezk";
|
|
107
32
|
|
|
108
33
|
// src/format/ids.ts
|
|
109
|
-
import {
|
|
110
|
-
|
|
111
|
-
const textHash = createHash("sha256").update(chunkText, "utf8").digest();
|
|
112
|
-
return createHash("sha256").update(documentId, "utf8").update(Buffer.from([0])).update(String(ordinal), "utf8").update(Buffer.from([0])).update(textHash).digest("hex").slice(0, 32);
|
|
113
|
-
}
|
|
114
|
-
function chunkContentHash(chunkText) {
|
|
115
|
-
return createHash("sha256").update(chunkText, "utf8").digest("hex");
|
|
116
|
-
}
|
|
117
|
-
function documentSlug(title) {
|
|
118
|
-
const slug = title.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
119
|
-
return slug || "untitled";
|
|
120
|
-
}
|
|
34
|
+
import { documentSlug } from "@bendyline/gezk";
|
|
35
|
+
import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
|
|
121
36
|
|
|
122
37
|
// src/format/quantize.ts
|
|
123
|
-
|
|
124
|
-
const out = new Int8Array(unitVector.length);
|
|
125
|
-
for (let i = 0; i < unitVector.length; i++) {
|
|
126
|
-
const q = Math.round(unitVector[i] * 127);
|
|
127
|
-
out[i] = q > 127 ? 127 : q < -127 ? -127 : q;
|
|
128
|
-
}
|
|
129
|
-
return out;
|
|
130
|
-
}
|
|
131
|
-
function quantizeBinary(unitVector) {
|
|
132
|
-
const bytes = Math.ceil(unitVector.length / 8);
|
|
133
|
-
const out = new Uint8Array(bytes);
|
|
134
|
-
for (let i = 0; i < unitVector.length; i++) {
|
|
135
|
-
if (unitVector[i] > 0) {
|
|
136
|
-
out[i >> 3] = out[i >> 3] | 1 << (i & 7);
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
return out;
|
|
140
|
-
}
|
|
141
|
-
function rerankScore(query, passageInt8) {
|
|
142
|
-
let dot = 0;
|
|
143
|
-
const n = Math.min(query.length, passageInt8.length);
|
|
144
|
-
for (let i = 0; i < n; i++) dot += query[i] * (passageInt8[i] / 127);
|
|
145
|
-
return dot;
|
|
146
|
-
}
|
|
147
|
-
function l2Normalize(vector) {
|
|
148
|
-
let sum = 0;
|
|
149
|
-
for (let i = 0; i < vector.length; i++) {
|
|
150
|
-
const v = vector[i];
|
|
151
|
-
sum += v * v;
|
|
152
|
-
}
|
|
153
|
-
const norm = Math.sqrt(sum);
|
|
154
|
-
if (!Number.isFinite(norm) || norm === 0) throw new Error("degenerate vector (zero norm)");
|
|
155
|
-
const out = new Float32Array(vector.length);
|
|
156
|
-
for (let i = 0; i < vector.length; i++) out[i] = vector[i] / norm;
|
|
157
|
-
return out;
|
|
158
|
-
}
|
|
38
|
+
import { l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from "@bendyline/gezk";
|
|
159
39
|
|
|
160
40
|
// src/chunking/markdown-chunker.ts
|
|
161
41
|
var HEADING = /^(#{1,6})\s+(.*)$/;
|
|
@@ -284,12 +164,12 @@ function splitOversizeLine(line, opts) {
|
|
|
284
164
|
}
|
|
285
165
|
|
|
286
166
|
// src/compiler/compile.ts
|
|
287
|
-
import { createHash
|
|
288
|
-
import { mkdirSync, rmSync } from "fs";
|
|
167
|
+
import { createHash } from "crypto";
|
|
168
|
+
import { mkdirSync, rmSync, statSync } from "fs";
|
|
289
169
|
import { stat } from "fs/promises";
|
|
290
170
|
import { join } from "path";
|
|
291
171
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
292
|
-
import { KnowledgeCatalogManifestSchema } from "@bendyline/
|
|
172
|
+
import { GEZK_MANIFEST_KIND, KnowledgeCatalogManifestSchema } from "@bendyline/gezk";
|
|
293
173
|
|
|
294
174
|
// src/archive/write.ts
|
|
295
175
|
import { createWriteStream } from "fs";
|
|
@@ -301,9 +181,13 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
301
181
|
const yazl = nodeRequire("yazl");
|
|
302
182
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
303
183
|
const zip = new yazl.ZipFile();
|
|
304
|
-
const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress:
|
|
184
|
+
const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress: false };
|
|
185
|
+
zip.addBuffer(Buffer.from(GEZK_MIME_TYPE, "utf8"), MIMETYPE_PATH, options);
|
|
305
186
|
const sorted = [...entries].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
|
306
187
|
for (const entry of sorted) {
|
|
188
|
+
if (entry.path === MIMETYPE_PATH) {
|
|
189
|
+
throw new Error(`the writer owns the ${MIMETYPE_PATH} entry`);
|
|
190
|
+
}
|
|
307
191
|
if (entry.path.includes("\\") || entry.path.startsWith("/") || entry.path.includes("..")) {
|
|
308
192
|
throw new Error(`unsafe archive path: ${entry.path}`);
|
|
309
193
|
}
|
|
@@ -311,10 +195,10 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
311
195
|
else if (entry.absPath) zip.addFile(entry.absPath, entry.path, options);
|
|
312
196
|
else throw new Error(`archive entry ${entry.path} has neither content nor absPath`);
|
|
313
197
|
}
|
|
314
|
-
await new Promise((
|
|
198
|
+
await new Promise((resolve4, reject) => {
|
|
315
199
|
const out = createWriteStream(outputPath);
|
|
316
200
|
out.on("error", reject);
|
|
317
|
-
out.on("close", () =>
|
|
201
|
+
out.on("close", () => resolve4());
|
|
318
202
|
zip.outputStream.on("error", reject);
|
|
319
203
|
zip.outputStream.pipe(out);
|
|
320
204
|
zip.end();
|
|
@@ -322,27 +206,7 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
322
206
|
}
|
|
323
207
|
|
|
324
208
|
// src/format/file-hash.ts
|
|
325
|
-
import {
|
|
326
|
-
import { createReadStream } from "fs";
|
|
327
|
-
async function hashFileStreaming(path, maxBytes = Number.MAX_SAFE_INTEGER) {
|
|
328
|
-
const hasher = createHash2("sha256");
|
|
329
|
-
let sizeBytes = 0;
|
|
330
|
-
await new Promise((resolve3, reject) => {
|
|
331
|
-
const stream = createReadStream(path, { highWaterMark: 4 * 1024 * 1024 });
|
|
332
|
-
stream.on("data", (chunk) => {
|
|
333
|
-
const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
|
|
334
|
-
sizeBytes += bytes.byteLength;
|
|
335
|
-
if (sizeBytes > maxBytes) {
|
|
336
|
-
stream.destroy(new Error(`file exceeds ${maxBytes} byte limit`));
|
|
337
|
-
return;
|
|
338
|
-
}
|
|
339
|
-
hasher.update(bytes);
|
|
340
|
-
});
|
|
341
|
-
stream.on("error", reject);
|
|
342
|
-
stream.on("end", resolve3);
|
|
343
|
-
});
|
|
344
|
-
return { sha256: hasher.digest("hex"), sizeBytes };
|
|
345
|
-
}
|
|
209
|
+
import { hashFileStreaming } from "@bendyline/gezk/node";
|
|
346
210
|
|
|
347
211
|
// src/format/node-sqlite.ts
|
|
348
212
|
import { createRequire as createRequire2 } from "module";
|
|
@@ -371,25 +235,13 @@ function documentSmokeQueryMisses(db, smoke) {
|
|
|
371
235
|
return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
|
|
372
236
|
}
|
|
373
237
|
|
|
374
|
-
// src/
|
|
238
|
+
// src/toolchain.ts
|
|
375
239
|
import { createRequire as createRequire3 } from "module";
|
|
376
240
|
var nodeRequire3 = createRequire3(import.meta.url);
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
db.loadExtension(sqliteVec.getLoadablePath());
|
|
382
|
-
} catch (err) {
|
|
383
|
-
throw new Error(
|
|
384
|
-
`sqlite-vec extension failed to load \u2014 the compiler cannot build vector tables: ${err instanceof Error ? err.message : String(err)}`
|
|
385
|
-
);
|
|
386
|
-
} finally {
|
|
387
|
-
try {
|
|
388
|
-
db.enableLoadExtension(false);
|
|
389
|
-
} catch {
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
}
|
|
241
|
+
var KNOWLEDGE_TOOLCHAIN = {
|
|
242
|
+
name: "@bendyline/gezel-knowledge",
|
|
243
|
+
version: nodeRequire3("../package.json").version
|
|
244
|
+
};
|
|
393
245
|
|
|
394
246
|
// src/compiler/compile.ts
|
|
395
247
|
async function compileKnowledgeCatalog(opts) {
|
|
@@ -401,9 +253,9 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
401
253
|
const topicIds = new Set(opts.topics.map((t) => t.id));
|
|
402
254
|
const profile = opts.embeddingProfile;
|
|
403
255
|
const chunkerOpts = {
|
|
404
|
-
unit:
|
|
405
|
-
target: opts.chunkingProfile.
|
|
406
|
-
overlap: opts.chunkingProfile.
|
|
256
|
+
unit: opts.chunkingProfile.unit,
|
|
257
|
+
target: opts.chunkingProfile.target,
|
|
258
|
+
overlap: opts.chunkingProfile.overlap,
|
|
407
259
|
countTokens: opts.countTokens
|
|
408
260
|
};
|
|
409
261
|
const prepared = [];
|
|
@@ -463,7 +315,6 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
463
315
|
router.exec("BEGIN");
|
|
464
316
|
if (embedded) {
|
|
465
317
|
router.exec(SHARD_DDL.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
|
|
466
|
-
router.exec(vecChunksDdl(profile.dimensions));
|
|
467
318
|
}
|
|
468
319
|
const commonMeta = {
|
|
469
320
|
format_version: String(GEZK_FORMAT_VERSION),
|
|
@@ -560,7 +411,6 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
560
411
|
if (!db) {
|
|
561
412
|
db = createCatalogDb(join(opts.workDir, shardPath(id)));
|
|
562
413
|
db.exec(SHARD_DDL);
|
|
563
|
-
db.exec(vecChunksDdl(profile.dimensions));
|
|
564
414
|
db.exec("BEGIN");
|
|
565
415
|
shardDbs.set(id, db);
|
|
566
416
|
}
|
|
@@ -571,7 +421,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
571
421
|
let stmts = vecStmts.get(db);
|
|
572
422
|
if (!stmts) {
|
|
573
423
|
stmts = {
|
|
574
|
-
vec: db.prepare("INSERT INTO
|
|
424
|
+
vec: db.prepare("INSERT INTO chunk_vectors_bit (chunk_id, v) VALUES (?, ?)"),
|
|
575
425
|
int8: db.prepare("INSERT INTO chunk_vectors_int8 (chunk_id, v) VALUES (?, ?)")
|
|
576
426
|
};
|
|
577
427
|
vecStmts.set(db, stmts);
|
|
@@ -650,7 +500,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
650
500
|
const header = buildContextHeader(
|
|
651
501
|
p.doc.title,
|
|
652
502
|
chunk.headingPath,
|
|
653
|
-
opts.chunkingProfile.contextHeader.
|
|
503
|
+
opts.chunkingProfile.contextHeader.max,
|
|
654
504
|
opts.countTokens
|
|
655
505
|
);
|
|
656
506
|
pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
|
|
@@ -659,6 +509,30 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
659
509
|
}
|
|
660
510
|
}
|
|
661
511
|
await flushEmbeds();
|
|
512
|
+
for (let id = 0; id < shardCount; id++) {
|
|
513
|
+
if (!embedded)
|
|
514
|
+
putMeta(shardDb(id), {
|
|
515
|
+
shard_id: String(id),
|
|
516
|
+
chunk_count: String(shardChunkCounts.get(id) ?? 0)
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
|
|
520
|
+
const finalDir = join(opts.workDir, "final");
|
|
521
|
+
mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
|
|
522
|
+
const vacuumed = [];
|
|
523
|
+
const vacuumInto = (db, archivePath) => {
|
|
524
|
+
db.exec("COMMIT");
|
|
525
|
+
const dest = join(finalDir, archivePath);
|
|
526
|
+
db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
|
|
527
|
+
vacuumed.push({ archivePath, absPath: dest });
|
|
528
|
+
return dest;
|
|
529
|
+
};
|
|
530
|
+
const shardBytes = /* @__PURE__ */ new Map();
|
|
531
|
+
if (!embedded) {
|
|
532
|
+
for (let id = 0; id < shardCount; id++) {
|
|
533
|
+
shardBytes.set(id, statSync(vacuumInto(shardDb(id), shardPath(id))).size);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
662
536
|
{
|
|
663
537
|
const insertShard = router.prepare(
|
|
664
538
|
"INSERT INTO shards (id, path, chunk_count, document_count, topic_ids_json, centroid_count, bytes) VALUES (?, ?, ?, ?, ?, ?, ?)"
|
|
@@ -681,35 +555,14 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
681
555
|
BigInt(shardDocCounts.get(id) ?? 0),
|
|
682
556
|
JSON.stringify([...shardTopicIds.get(id) ?? []].sort()),
|
|
683
557
|
BigInt(centroids.length),
|
|
684
|
-
BigInt(0)
|
|
685
|
-
// authoritative sizes live in the manifest (post-VACUUM)
|
|
558
|
+
BigInt(shardBytes.get(id) ?? 0)
|
|
686
559
|
);
|
|
687
560
|
for (const c of centroids) {
|
|
688
561
|
insertCentroid.run(BigInt(id), Buffer.from(c.centroid.buffer.slice(0)), BigInt(c.weight));
|
|
689
562
|
}
|
|
690
563
|
}
|
|
691
564
|
}
|
|
692
|
-
for (let id = 0; id < shardCount; id++) {
|
|
693
|
-
if (!embedded)
|
|
694
|
-
putMeta(shardDb(id), {
|
|
695
|
-
shard_id: String(id),
|
|
696
|
-
chunk_count: String(shardChunkCounts.get(id) ?? 0)
|
|
697
|
-
});
|
|
698
|
-
}
|
|
699
|
-
if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
|
|
700
|
-
const finalDir = join(opts.workDir, "final");
|
|
701
|
-
mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
|
|
702
|
-
const vacuumed = [];
|
|
703
|
-
const vacuumInto = (db, archivePath) => {
|
|
704
|
-
db.exec("COMMIT");
|
|
705
|
-
const dest = join(finalDir, archivePath);
|
|
706
|
-
db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
|
|
707
|
-
vacuumed.push({ archivePath, absPath: dest });
|
|
708
|
-
};
|
|
709
565
|
vacuumInto(router, ROUTER_DB_PATH);
|
|
710
|
-
if (!embedded) {
|
|
711
|
-
for (let id = 0; id < shardCount; id++) vacuumInto(shardDb(id), shardPath(id));
|
|
712
|
-
}
|
|
713
566
|
router.close();
|
|
714
567
|
for (const db of shardDbs.values()) db.close();
|
|
715
568
|
let smokeQueries = opts.smokeQueries;
|
|
@@ -765,17 +618,21 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
765
618
|
});
|
|
766
619
|
}
|
|
767
620
|
}
|
|
768
|
-
const extraFiles =
|
|
621
|
+
const extraFiles = {
|
|
622
|
+
[README_PATH]: defaultReadme(opts),
|
|
623
|
+
[LICENSE_NOTICE_PATH]: defaultLicenseNotice(opts),
|
|
624
|
+
...opts.extraFiles ?? {}
|
|
625
|
+
};
|
|
769
626
|
for (const [path, content] of Object.entries(extraFiles)) {
|
|
770
627
|
const bytes = Buffer.from(content, "utf8");
|
|
771
628
|
files.push({
|
|
772
629
|
path,
|
|
773
630
|
sizeBytes: bytes.length,
|
|
774
|
-
sha256:
|
|
631
|
+
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
775
632
|
});
|
|
776
633
|
}
|
|
777
634
|
const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
|
|
778
|
-
kind:
|
|
635
|
+
kind: GEZK_MANIFEST_KIND,
|
|
779
636
|
formatVersion: GEZK_FORMAT_VERSION,
|
|
780
637
|
indexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION,
|
|
781
638
|
id: opts.catalog.id,
|
|
@@ -786,7 +643,10 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
786
643
|
publisher: opts.catalog.publisher,
|
|
787
644
|
createdAt: opts.catalog.createdAt,
|
|
788
645
|
...opts.catalog.sourceSnapshot ? { sourceSnapshot: opts.catalog.sourceSnapshot } : {},
|
|
789
|
-
license:
|
|
646
|
+
license: {
|
|
647
|
+
...opts.catalog.license,
|
|
648
|
+
noticePath: opts.catalog.license.noticePath ?? LICENSE_NOTICE_PATH
|
|
649
|
+
},
|
|
790
650
|
embedding: profile,
|
|
791
651
|
chunking: opts.chunkingProfile,
|
|
792
652
|
topics: opts.topics.map((t) => ({ id: t.id, name: t.name })),
|
|
@@ -797,9 +657,9 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
797
657
|
},
|
|
798
658
|
counts: { documents: prepared.length, chunks: totalChunks, shards: shardCount },
|
|
799
659
|
files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
800
|
-
|
|
660
|
+
requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
|
|
801
661
|
...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
|
|
802
|
-
|
|
662
|
+
toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
|
|
803
663
|
});
|
|
804
664
|
const manifest = opts.finalizeManifest ? KnowledgeCatalogManifestSchema.parse(opts.finalizeManifest(unsignedManifest)) : unsignedManifest;
|
|
805
665
|
const archiveEntries = [
|
|
@@ -829,14 +689,37 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
829
689
|
}
|
|
830
690
|
}
|
|
831
691
|
function createCatalogDb(absPath) {
|
|
832
|
-
const db = new DatabaseSync(absPath
|
|
833
|
-
loadVecExtension(db);
|
|
692
|
+
const db = new DatabaseSync(absPath);
|
|
834
693
|
db.exec("PRAGMA page_size=8192");
|
|
835
694
|
db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
|
|
836
695
|
db.exec(`PRAGMA user_version=${GEZK_INDEX_SCHEMA_VERSION}`);
|
|
837
696
|
db.exec("PRAGMA journal_mode=DELETE");
|
|
838
697
|
return db;
|
|
839
698
|
}
|
|
699
|
+
function defaultReadme(opts) {
|
|
700
|
+
const c = opts.catalog;
|
|
701
|
+
return [
|
|
702
|
+
`# ${c.name}`,
|
|
703
|
+
"",
|
|
704
|
+
...c.description ? [c.description, ""] : [],
|
|
705
|
+
`A gezk knowledge catalog (${c.id}@${c.version}) published by ${c.publisher.name}.`,
|
|
706
|
+
`Language: ${c.language}. License: ${c.license.name} (see ${c.license.noticePath ?? LICENSE_NOTICE_PATH}).`,
|
|
707
|
+
...c.sourceSnapshot ? [`Source snapshot: ${c.sourceSnapshot.name} (${c.sourceSnapshot.date}).`] : [],
|
|
708
|
+
""
|
|
709
|
+
].join("\n");
|
|
710
|
+
}
|
|
711
|
+
function defaultLicenseNotice(opts) {
|
|
712
|
+
const c = opts.catalog;
|
|
713
|
+
const publisher = c.publisher.url ? `${c.publisher.name} <${c.publisher.url}>` : c.publisher.name;
|
|
714
|
+
const license = c.license.spdx ? `${c.license.name} (SPDX: ${c.license.spdx})` : c.license.name;
|
|
715
|
+
return [
|
|
716
|
+
`${c.name} (${c.id}@${c.version})`,
|
|
717
|
+
`Publisher: ${publisher}`,
|
|
718
|
+
`License: ${license}`,
|
|
719
|
+
...c.license.attributionRequired ? ["Attribution is required when redistributing this content."] : [],
|
|
720
|
+
""
|
|
721
|
+
].join("\n");
|
|
722
|
+
}
|
|
840
723
|
function normalizeMarkdown(md) {
|
|
841
724
|
return md.replace(/\r\n/g, "\n").normalize("NFC");
|
|
842
725
|
}
|
|
@@ -854,7 +737,7 @@ ${path.join(" > ")}
|
|
|
854
737
|
return header;
|
|
855
738
|
}
|
|
856
739
|
function seedFor(catalogId, shardId) {
|
|
857
|
-
const digest =
|
|
740
|
+
const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
|
|
858
741
|
return digest.readBigUInt64LE(24);
|
|
859
742
|
}
|
|
860
743
|
function makePrng(seed) {
|
|
@@ -939,14 +822,14 @@ function kMeans(samples, k, seed) {
|
|
|
939
822
|
}
|
|
940
823
|
|
|
941
824
|
// src/archive/read.ts
|
|
942
|
-
import { createHash as
|
|
825
|
+
import { createHash as createHash2 } from "crypto";
|
|
943
826
|
import { createWriteStream as createWriteStream2 } from "fs";
|
|
944
827
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
945
828
|
import { createRequire as createRequire4 } from "module";
|
|
946
829
|
import { dirname as dirname2, isAbsolute, normalize, relative, resolve } from "path";
|
|
947
830
|
import { Transform } from "stream";
|
|
948
831
|
import { pipeline } from "stream/promises";
|
|
949
|
-
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/
|
|
832
|
+
import { GEZK_MANIFEST_KIND as GEZK_MANIFEST_KIND2, KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/gezk";
|
|
950
833
|
var nodeRequire4 = createRequire4(import.meta.url);
|
|
951
834
|
var GEZK_ARCHIVE_LIMITS = {
|
|
952
835
|
maxEntries: 16384,
|
|
@@ -1036,7 +919,7 @@ async function inspectEntries(archivePath) {
|
|
|
1036
919
|
function reconcileEntries(manifest, entries) {
|
|
1037
920
|
const declared = new Map(manifest.files.map((file) => [file.path, file]));
|
|
1038
921
|
for (const path of entries.keys()) {
|
|
1039
|
-
if (path === MANIFEST_PATH) continue;
|
|
922
|
+
if (path === MANIFEST_PATH || path === MIMETYPE_PATH) continue;
|
|
1040
923
|
if (!declared.has(path)) {
|
|
1041
924
|
throw new GezkArchiveError(`archive contains undeclared file: ${path}`, "undeclared-file");
|
|
1042
925
|
}
|
|
@@ -1066,14 +949,88 @@ async function inspectGezkArchive(archivePath) {
|
|
|
1066
949
|
totalUncompressedBytes: inspected.totalUncompressedBytes
|
|
1067
950
|
};
|
|
1068
951
|
}
|
|
952
|
+
function assertMimetypeEntry(entry, position) {
|
|
953
|
+
if (position !== 0) {
|
|
954
|
+
throw new GezkArchiveError("the mimetype entry must be the first archive entry", "mimetype");
|
|
955
|
+
}
|
|
956
|
+
if (entry.compressionMethod !== 0 || entry.uncompressedSize !== GEZK_MIME_TYPE.length) {
|
|
957
|
+
throw new GezkArchiveError("the mimetype entry must be stored, not compressed", "mimetype");
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
function assertSupportedFormat(raw) {
|
|
961
|
+
const kind = raw?.kind;
|
|
962
|
+
const version = raw?.formatVersion;
|
|
963
|
+
if (kind !== GEZK_MANIFEST_KIND2 || version !== GEZK_FORMAT_VERSION) {
|
|
964
|
+
throw new GezkArchiveError(
|
|
965
|
+
`unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${GEZK_FORMAT_VERSION} \u2014 catalogs built for an earlier version must be rebuilt`,
|
|
966
|
+
"format-version"
|
|
967
|
+
);
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
function readEntryText(zip, entry) {
|
|
971
|
+
return new Promise((resolveText, rejectText) => {
|
|
972
|
+
zip.openReadStream(entry, (err, stream) => {
|
|
973
|
+
if (err || !stream) {
|
|
974
|
+
rejectText(new GezkArchiveError(`${entry.fileName} read failed: ${err?.message}`, "io"));
|
|
975
|
+
return;
|
|
976
|
+
}
|
|
977
|
+
const parts = [];
|
|
978
|
+
stream.on("data", (d) => parts.push(d));
|
|
979
|
+
stream.on("end", () => resolveText(Buffer.concat(parts).toString("utf8")));
|
|
980
|
+
stream.on(
|
|
981
|
+
"error",
|
|
982
|
+
(streamErr) => rejectText(
|
|
983
|
+
new GezkArchiveError(`${entry.fileName} read failed: ${streamErr.message}`, "io")
|
|
984
|
+
)
|
|
985
|
+
);
|
|
986
|
+
});
|
|
987
|
+
});
|
|
988
|
+
}
|
|
1069
989
|
async function readGezkManifest(archivePath) {
|
|
1070
990
|
const zip = await openZip(archivePath);
|
|
1071
991
|
return new Promise((resolvePromise, reject) => {
|
|
992
|
+
let position = 0;
|
|
993
|
+
let mimetypeSeen = false;
|
|
994
|
+
const fail = (err) => {
|
|
995
|
+
zip.close();
|
|
996
|
+
reject(
|
|
997
|
+
err instanceof GezkArchiveError ? err : new GezkArchiveError(err instanceof Error ? err.message : String(err), "io")
|
|
998
|
+
);
|
|
999
|
+
};
|
|
1072
1000
|
zip.on("entry", (entry) => {
|
|
1001
|
+
const index = position++;
|
|
1002
|
+
if (entry.fileName === MIMETYPE_PATH) {
|
|
1003
|
+
try {
|
|
1004
|
+
assertMimetypeEntry(entry, index);
|
|
1005
|
+
} catch (err) {
|
|
1006
|
+
fail(err);
|
|
1007
|
+
return;
|
|
1008
|
+
}
|
|
1009
|
+
readEntryText(zip, entry).then((text) => {
|
|
1010
|
+
if (text !== GEZK_MIME_TYPE) {
|
|
1011
|
+
fail(
|
|
1012
|
+
new GezkArchiveError(
|
|
1013
|
+
`not a gezk archive (mimetype ${JSON.stringify(text)})`,
|
|
1014
|
+
"mimetype"
|
|
1015
|
+
)
|
|
1016
|
+
);
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
1019
|
+
mimetypeSeen = true;
|
|
1020
|
+
zip.readEntry();
|
|
1021
|
+
}, fail);
|
|
1022
|
+
return;
|
|
1023
|
+
}
|
|
1073
1024
|
if (entry.fileName !== MANIFEST_PATH) {
|
|
1074
1025
|
zip.readEntry();
|
|
1075
1026
|
return;
|
|
1076
1027
|
}
|
|
1028
|
+
if (!mimetypeSeen) {
|
|
1029
|
+
fail(
|
|
1030
|
+
new GezkArchiveError("archive does not start with the gezk mimetype entry", "mimetype")
|
|
1031
|
+
);
|
|
1032
|
+
return;
|
|
1033
|
+
}
|
|
1077
1034
|
if (entry.uncompressedSize > GEZK_ARCHIVE_LIMITS.maxManifestBytes) {
|
|
1078
1035
|
zip.close();
|
|
1079
1036
|
reject(new GezkArchiveError("manifest exceeds size limit", "limits"));
|
|
@@ -1090,12 +1047,14 @@ async function readGezkManifest(archivePath) {
|
|
|
1090
1047
|
stream.on("end", () => {
|
|
1091
1048
|
zip.close();
|
|
1092
1049
|
try {
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
)
|
|
1097
|
-
);
|
|
1050
|
+
const raw = JSON.parse(Buffer.concat(parts).toString("utf8"));
|
|
1051
|
+
assertSupportedFormat(raw);
|
|
1052
|
+
resolvePromise(KnowledgeCatalogManifestSchema2.parse(raw));
|
|
1098
1053
|
} catch (parseErr) {
|
|
1054
|
+
if (parseErr instanceof GezkArchiveError) {
|
|
1055
|
+
reject(parseErr);
|
|
1056
|
+
return;
|
|
1057
|
+
}
|
|
1099
1058
|
reject(
|
|
1100
1059
|
new GezkArchiveError(
|
|
1101
1060
|
`invalid manifest: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,
|
|
@@ -1153,7 +1112,7 @@ async function extractGezkVerified(archivePath, destDir) {
|
|
|
1153
1112
|
pipeline(stream, createWriteStream2(target)).then(res, rej);
|
|
1154
1113
|
return;
|
|
1155
1114
|
}
|
|
1156
|
-
const hasher =
|
|
1115
|
+
const hasher = createHash2("sha256");
|
|
1157
1116
|
let sizeBytes = 0;
|
|
1158
1117
|
const verify = new Transform({
|
|
1159
1118
|
transform(chunk, _encoding, callback) {
|
|
@@ -1214,9 +1173,7 @@ async function extractGezkVerified(archivePath, destDir) {
|
|
|
1214
1173
|
}
|
|
1215
1174
|
|
|
1216
1175
|
// src/reader/open.ts
|
|
1217
|
-
import { createRequire as createRequire5 } from "module";
|
|
1218
1176
|
import { pathToFileURL } from "url";
|
|
1219
|
-
var nodeRequire5 = createRequire5(import.meta.url);
|
|
1220
1177
|
var CatalogOpenError = class extends Error {
|
|
1221
1178
|
constructor(message, reason) {
|
|
1222
1179
|
super(message);
|
|
@@ -1229,7 +1186,7 @@ function openCatalogDatabase(absPath) {
|
|
|
1229
1186
|
const uri = `${pathToFileURL(absPath).href}?immutable=1`;
|
|
1230
1187
|
let db;
|
|
1231
1188
|
try {
|
|
1232
|
-
db = new DatabaseSync(uri, { readOnly: true
|
|
1189
|
+
db = new DatabaseSync(uri, { readOnly: true });
|
|
1233
1190
|
} catch (err) {
|
|
1234
1191
|
throw new CatalogOpenError(
|
|
1235
1192
|
`cannot open catalog database ${absPath}: ${err instanceof Error ? err.message : String(err)}`,
|
|
@@ -1237,20 +1194,6 @@ function openCatalogDatabase(absPath) {
|
|
|
1237
1194
|
);
|
|
1238
1195
|
}
|
|
1239
1196
|
try {
|
|
1240
|
-
let vecLoaded = false;
|
|
1241
|
-
try {
|
|
1242
|
-
db.enableLoadExtension(true);
|
|
1243
|
-
const sqliteVec = nodeRequire5("sqlite-vec");
|
|
1244
|
-
db.loadExtension(sqliteVec.getLoadablePath());
|
|
1245
|
-
vecLoaded = true;
|
|
1246
|
-
} catch {
|
|
1247
|
-
vecLoaded = false;
|
|
1248
|
-
} finally {
|
|
1249
|
-
try {
|
|
1250
|
-
db.enableLoadExtension(false);
|
|
1251
|
-
} catch {
|
|
1252
|
-
}
|
|
1253
|
-
}
|
|
1254
1197
|
const appId = readPragmaNumber(db, "application_id");
|
|
1255
1198
|
if (appId !== GEZK_APPLICATION_ID) {
|
|
1256
1199
|
throw new CatalogOpenError(
|
|
@@ -1261,13 +1204,10 @@ function openCatalogDatabase(absPath) {
|
|
|
1261
1204
|
const schemaVersion = readPragmaNumber(db, "user_version");
|
|
1262
1205
|
if (schemaVersion !== GEZK_INDEX_SCHEMA_VERSION) {
|
|
1263
1206
|
throw new CatalogOpenError(
|
|
1264
|
-
`unsupported index schema version ${schemaVersion} (reader supports ${GEZK_INDEX_SCHEMA_VERSION})`,
|
|
1207
|
+
`unsupported index schema version ${schemaVersion} (this reader supports ${GEZK_INDEX_SCHEMA_VERSION}; catalogs built for gezk 0.4 and earlier must be rebuilt)`,
|
|
1265
1208
|
"schema-version"
|
|
1266
1209
|
);
|
|
1267
1210
|
}
|
|
1268
|
-
if (!vecLoaded) {
|
|
1269
|
-
throw new CatalogOpenError("sqlite-vec extension failed to load", "vec-unavailable");
|
|
1270
|
-
}
|
|
1271
1211
|
db.exec("PRAGMA mmap_size=536870912");
|
|
1272
1212
|
db.exec("PRAGMA cache_size=-8192");
|
|
1273
1213
|
return {
|
|
@@ -1296,6 +1236,80 @@ function readPragmaNumber(db, pragma) {
|
|
|
1296
1236
|
// src/reader/catalog-handle.ts
|
|
1297
1237
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1298
1238
|
import { brotliDecompressSync } from "zlib";
|
|
1239
|
+
|
|
1240
|
+
// src/reader/bit-scan.ts
|
|
1241
|
+
var POPCOUNT8 = new Uint8Array(256);
|
|
1242
|
+
for (let i = 1; i < 256; i++) POPCOUNT8[i] = (i & 1) + POPCOUNT8[i >> 1];
|
|
1243
|
+
function popcount32(x) {
|
|
1244
|
+
let v = x - (x >>> 1 & 1431655765);
|
|
1245
|
+
v = (v & 858993459) + (v >>> 2 & 858993459);
|
|
1246
|
+
return Math.imul(v + (v >>> 4) & 252645135, 16843009) >>> 24;
|
|
1247
|
+
}
|
|
1248
|
+
function hammingAll(index, query, out) {
|
|
1249
|
+
const { bits, bytesPerRow, rows } = index;
|
|
1250
|
+
if (bytesPerRow % 4 === 0 && bits.byteOffset % 4 === 0) {
|
|
1251
|
+
const perRow = bytesPerRow / 4;
|
|
1252
|
+
const words = new Uint32Array(bits.buffer, bits.byteOffset, rows * perRow);
|
|
1253
|
+
const aligned = new Uint8Array(bytesPerRow);
|
|
1254
|
+
aligned.set(query);
|
|
1255
|
+
const qwords = new Uint32Array(aligned.buffer, 0, perRow);
|
|
1256
|
+
let offset2 = 0;
|
|
1257
|
+
for (let r = 0; r < rows; r++) {
|
|
1258
|
+
let d = 0;
|
|
1259
|
+
for (let w = 0; w < perRow; w++) {
|
|
1260
|
+
d += popcount32((words[offset2 + w] ^ qwords[w]) >>> 0);
|
|
1261
|
+
}
|
|
1262
|
+
out[r] = d;
|
|
1263
|
+
offset2 += perRow;
|
|
1264
|
+
}
|
|
1265
|
+
return;
|
|
1266
|
+
}
|
|
1267
|
+
let offset = 0;
|
|
1268
|
+
for (let r = 0; r < rows; r++) {
|
|
1269
|
+
let d = 0;
|
|
1270
|
+
for (let b = 0; b < bytesPerRow; b++) {
|
|
1271
|
+
d += POPCOUNT8[bits[offset + b] ^ query[b]];
|
|
1272
|
+
}
|
|
1273
|
+
out[r] = d;
|
|
1274
|
+
offset += bytesPerRow;
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
function hammingTopK(index, query, k) {
|
|
1278
|
+
if (query.length !== index.bytesPerRow) {
|
|
1279
|
+
throw new Error(`query has ${query.length} bytes, shard rows have ${index.bytesPerRow}`);
|
|
1280
|
+
}
|
|
1281
|
+
const limit = Math.min(k, index.rows);
|
|
1282
|
+
if (limit <= 0) return [];
|
|
1283
|
+
const distances = new Uint16Array(index.rows);
|
|
1284
|
+
hammingAll(index, query, distances);
|
|
1285
|
+
const maxDistance = index.bytesPerRow * 8;
|
|
1286
|
+
const histogram = new Uint32Array(maxDistance + 1);
|
|
1287
|
+
for (let r = 0; r < index.rows; r++) {
|
|
1288
|
+
const d = distances[r];
|
|
1289
|
+
histogram[d] = histogram[d] + 1;
|
|
1290
|
+
}
|
|
1291
|
+
let threshold = 0;
|
|
1292
|
+
let seen = 0;
|
|
1293
|
+
for (; threshold <= maxDistance; threshold++) {
|
|
1294
|
+
seen += histogram[threshold];
|
|
1295
|
+
if (seen >= limit) break;
|
|
1296
|
+
}
|
|
1297
|
+
let atThreshold = limit - (seen - histogram[threshold]);
|
|
1298
|
+
const hits = [];
|
|
1299
|
+
for (let r = 0; r < index.rows && hits.length < limit; r++) {
|
|
1300
|
+
const d = distances[r];
|
|
1301
|
+
if (d < threshold) hits.push({ chunkId: r + 1, distance: d });
|
|
1302
|
+
else if (d === threshold && atThreshold > 0) {
|
|
1303
|
+
hits.push({ chunkId: r + 1, distance: d });
|
|
1304
|
+
atThreshold--;
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
|
|
1308
|
+
return hits;
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
// src/reader/catalog-handle.ts
|
|
1312
|
+
var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
|
|
1299
1313
|
var CatalogHandle = class _CatalogHandle {
|
|
1300
1314
|
constructor(rootDir, router) {
|
|
1301
1315
|
this.rootDir = rootDir;
|
|
@@ -1310,6 +1324,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1310
1324
|
rootDir;
|
|
1311
1325
|
router;
|
|
1312
1326
|
connections = /* @__PURE__ */ new Map();
|
|
1327
|
+
bitIndexes = /* @__PURE__ */ new Map();
|
|
1313
1328
|
meta;
|
|
1314
1329
|
shards;
|
|
1315
1330
|
/** Open a catalog's extracted version directory (the dir holding manifest.json). */
|
|
@@ -1320,6 +1335,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1320
1335
|
close() {
|
|
1321
1336
|
for (const conn of this.connections.values()) conn.close();
|
|
1322
1337
|
this.connections.clear();
|
|
1338
|
+
this.bitIndexes.clear();
|
|
1323
1339
|
}
|
|
1324
1340
|
/** Integrity check for the router connection, used by deep validation. */
|
|
1325
1341
|
routerQuickCheck() {
|
|
@@ -1350,6 +1366,57 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1350
1366
|
}
|
|
1351
1367
|
return conn.db;
|
|
1352
1368
|
}
|
|
1369
|
+
/**
|
|
1370
|
+
* A shard's sign-bit rows as one contiguous array, loaded on first use and
|
|
1371
|
+
* kept while the resident budget allows. Chunk ids must be dense from 1 —
|
|
1372
|
+
* the row index IS the id — so a gap is a corrupt shard, not a sparse one.
|
|
1373
|
+
*/
|
|
1374
|
+
shardBits(shard) {
|
|
1375
|
+
const cached = this.bitIndexes.get(shard.path);
|
|
1376
|
+
if (cached) return cached;
|
|
1377
|
+
const db = this.shardDb(shard);
|
|
1378
|
+
const span = db.prepare(
|
|
1379
|
+
"SELECT COUNT(*) AS n, MIN(chunk_id) AS lo, MAX(chunk_id) AS hi FROM chunk_vectors_bit"
|
|
1380
|
+
).get();
|
|
1381
|
+
const rows = Number(span.n);
|
|
1382
|
+
if (rows > 0 && (Number(span.lo) !== 1 || Number(span.hi) !== rows)) {
|
|
1383
|
+
throw new CatalogOpenError(`chunk ids are not dense in ${shard.path}`, "corrupt");
|
|
1384
|
+
}
|
|
1385
|
+
const bytesPerRow = Math.ceil(this.dimensions() / 8);
|
|
1386
|
+
const bits = new Uint8Array(rows * bytesPerRow);
|
|
1387
|
+
const rowsIter = db.prepare("SELECT chunk_id, v FROM chunk_vectors_bit ORDER BY chunk_id").iterate();
|
|
1388
|
+
for (const row of rowsIter) {
|
|
1389
|
+
if (row.v.byteLength !== bytesPerRow) {
|
|
1390
|
+
throw new CatalogOpenError(
|
|
1391
|
+
`bit vector width ${row.v.byteLength} != ${bytesPerRow} in ${shard.path}`,
|
|
1392
|
+
"corrupt"
|
|
1393
|
+
);
|
|
1394
|
+
}
|
|
1395
|
+
bits.set(row.v, (Number(row.chunk_id) - 1) * bytesPerRow);
|
|
1396
|
+
}
|
|
1397
|
+
const index = { bits, bytesPerRow, rows };
|
|
1398
|
+
let resident = bits.byteLength;
|
|
1399
|
+
for (const other of this.bitIndexes.values()) resident += other.bits.byteLength;
|
|
1400
|
+
for (const [key, other] of this.bitIndexes) {
|
|
1401
|
+
if (resident <= BIT_INDEX_BUDGET_BYTES) break;
|
|
1402
|
+
this.bitIndexes.delete(key);
|
|
1403
|
+
resident -= other.bits.byteLength;
|
|
1404
|
+
}
|
|
1405
|
+
this.bitIndexes.set(shard.path, index);
|
|
1406
|
+
return index;
|
|
1407
|
+
}
|
|
1408
|
+
/** The embedding dimension from the router's profile echo. */
|
|
1409
|
+
dimensions() {
|
|
1410
|
+
const raw = this.meta.embedding_profile_json;
|
|
1411
|
+
if (raw) {
|
|
1412
|
+
try {
|
|
1413
|
+
const dims = JSON.parse(raw).dimensions;
|
|
1414
|
+
if (typeof dims === "number" && dims > 0) return dims;
|
|
1415
|
+
} catch {
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1418
|
+
throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
|
|
1419
|
+
}
|
|
1353
1420
|
/** Resolve a manifest/router path without allowing it to escape the catalog. */
|
|
1354
1421
|
resolveCatalogPath(path) {
|
|
1355
1422
|
const root = resolve2(this.rootDir);
|
|
@@ -1487,16 +1554,16 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1487
1554
|
* after global cross-catalog routing has already spent the S budget.
|
|
1488
1555
|
*/
|
|
1489
1556
|
searchShards(queryVector, shardIds, finalK) {
|
|
1490
|
-
const queryBits =
|
|
1557
|
+
const queryBits = quantizeBinary(queryVector);
|
|
1491
1558
|
const hits = [];
|
|
1492
1559
|
for (const shardId of shardIds) {
|
|
1493
1560
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
1494
1561
|
if (!shard) continue;
|
|
1495
1562
|
const db = this.shardDb(shard);
|
|
1496
1563
|
const k = rerankK(finalK, shard.chunkCount);
|
|
1497
|
-
const candidates =
|
|
1498
|
-
|
|
1499
|
-
)
|
|
1564
|
+
const candidates = hammingTopK(this.shardBits(shard), queryBits, k).map((hit) => ({
|
|
1565
|
+
chunk_id: hit.chunkId
|
|
1566
|
+
}));
|
|
1500
1567
|
if (candidates.length === 0) continue;
|
|
1501
1568
|
const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
|
|
1502
1569
|
const reranked = candidates.map((c) => {
|
|
@@ -1563,21 +1630,19 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1563
1630
|
return hits;
|
|
1564
1631
|
}
|
|
1565
1632
|
/**
|
|
1566
|
-
* Embedder-free integrity smoke
|
|
1567
|
-
*
|
|
1568
|
-
*
|
|
1633
|
+
* Embedder-free integrity smoke: chunk 1's own bit vector must be its own
|
|
1634
|
+
* nearest neighbor — proves the shard's vectors load and scan. Returns
|
|
1635
|
+
* false instead of throwing (callers report a reason).
|
|
1569
1636
|
*/
|
|
1570
1637
|
selfKnnSmoke(shardId) {
|
|
1571
1638
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
1572
1639
|
if (!shard) return false;
|
|
1573
1640
|
try {
|
|
1574
|
-
const
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
const nearest =
|
|
1578
|
-
|
|
1579
|
-
).get(Buffer.from(first.embedding));
|
|
1580
|
-
return nearest !== void 0 && Number(nearest.chunk_id) === 1;
|
|
1641
|
+
const index = this.shardBits(shard);
|
|
1642
|
+
if (index.rows === 0) return false;
|
|
1643
|
+
const first = index.bits.subarray(0, index.bytesPerRow);
|
|
1644
|
+
const nearest = hammingTopK(index, first, 1)[0];
|
|
1645
|
+
return nearest !== void 0 && nearest.chunkId === 1 && nearest.distance === 0;
|
|
1581
1646
|
} catch {
|
|
1582
1647
|
return false;
|
|
1583
1648
|
}
|
|
@@ -1618,7 +1683,7 @@ var ftsQuery = sanitizeFtsQuery;
|
|
|
1618
1683
|
// src/reader/validate.ts
|
|
1619
1684
|
import { readFile } from "fs/promises";
|
|
1620
1685
|
import { join as join3 } from "path";
|
|
1621
|
-
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/
|
|
1686
|
+
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/gezk";
|
|
1622
1687
|
async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
1623
1688
|
const checks = [];
|
|
1624
1689
|
const check = (name, ok, detail) => {
|
|
@@ -1687,6 +1752,11 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1687
1752
|
);
|
|
1688
1753
|
const topics = handle.topics();
|
|
1689
1754
|
check("toc-present", topics.length >= 1, `${topics.length} topics`);
|
|
1755
|
+
check(
|
|
1756
|
+
"license-notice",
|
|
1757
|
+
manifest.files.some((f) => f.path === manifest.license.noticePath),
|
|
1758
|
+
`manifest.files lacks the declared notice ${manifest.license.noticePath}`
|
|
1759
|
+
);
|
|
1690
1760
|
const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
|
|
1691
1761
|
check(
|
|
1692
1762
|
"counts-documents",
|
|
@@ -1737,7 +1807,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1737
1807
|
db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n
|
|
1738
1808
|
);
|
|
1739
1809
|
const vecCount = Number(
|
|
1740
|
-
db.prepare("SELECT COUNT(*) AS n FROM
|
|
1810
|
+
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit").get().n
|
|
1741
1811
|
);
|
|
1742
1812
|
const int8Count = Number(
|
|
1743
1813
|
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8").get().n
|
|
@@ -1747,6 +1817,24 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1747
1817
|
chunkCount === shard.chunkCount && vecCount === chunkCount && int8Count === chunkCount,
|
|
1748
1818
|
`chunks ${chunkCount}, vec ${vecCount}, int8 ${int8Count}, shards row ${shard.chunkCount}`
|
|
1749
1819
|
);
|
|
1820
|
+
const dims = manifest.embedding.dimensions;
|
|
1821
|
+
const badBit = Number(
|
|
1822
|
+
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit WHERE length(v) != ?").get(Math.ceil(dims / 8)).n
|
|
1823
|
+
);
|
|
1824
|
+
const badInt8 = Number(
|
|
1825
|
+
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8 WHERE length(v) != ?").get(dims).n
|
|
1826
|
+
);
|
|
1827
|
+
check(
|
|
1828
|
+
`vector-widths:${shard.path}`,
|
|
1829
|
+
badBit === 0 && badInt8 === 0,
|
|
1830
|
+
`${badBit} bit rows and ${badInt8} int8 rows have the wrong width for ${dims} dimensions`
|
|
1831
|
+
);
|
|
1832
|
+
const span = db.prepare("SELECT MIN(id) AS lo, MAX(id) AS hi FROM chunks").get();
|
|
1833
|
+
check(
|
|
1834
|
+
`chunk-ids-dense:${shard.path}`,
|
|
1835
|
+
chunkCount === 0 || Number(span.lo) === 1 && Number(span.hi) === chunkCount,
|
|
1836
|
+
`ids span ${String(span.lo)}..${String(span.hi)} for ${chunkCount} chunks`
|
|
1837
|
+
);
|
|
1750
1838
|
} finally {
|
|
1751
1839
|
conn.close();
|
|
1752
1840
|
}
|
|
@@ -1768,56 +1856,642 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1768
1856
|
return report(manifest);
|
|
1769
1857
|
}
|
|
1770
1858
|
|
|
1859
|
+
// src/export/duckdb.ts
|
|
1860
|
+
import { spawn } from "child_process";
|
|
1861
|
+
import { access } from "fs/promises";
|
|
1862
|
+
var DuckdbError = class extends Error {
|
|
1863
|
+
constructor(message, stderr) {
|
|
1864
|
+
super(message);
|
|
1865
|
+
this.stderr = stderr;
|
|
1866
|
+
this.name = "DuckdbError";
|
|
1867
|
+
}
|
|
1868
|
+
stderr;
|
|
1869
|
+
};
|
|
1870
|
+
async function duckdbVersion(binaryPath) {
|
|
1871
|
+
const { stdout } = await runDuckdbProcess(binaryPath, ["--version"], "");
|
|
1872
|
+
const match = /v?(\d+\.\d+\.\d+)/.exec(stdout);
|
|
1873
|
+
if (!match?.[1])
|
|
1874
|
+
throw new DuckdbError(`could not read the DuckDB version from: ${stdout.trim()}`, "");
|
|
1875
|
+
return match[1];
|
|
1876
|
+
}
|
|
1877
|
+
async function assertDuckdbCli(cli) {
|
|
1878
|
+
await access(cli.binaryPath).catch(() => {
|
|
1879
|
+
throw new DuckdbError(`DuckDB CLI not found at ${cli.binaryPath}`, "");
|
|
1880
|
+
});
|
|
1881
|
+
const version = await duckdbVersion(cli.binaryPath);
|
|
1882
|
+
if (cli.expectedVersion && version !== cli.expectedVersion) {
|
|
1883
|
+
throw new DuckdbError(
|
|
1884
|
+
`DuckDB ${cli.expectedVersion} expected, but ${cli.binaryPath} is ${version}; Parquet output is only reproducible against the pinned release`,
|
|
1885
|
+
""
|
|
1886
|
+
);
|
|
1887
|
+
}
|
|
1888
|
+
return version;
|
|
1889
|
+
}
|
|
1890
|
+
async function runDuckdbScript(cli, sql) {
|
|
1891
|
+
const { stdout } = await runDuckdbProcess(cli.binaryPath, ["-batch", ":memory:"], sql);
|
|
1892
|
+
return stdout;
|
|
1893
|
+
}
|
|
1894
|
+
function runDuckdbProcess(binaryPath, args, stdin) {
|
|
1895
|
+
return new Promise((resolve4, reject) => {
|
|
1896
|
+
const child = spawn(binaryPath, args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
|
|
1897
|
+
let stdout = "";
|
|
1898
|
+
let stderr = "";
|
|
1899
|
+
child.stdout.setEncoding("utf8");
|
|
1900
|
+
child.stderr.setEncoding("utf8");
|
|
1901
|
+
child.stdout.on("data", (chunk) => {
|
|
1902
|
+
stdout += chunk;
|
|
1903
|
+
});
|
|
1904
|
+
child.stderr.on("data", (chunk) => {
|
|
1905
|
+
stderr += chunk;
|
|
1906
|
+
});
|
|
1907
|
+
child.on(
|
|
1908
|
+
"error",
|
|
1909
|
+
(err) => reject(new DuckdbError(`could not start DuckDB: ${err.message}`, stderr))
|
|
1910
|
+
);
|
|
1911
|
+
child.on("close", (code) => {
|
|
1912
|
+
if (code === 0) resolve4({ stdout });
|
|
1913
|
+
else
|
|
1914
|
+
reject(
|
|
1915
|
+
new DuckdbError(
|
|
1916
|
+
`DuckDB exited with code ${code}: ${stderr.trim() || stdout.trim()}`,
|
|
1917
|
+
stderr
|
|
1918
|
+
)
|
|
1919
|
+
);
|
|
1920
|
+
});
|
|
1921
|
+
child.stdin.on("error", () => {
|
|
1922
|
+
});
|
|
1923
|
+
child.stdin.end(stdin);
|
|
1924
|
+
});
|
|
1925
|
+
}
|
|
1926
|
+
|
|
1927
|
+
// src/export/parquet.ts
|
|
1928
|
+
import { createHash as createHash3 } from "crypto";
|
|
1929
|
+
import { createWriteStream as createWriteStream3 } from "fs";
|
|
1930
|
+
import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
|
|
1931
|
+
import { join as join4, resolve as resolve3 } from "path";
|
|
1932
|
+
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
1933
|
+
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
|
|
1934
|
+
var PARQUET_EXPORT_VERSION = 1;
|
|
1935
|
+
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
1936
|
+
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
1937
|
+
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
1938
|
+
async function exportCatalogParquet(opts) {
|
|
1939
|
+
const duckdbVersion2 = await assertDuckdbCli(opts.duckdb);
|
|
1940
|
+
const outDir = resolve3(opts.outDir);
|
|
1941
|
+
await mkdir3(outDir, { recursive: true });
|
|
1942
|
+
const work = join4(outDir, `.export-${process.pid}-${Date.now().toString(36)}`);
|
|
1943
|
+
await mkdir3(work, { recursive: true });
|
|
1944
|
+
try {
|
|
1945
|
+
let rootDir;
|
|
1946
|
+
if ("archivePath" in opts.source) {
|
|
1947
|
+
opts.onProgress?.({ phase: "extract" });
|
|
1948
|
+
rootDir = join4(work, "catalog");
|
|
1949
|
+
await extractGezkVerified(opts.source.archivePath, rootDir);
|
|
1950
|
+
} else {
|
|
1951
|
+
rootDir = resolve3(opts.source.rootDir);
|
|
1952
|
+
}
|
|
1953
|
+
const manifest = KnowledgeCatalogManifestSchema4.parse(
|
|
1954
|
+
JSON.parse(await readFile2(join4(rootDir, MANIFEST_PATH), "utf8"))
|
|
1955
|
+
);
|
|
1956
|
+
const staging = join4(work, "ndjson");
|
|
1957
|
+
await mkdir3(staging, { recursive: true });
|
|
1958
|
+
const files = [];
|
|
1959
|
+
const metadata = kvMetadata(manifest);
|
|
1960
|
+
const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
|
|
1961
|
+
const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
|
|
1962
|
+
try {
|
|
1963
|
+
const topicPaths = topicPathsOf(router);
|
|
1964
|
+
const topicRows = await stageTopics(router, join4(staging, "topics.ndjson"));
|
|
1965
|
+
opts.onProgress?.({ phase: "stage", table: "topics", rows: topicRows });
|
|
1966
|
+
files.push(
|
|
1967
|
+
await writeParquet(opts.duckdb, {
|
|
1968
|
+
outDir,
|
|
1969
|
+
file: "topics.parquet",
|
|
1970
|
+
table: "topics",
|
|
1971
|
+
rows: topicRows,
|
|
1972
|
+
sql: topicsCopy(
|
|
1973
|
+
join4(staging, "topics.ndjson"),
|
|
1974
|
+
join4(outDir, "topics.parquet"),
|
|
1975
|
+
rowGroupSize,
|
|
1976
|
+
metadata
|
|
1977
|
+
),
|
|
1978
|
+
onProgress: opts.onProgress
|
|
1979
|
+
})
|
|
1980
|
+
);
|
|
1981
|
+
for (const shard of manifest.router.shards) {
|
|
1982
|
+
const tag = String(shard.id).padStart(3, "0");
|
|
1983
|
+
const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
|
|
1984
|
+
const documentRows = await stageDocuments(router, shard.id, topicPaths, documentsNdjson);
|
|
1985
|
+
opts.onProgress?.({
|
|
1986
|
+
phase: "stage",
|
|
1987
|
+
table: "documents",
|
|
1988
|
+
shardId: shard.id,
|
|
1989
|
+
rows: documentRows
|
|
1990
|
+
});
|
|
1991
|
+
files.push(
|
|
1992
|
+
await writeParquet(opts.duckdb, {
|
|
1993
|
+
outDir,
|
|
1994
|
+
file: `documents-${tag}.parquet`,
|
|
1995
|
+
table: "documents",
|
|
1996
|
+
shardId: shard.id,
|
|
1997
|
+
rows: documentRows,
|
|
1998
|
+
sql: documentsCopy(
|
|
1999
|
+
documentsNdjson,
|
|
2000
|
+
join4(outDir, `documents-${tag}.parquet`),
|
|
2001
|
+
rowGroupSize,
|
|
2002
|
+
metadata
|
|
2003
|
+
),
|
|
2004
|
+
onProgress: opts.onProgress
|
|
2005
|
+
})
|
|
2006
|
+
);
|
|
2007
|
+
const shardDb = openCatalogDatabase(join4(rootDir, shard.path));
|
|
2008
|
+
const chunksNdjson = join4(staging, `chunks-${tag}.ndjson`);
|
|
2009
|
+
let chunkRows;
|
|
2010
|
+
try {
|
|
2011
|
+
chunkRows = await stageChunks(
|
|
2012
|
+
shardDb,
|
|
2013
|
+
shard.id,
|
|
2014
|
+
manifest.embedding.dimensions,
|
|
2015
|
+
chunksNdjson
|
|
2016
|
+
);
|
|
2017
|
+
} finally {
|
|
2018
|
+
shardDb.close();
|
|
2019
|
+
}
|
|
2020
|
+
opts.onProgress?.({ phase: "stage", table: "chunks", shardId: shard.id, rows: chunkRows });
|
|
2021
|
+
files.push(
|
|
2022
|
+
await writeParquet(opts.duckdb, {
|
|
2023
|
+
outDir,
|
|
2024
|
+
file: `chunks-${tag}.parquet`,
|
|
2025
|
+
table: "chunks",
|
|
2026
|
+
shardId: shard.id,
|
|
2027
|
+
rows: chunkRows,
|
|
2028
|
+
sql: chunksCopy(
|
|
2029
|
+
chunksNdjson,
|
|
2030
|
+
join4(outDir, `chunks-${tag}.parquet`),
|
|
2031
|
+
manifest.embedding.dimensions,
|
|
2032
|
+
rowGroupSize,
|
|
2033
|
+
metadata
|
|
2034
|
+
),
|
|
2035
|
+
onProgress: opts.onProgress
|
|
2036
|
+
})
|
|
2037
|
+
);
|
|
2038
|
+
}
|
|
2039
|
+
} finally {
|
|
2040
|
+
router.close();
|
|
2041
|
+
}
|
|
2042
|
+
const report = {
|
|
2043
|
+
exportVersion: PARQUET_EXPORT_VERSION,
|
|
2044
|
+
catalogId: manifest.id,
|
|
2045
|
+
version: manifest.version,
|
|
2046
|
+
publisherId: manifest.publisher.id,
|
|
2047
|
+
formatVersion: manifest.formatVersion,
|
|
2048
|
+
embeddingProfileId: manifest.embedding.id,
|
|
2049
|
+
dimensions: manifest.embedding.dimensions,
|
|
2050
|
+
files,
|
|
2051
|
+
duckdbVersion: duckdbVersion2,
|
|
2052
|
+
exporter: { name: KNOWLEDGE_TOOLCHAIN.name, version: KNOWLEDGE_TOOLCHAIN.version }
|
|
2053
|
+
};
|
|
2054
|
+
const { writeFile } = await import("fs/promises");
|
|
2055
|
+
await writeFile(
|
|
2056
|
+
join4(outDir, PARQUET_REPORT_PATH),
|
|
2057
|
+
`${JSON.stringify(report, null, 2)}
|
|
2058
|
+
`,
|
|
2059
|
+
"utf8"
|
|
2060
|
+
);
|
|
2061
|
+
return report;
|
|
2062
|
+
} finally {
|
|
2063
|
+
await rm(work, { recursive: true, force: true }).catch(() => {
|
|
2064
|
+
});
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
function ndjsonWriter(path) {
|
|
2068
|
+
const stream = createWriteStream3(path, { encoding: "utf8" });
|
|
2069
|
+
return {
|
|
2070
|
+
write(row) {
|
|
2071
|
+
const line = `${JSON.stringify(row)}
|
|
2072
|
+
`;
|
|
2073
|
+
if (stream.write(line)) return Promise.resolve();
|
|
2074
|
+
return new Promise((resolveDrain) => stream.once("drain", () => resolveDrain()));
|
|
2075
|
+
},
|
|
2076
|
+
close() {
|
|
2077
|
+
return new Promise((resolveClose, reject) => {
|
|
2078
|
+
stream.on("error", reject);
|
|
2079
|
+
stream.end(() => resolveClose());
|
|
2080
|
+
});
|
|
2081
|
+
}
|
|
2082
|
+
};
|
|
2083
|
+
}
|
|
2084
|
+
function topicPathsOf(router) {
|
|
2085
|
+
const parents = /* @__PURE__ */ new Map();
|
|
2086
|
+
for (const row of router.db.prepare("SELECT id, parent_id FROM topics").all()) {
|
|
2087
|
+
parents.set(row.id, row.parent_id);
|
|
2088
|
+
}
|
|
2089
|
+
const paths = /* @__PURE__ */ new Map();
|
|
2090
|
+
for (const id of parents.keys()) {
|
|
2091
|
+
const path = [];
|
|
2092
|
+
let cursor = id;
|
|
2093
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2094
|
+
while (cursor && !seen.has(cursor)) {
|
|
2095
|
+
seen.add(cursor);
|
|
2096
|
+
path.unshift(cursor);
|
|
2097
|
+
cursor = parents.get(cursor) ?? null;
|
|
2098
|
+
}
|
|
2099
|
+
paths.set(id, path);
|
|
2100
|
+
}
|
|
2101
|
+
return paths;
|
|
2102
|
+
}
|
|
2103
|
+
async function stageTopics(router, path) {
|
|
2104
|
+
const out = ndjsonWriter(path);
|
|
2105
|
+
let rows = 0;
|
|
2106
|
+
const statement = router.db.prepare(
|
|
2107
|
+
"SELECT id, parent_id, name, description, sort_key, document_count FROM topics ORDER BY sort_key, id"
|
|
2108
|
+
);
|
|
2109
|
+
for (const row of statement.iterate()) {
|
|
2110
|
+
await out.write({
|
|
2111
|
+
id: row.id,
|
|
2112
|
+
parent_id: row.parent_id ?? null,
|
|
2113
|
+
name: row.name,
|
|
2114
|
+
description: row.description ?? null,
|
|
2115
|
+
sort_key: row.sort_key,
|
|
2116
|
+
document_count: Number(row.document_count)
|
|
2117
|
+
});
|
|
2118
|
+
rows++;
|
|
2119
|
+
}
|
|
2120
|
+
await out.close();
|
|
2121
|
+
return rows;
|
|
2122
|
+
}
|
|
2123
|
+
async function stageDocuments(router, shardId, topicPaths, path) {
|
|
2124
|
+
const out = ndjsonWriter(path);
|
|
2125
|
+
let rows = 0;
|
|
2126
|
+
const aliasesOf = router.db.prepare(
|
|
2127
|
+
"SELECT alias FROM aliases WHERE document_id = ? ORDER BY alias"
|
|
2128
|
+
);
|
|
2129
|
+
const statement = router.db.prepare(
|
|
2130
|
+
`SELECT id, title, slug, summary, language, topic_id, shard_id, chunk_count,
|
|
2131
|
+
source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob
|
|
2132
|
+
FROM documents WHERE shard_id = ? ORDER BY id`
|
|
2133
|
+
);
|
|
2134
|
+
for (const row of statement.iterate(shardId)) {
|
|
2135
|
+
const aliases = aliasesOf.all(row.id).map(
|
|
2136
|
+
(a) => a.alias
|
|
2137
|
+
);
|
|
2138
|
+
await out.write({
|
|
2139
|
+
id: row.id,
|
|
2140
|
+
title: row.title,
|
|
2141
|
+
slug: row.slug,
|
|
2142
|
+
summary: row.summary ?? null,
|
|
2143
|
+
language: row.language,
|
|
2144
|
+
topic_id: row.topic_id,
|
|
2145
|
+
topic_path: topicPaths.get(row.topic_id) ?? [row.topic_id],
|
|
2146
|
+
shard_id: Number(row.shard_id),
|
|
2147
|
+
chunk_count: Number(row.chunk_count),
|
|
2148
|
+
markdown: decodeBody(row.body_codec, row.body_blob),
|
|
2149
|
+
source_url: row.source_url ?? null,
|
|
2150
|
+
source_revision: row.source_revision ?? null,
|
|
2151
|
+
source_updated_at: row.source_updated_at ?? null,
|
|
2152
|
+
attribution: row.attribution_json ?? null,
|
|
2153
|
+
aliases
|
|
2154
|
+
});
|
|
2155
|
+
rows++;
|
|
2156
|
+
}
|
|
2157
|
+
await out.close();
|
|
2158
|
+
return rows;
|
|
2159
|
+
}
|
|
2160
|
+
function decodeBody(codec, blob) {
|
|
2161
|
+
const bytes = Buffer.from(blob);
|
|
2162
|
+
if (codec === "br") {
|
|
2163
|
+
return brotliDecompressSync2(bytes, { maxOutputLength: MAX_KNOWLEDGE_DOCUMENT_BYTES }).toString(
|
|
2164
|
+
"utf8"
|
|
2165
|
+
);
|
|
2166
|
+
}
|
|
2167
|
+
if (codec === "none") return bytes.toString("utf8");
|
|
2168
|
+
throw new Error(`unknown body codec ${codec}`);
|
|
2169
|
+
}
|
|
2170
|
+
async function stageChunks(shard, shardId, dimensions, path) {
|
|
2171
|
+
const out = ndjsonWriter(path);
|
|
2172
|
+
const bitBytes = Math.ceil(dimensions / 8);
|
|
2173
|
+
let rows = 0;
|
|
2174
|
+
const statement = shard.db.prepare(
|
|
2175
|
+
`SELECT c.id, c.chunk_uid, c.document_id, c.ordinal, c.title, c.heading_path, c.heading_text,
|
|
2176
|
+
c.text, c.token_count, c.content_hash, c.line_start, c.line_end,
|
|
2177
|
+
i.v AS int8, b.v AS bits
|
|
2178
|
+
FROM chunks c
|
|
2179
|
+
JOIN chunk_vectors_int8 i ON i.chunk_id = c.id
|
|
2180
|
+
JOIN chunk_vectors_bit b ON b.chunk_id = c.id
|
|
2181
|
+
ORDER BY c.id`
|
|
2182
|
+
);
|
|
2183
|
+
for (const row of statement.iterate()) {
|
|
2184
|
+
const int8 = Buffer.from(row.int8);
|
|
2185
|
+
const bits = Buffer.from(row.bits);
|
|
2186
|
+
if (int8.length !== dimensions || bits.length !== bitBytes) {
|
|
2187
|
+
throw new Error(
|
|
2188
|
+
`chunk ${row.chunk_uid}: vector widths ${int8.length}/${bits.length} do not match the ${dimensions}-dimension profile`
|
|
2189
|
+
);
|
|
2190
|
+
}
|
|
2191
|
+
await out.write({
|
|
2192
|
+
id: Number(row.id),
|
|
2193
|
+
chunk_uid: row.chunk_uid,
|
|
2194
|
+
document_id: row.document_id,
|
|
2195
|
+
ordinal: Number(row.ordinal),
|
|
2196
|
+
title: row.title,
|
|
2197
|
+
heading_path: parseJsonStrings(row.heading_path),
|
|
2198
|
+
heading_text: row.heading_text,
|
|
2199
|
+
text: row.text,
|
|
2200
|
+
token_count: Number(row.token_count),
|
|
2201
|
+
content_hash: row.content_hash,
|
|
2202
|
+
line_start: Number(row.line_start),
|
|
2203
|
+
line_end: Number(row.line_end),
|
|
2204
|
+
shard_id: shardId,
|
|
2205
|
+
embedding: Array.from(new Int8Array(int8.buffer, int8.byteOffset, int8.length)),
|
|
2206
|
+
embedding_bit: bits.toString("base64")
|
|
2207
|
+
});
|
|
2208
|
+
rows++;
|
|
2209
|
+
}
|
|
2210
|
+
await out.close();
|
|
2211
|
+
return rows;
|
|
2212
|
+
}
|
|
2213
|
+
function parseJsonStrings(raw) {
|
|
2214
|
+
try {
|
|
2215
|
+
const parsed = JSON.parse(raw);
|
|
2216
|
+
return Array.isArray(parsed) ? parsed.filter((p) => typeof p === "string") : [];
|
|
2217
|
+
} catch {
|
|
2218
|
+
return [];
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
function sqlString(value) {
|
|
2222
|
+
return `'${value.replace(/'/g, "''")}'`;
|
|
2223
|
+
}
|
|
2224
|
+
function kvMetadata(manifest) {
|
|
2225
|
+
return {
|
|
2226
|
+
gezk_catalog_id: manifest.id,
|
|
2227
|
+
gezk_catalog_version: manifest.version,
|
|
2228
|
+
gezk_publisher_id: manifest.publisher.id,
|
|
2229
|
+
gezk_format_version: manifest.formatVersion,
|
|
2230
|
+
gezk_embedding_profile: manifest.embedding.id,
|
|
2231
|
+
gezk_embedding_model: manifest.embedding.model.repo,
|
|
2232
|
+
gezk_embedding_dimensions: String(manifest.embedding.dimensions),
|
|
2233
|
+
gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
|
|
2234
|
+
gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
|
|
2235
|
+
gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
|
|
2236
|
+
gezk_license: manifest.license.name
|
|
2237
|
+
};
|
|
2238
|
+
}
|
|
2239
|
+
function copyOptions(rowGroupSize, metadata) {
|
|
2240
|
+
const kv = Object.entries(metadata).map(([key, value]) => `${key}: ${sqlString(value)}`).join(", ");
|
|
2241
|
+
return `(FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 3, ROW_GROUP_SIZE ${rowGroupSize}, KV_METADATA {${kv}})`;
|
|
2242
|
+
}
|
|
2243
|
+
var PRELUDE = "SET threads = 1;\nSET preserve_insertion_order = true;\n";
|
|
2244
|
+
function readNdjson(path, columns) {
|
|
2245
|
+
const spec = Object.entries(columns).map(([name, type]) => `${name}: ${sqlString(type)}`).join(", ");
|
|
2246
|
+
return `read_ndjson(${sqlString(path)}, columns = {${spec}}, maximum_object_size = ${NDJSON_MAX_OBJECT_BYTES})`;
|
|
2247
|
+
}
|
|
2248
|
+
function topicsCopy(ndjson, out, rowGroupSize, metadata) {
|
|
2249
|
+
const source = readNdjson(ndjson, {
|
|
2250
|
+
id: "VARCHAR",
|
|
2251
|
+
parent_id: "VARCHAR",
|
|
2252
|
+
name: "VARCHAR",
|
|
2253
|
+
description: "VARCHAR",
|
|
2254
|
+
sort_key: "VARCHAR",
|
|
2255
|
+
document_count: "BIGINT"
|
|
2256
|
+
});
|
|
2257
|
+
return `${PRELUDE}COPY (SELECT id, parent_id, name, description, sort_key, document_count FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2258
|
+
`;
|
|
2259
|
+
}
|
|
2260
|
+
function documentsCopy(ndjson, out, rowGroupSize, metadata) {
|
|
2261
|
+
const source = readNdjson(ndjson, {
|
|
2262
|
+
id: "VARCHAR",
|
|
2263
|
+
title: "VARCHAR",
|
|
2264
|
+
slug: "VARCHAR",
|
|
2265
|
+
summary: "VARCHAR",
|
|
2266
|
+
language: "VARCHAR",
|
|
2267
|
+
topic_id: "VARCHAR",
|
|
2268
|
+
topic_path: "VARCHAR[]",
|
|
2269
|
+
shard_id: "INTEGER",
|
|
2270
|
+
chunk_count: "INTEGER",
|
|
2271
|
+
markdown: "VARCHAR",
|
|
2272
|
+
source_url: "VARCHAR",
|
|
2273
|
+
source_revision: "VARCHAR",
|
|
2274
|
+
source_updated_at: "VARCHAR",
|
|
2275
|
+
attribution: "VARCHAR",
|
|
2276
|
+
aliases: "VARCHAR[]"
|
|
2277
|
+
});
|
|
2278
|
+
return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2279
|
+
`;
|
|
2280
|
+
}
|
|
2281
|
+
function chunksCopy(ndjson, out, dimensions, rowGroupSize, metadata) {
|
|
2282
|
+
const source = readNdjson(ndjson, {
|
|
2283
|
+
id: "BIGINT",
|
|
2284
|
+
chunk_uid: "VARCHAR",
|
|
2285
|
+
document_id: "VARCHAR",
|
|
2286
|
+
ordinal: "INTEGER",
|
|
2287
|
+
title: "VARCHAR",
|
|
2288
|
+
heading_path: "VARCHAR[]",
|
|
2289
|
+
heading_text: "VARCHAR",
|
|
2290
|
+
text: "VARCHAR",
|
|
2291
|
+
token_count: "INTEGER",
|
|
2292
|
+
content_hash: "VARCHAR",
|
|
2293
|
+
line_start: "INTEGER",
|
|
2294
|
+
line_end: "INTEGER",
|
|
2295
|
+
shard_id: "INTEGER",
|
|
2296
|
+
embedding: "TINYINT[]",
|
|
2297
|
+
embedding_bit: "VARCHAR"
|
|
2298
|
+
});
|
|
2299
|
+
return `${PRELUDE}COPY (SELECT id, chunk_uid, document_id, ordinal, title, heading_path, heading_text, text, token_count, content_hash, line_start, line_end, shard_id, embedding::TINYINT[${dimensions}] AS embedding, from_base64(embedding_bit) AS embedding_bit FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2300
|
+
`;
|
|
2301
|
+
}
|
|
2302
|
+
async function writeParquet(cli, spec) {
|
|
2303
|
+
spec.onProgress?.({ phase: "write", file: spec.file });
|
|
2304
|
+
const target = join4(spec.outDir, spec.file);
|
|
2305
|
+
await rm(target, { force: true });
|
|
2306
|
+
await runDuckdbScript(cli, spec.sql);
|
|
2307
|
+
const bytes = await readFile2(target);
|
|
2308
|
+
return {
|
|
2309
|
+
path: spec.file,
|
|
2310
|
+
table: spec.table,
|
|
2311
|
+
...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
|
|
2312
|
+
rows: spec.rows,
|
|
2313
|
+
sizeBytes: (await stat2(target)).size,
|
|
2314
|
+
sha256: createHash3("sha256").update(bytes).digest("hex")
|
|
2315
|
+
};
|
|
2316
|
+
}
|
|
2317
|
+
async function findDuckdbBinary(env = process.env) {
|
|
2318
|
+
const configured = env.GEZEL_DUCKDB_BIN?.trim();
|
|
2319
|
+
if (configured) return configured;
|
|
2320
|
+
const { readdir: readdir2 } = await import("fs/promises");
|
|
2321
|
+
const { homedir } = await import("os");
|
|
2322
|
+
const name = process.platform === "win32" ? "duckdb.exe" : "duckdb";
|
|
2323
|
+
const homes = [env.GEZEL_HOME, join4(homedir(), ".gezel-dev"), join4(homedir(), ".gezel")].filter(
|
|
2324
|
+
(h) => Boolean(h)
|
|
2325
|
+
);
|
|
2326
|
+
for (const home of homes) {
|
|
2327
|
+
const engines = join4(home, "engines", "duckdb");
|
|
2328
|
+
const versions = (await readdir2(engines).catch(() => [])).sort().reverse();
|
|
2329
|
+
for (const version of versions) {
|
|
2330
|
+
const candidate = join4(engines, version, name);
|
|
2331
|
+
if (await stat2(candidate).catch(() => null)) return candidate;
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
const pathDirs = (env.PATH ?? "").split(process.platform === "win32" ? ";" : ":");
|
|
2335
|
+
for (const dir of pathDirs) {
|
|
2336
|
+
if (!dir) continue;
|
|
2337
|
+
const candidate = join4(dir, name);
|
|
2338
|
+
if (await stat2(candidate).catch(() => null)) return candidate;
|
|
2339
|
+
}
|
|
2340
|
+
return null;
|
|
2341
|
+
}
|
|
2342
|
+
|
|
1771
2343
|
// src/profiles/registry.ts
|
|
1772
2344
|
var SHARED_ENCODING = {
|
|
1773
2345
|
pooling: "mean",
|
|
1774
2346
|
normalized: true,
|
|
1775
2347
|
dimensions: 384,
|
|
1776
2348
|
maxTokens: 512,
|
|
1777
|
-
vectorEncoding: "
|
|
2349
|
+
vectorEncoding: "bit+int8",
|
|
1778
2350
|
distance: { stage1: "hamming", stage2: "cosine" },
|
|
1779
2351
|
quantization: {
|
|
1780
2352
|
int8: { method: "symmetric-linear", scale: 127 },
|
|
1781
2353
|
binary: { method: "sign", threshold: 0, packing: "lsb-first" }
|
|
1782
2354
|
}
|
|
1783
2355
|
};
|
|
1784
|
-
var
|
|
1785
|
-
id: "
|
|
2356
|
+
var MULTILINGUAL_E5_SMALL_1 = {
|
|
2357
|
+
id: "multilingual-e5-small@1",
|
|
1786
2358
|
model: {
|
|
1787
2359
|
repo: "Xenova/multilingual-e5-small",
|
|
1788
|
-
revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78"
|
|
2360
|
+
revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
|
|
2361
|
+
onnxFile: "onnx/model.onnx",
|
|
2362
|
+
onnxDigest: "sha256:4aa845c27760e06e9a686b9d8b5d440eae4b6612cd09e5b522b716d3941f77ff"
|
|
2363
|
+
},
|
|
2364
|
+
tokenizer: {
|
|
2365
|
+
kind: "sentencepiece-xlmr",
|
|
2366
|
+
file: "tokenizer.json",
|
|
2367
|
+
digest: "sha256:0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39"
|
|
1789
2368
|
},
|
|
1790
|
-
tokenizer: { kind: "sentencepiece-xlmr" },
|
|
1791
2369
|
queryInstruction: "query: ",
|
|
1792
2370
|
passageInstruction: "passage: ",
|
|
1793
2371
|
...SHARED_ENCODING
|
|
1794
2372
|
};
|
|
1795
|
-
var
|
|
1796
|
-
id: "
|
|
2373
|
+
var BGE_SMALL_EN_V15_1 = {
|
|
2374
|
+
id: "bge-small-en-v1.5@1",
|
|
1797
2375
|
model: {
|
|
1798
2376
|
repo: "Xenova/bge-small-en-v1.5",
|
|
1799
|
-
revision: "ea104dacec62c0de699686887e3f920caeb4f3e3"
|
|
2377
|
+
revision: "ea104dacec62c0de699686887e3f920caeb4f3e3",
|
|
2378
|
+
onnxFile: "onnx/model.onnx",
|
|
2379
|
+
onnxDigest: "sha256:828e1496d7fabb79cfa4dcd84fa38625c0d3d21da474a00f08db0f559940cf35"
|
|
2380
|
+
},
|
|
2381
|
+
tokenizer: {
|
|
2382
|
+
kind: "wordpiece-bert",
|
|
2383
|
+
file: "tokenizer.json",
|
|
2384
|
+
digest: "sha256:d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66"
|
|
1800
2385
|
},
|
|
1801
|
-
tokenizer: { kind: "wordpiece-bert" },
|
|
1802
2386
|
queryInstruction: "Represent this sentence for searching relevant passages: ",
|
|
1803
2387
|
passageInstruction: "",
|
|
1804
2388
|
...SHARED_ENCODING
|
|
1805
2389
|
};
|
|
1806
2390
|
var KNOWLEDGE_EMBEDDING_PROFILES = [
|
|
1807
|
-
|
|
1808
|
-
|
|
2391
|
+
MULTILINGUAL_E5_SMALL_1,
|
|
2392
|
+
BGE_SMALL_EN_V15_1
|
|
1809
2393
|
];
|
|
1810
2394
|
function knowledgeEmbeddingProfile(id) {
|
|
1811
2395
|
return KNOWLEDGE_EMBEDDING_PROFILES.find((p) => p.id === id) ?? null;
|
|
1812
2396
|
}
|
|
1813
|
-
var
|
|
1814
|
-
id: "
|
|
2397
|
+
var MARKDOWN_CHUNKS_2 = {
|
|
2398
|
+
id: "markdown-chunks@2",
|
|
1815
2399
|
unit: "tokens",
|
|
1816
2400
|
tokenizer: "profile",
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
contextHeader: {
|
|
2401
|
+
target: 420,
|
|
2402
|
+
overlap: 64,
|
|
2403
|
+
contextHeader: { max: 64 }
|
|
1820
2404
|
};
|
|
2405
|
+
var GEZEL_MULTILINGUAL_E5_SMALL_1 = MULTILINGUAL_E5_SMALL_1;
|
|
2406
|
+
var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
|
|
2407
|
+
var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
|
|
2408
|
+
|
|
2409
|
+
// src/embedding/profile-embedder.ts
|
|
2410
|
+
import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
|
|
2411
|
+
|
|
2412
|
+
// src/embedding/artifact-verify.ts
|
|
2413
|
+
import { createHash as createHash4 } from "crypto";
|
|
2414
|
+
import { createReadStream } from "fs";
|
|
2415
|
+
import { stat as stat3 } from "fs/promises";
|
|
2416
|
+
import { join as join5 } from "path";
|
|
2417
|
+
import { embeddingProfileArtifacts } from "@bendyline/gezk";
|
|
2418
|
+
var EmbedderArtifactError = class extends Error {
|
|
2419
|
+
constructor(reason, profileId, checks) {
|
|
2420
|
+
super(describeFailure(reason, profileId, checks));
|
|
2421
|
+
this.reason = reason;
|
|
2422
|
+
this.profileId = profileId;
|
|
2423
|
+
this.checks = checks;
|
|
2424
|
+
this.name = "EmbedderArtifactError";
|
|
2425
|
+
}
|
|
2426
|
+
reason;
|
|
2427
|
+
profileId;
|
|
2428
|
+
checks;
|
|
2429
|
+
isActionable = true;
|
|
2430
|
+
};
|
|
2431
|
+
function describeFailure(reason, profileId, checks) {
|
|
2432
|
+
const detail = checks.filter((c) => c.actual !== c.expected).map(
|
|
2433
|
+
(c) => c.actual === null ? `${c.file} not found at ${c.path}` : `${c.file} is ${c.actual}, profile pins ${c.expected}`
|
|
2434
|
+
).join("; ");
|
|
2435
|
+
return reason === "missing" ? `cannot verify the model files for profile ${profileId}: ${detail}` : `the model files loaded for profile ${profileId} differ from the ones it pins: ${detail}`;
|
|
2436
|
+
}
|
|
2437
|
+
function transformersCachePath(cacheDir, repo, revision, file) {
|
|
2438
|
+
return revision === "main" ? join5(cacheDir, repo, file) : join5(cacheDir, repo, revision, file);
|
|
2439
|
+
}
|
|
2440
|
+
var digestMemo = /* @__PURE__ */ new Map();
|
|
2441
|
+
var DIGEST_MEMO_MAX = 32;
|
|
2442
|
+
async function fileDigest(path) {
|
|
2443
|
+
let size;
|
|
2444
|
+
let mtimeMs;
|
|
2445
|
+
try {
|
|
2446
|
+
const info = await stat3(path);
|
|
2447
|
+
if (!info.isFile()) return null;
|
|
2448
|
+
size = info.size;
|
|
2449
|
+
mtimeMs = info.mtimeMs;
|
|
2450
|
+
} catch {
|
|
2451
|
+
return null;
|
|
2452
|
+
}
|
|
2453
|
+
const key = `${path} ${size} ${mtimeMs}`;
|
|
2454
|
+
const memo = digestMemo.get(key);
|
|
2455
|
+
if (memo) return memo;
|
|
2456
|
+
const digest = await new Promise((resolve4, reject) => {
|
|
2457
|
+
const hash = createHash4("sha256");
|
|
2458
|
+
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve4(`sha256:${hash.digest("hex")}`));
|
|
2459
|
+
});
|
|
2460
|
+
if (digestMemo.size >= DIGEST_MEMO_MAX) {
|
|
2461
|
+
const oldest = digestMemo.keys().next().value;
|
|
2462
|
+
if (oldest !== void 0) digestMemo.delete(oldest);
|
|
2463
|
+
}
|
|
2464
|
+
digestMemo.set(key, digest);
|
|
2465
|
+
return digest;
|
|
2466
|
+
}
|
|
2467
|
+
async function verifyProfileArtifacts(profile, opts) {
|
|
2468
|
+
const artifacts = embeddingProfileArtifacts(profile);
|
|
2469
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
2470
|
+
const pinned = [];
|
|
2471
|
+
if (artifacts.onnxDigest) {
|
|
2472
|
+
pinned.push({ role: "onnx", file: artifacts.onnxFile, expected: artifacts.onnxDigest });
|
|
2473
|
+
}
|
|
2474
|
+
if (artifacts.tokenizerDigest) {
|
|
2475
|
+
pinned.push({
|
|
2476
|
+
role: "tokenizer",
|
|
2477
|
+
file: artifacts.tokenizerFile,
|
|
2478
|
+
expected: artifacts.tokenizerDigest
|
|
2479
|
+
});
|
|
2480
|
+
}
|
|
2481
|
+
if (pinned.length === 0) return { status: "unpinned", checks: [] };
|
|
2482
|
+
const checks = [];
|
|
2483
|
+
for (const { role, file, expected } of pinned) {
|
|
2484
|
+
const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file);
|
|
2485
|
+
checks.push({ role, file, path, expected, actual: await fileDigest(path) });
|
|
2486
|
+
}
|
|
2487
|
+
if (checks.some((c) => c.actual === null)) {
|
|
2488
|
+
throw new EmbedderArtifactError("missing", profile.id, checks);
|
|
2489
|
+
}
|
|
2490
|
+
if (checks.some((c) => c.actual !== c.expected)) {
|
|
2491
|
+
throw new EmbedderArtifactError("mismatch", profile.id, checks);
|
|
2492
|
+
}
|
|
2493
|
+
return { status: "verified", checks };
|
|
2494
|
+
}
|
|
1821
2495
|
|
|
1822
2496
|
// src/embedding/profile-embedder.ts
|
|
1823
2497
|
var MAX_BATCH = 8;
|
|
@@ -1829,6 +2503,42 @@ var EmbedderUnavailableError = class extends Error {
|
|
|
1829
2503
|
this.name = "EmbedderUnavailableError";
|
|
1830
2504
|
}
|
|
1831
2505
|
};
|
|
2506
|
+
var ONNX_DTYPE_BY_SUFFIX = [
|
|
2507
|
+
["_q4f16", "q4f16"],
|
|
2508
|
+
["_quantized", "q8"],
|
|
2509
|
+
["_uint8", "uint8"],
|
|
2510
|
+
["_int8", "int8"],
|
|
2511
|
+
["_bnb4", "bnb4"],
|
|
2512
|
+
["_fp16", "fp16"],
|
|
2513
|
+
["_q4", "q4"]
|
|
2514
|
+
];
|
|
2515
|
+
var POOLING_BY_PROFILE = {
|
|
2516
|
+
mean: "mean",
|
|
2517
|
+
cls: "cls"
|
|
2518
|
+
};
|
|
2519
|
+
function resolveTransformersModelOptions(profile) {
|
|
2520
|
+
const { onnxFile, tokenizerFile } = embeddingProfileArtifacts2(profile);
|
|
2521
|
+
if (tokenizerFile !== "tokenizer.json") {
|
|
2522
|
+
throw new EmbedderUnavailableError(
|
|
2523
|
+
`profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
|
|
2524
|
+
);
|
|
2525
|
+
}
|
|
2526
|
+
if (!onnxFile.endsWith(".onnx")) {
|
|
2527
|
+
throw new EmbedderUnavailableError(
|
|
2528
|
+
`profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
|
|
2529
|
+
);
|
|
2530
|
+
}
|
|
2531
|
+
const slash = onnxFile.lastIndexOf("/");
|
|
2532
|
+
const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
|
|
2533
|
+
const base = onnxFile.slice(slash + 1, -".onnx".length);
|
|
2534
|
+
const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
|
|
2535
|
+
const [suffix, dtype] = variant ?? ["", "fp32"];
|
|
2536
|
+
const modelFileName = base.slice(0, base.length - suffix.length);
|
|
2537
|
+
if (modelFileName === "") {
|
|
2538
|
+
throw new EmbedderUnavailableError(`profile ${profile.id} pins an unnamed graph ${onnxFile}`);
|
|
2539
|
+
}
|
|
2540
|
+
return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
|
|
2541
|
+
}
|
|
1832
2542
|
async function loadTransformers() {
|
|
1833
2543
|
try {
|
|
1834
2544
|
const specifier = "@huggingface/transformers";
|
|
@@ -1844,33 +2554,52 @@ async function loadTransformers() {
|
|
|
1844
2554
|
}
|
|
1845
2555
|
}
|
|
1846
2556
|
async function createProfileEmbedder(profile, opts = {}) {
|
|
1847
|
-
const
|
|
2557
|
+
const pooling = POOLING_BY_PROFILE[profile.pooling];
|
|
2558
|
+
if (!pooling) {
|
|
2559
|
+
throw new EmbedderUnavailableError(
|
|
2560
|
+
`profile ${profile.id} uses ${profile.pooling} pooling, which this runtime does not implement`
|
|
2561
|
+
);
|
|
2562
|
+
}
|
|
2563
|
+
const modelOptions = resolveTransformersModelOptions(profile);
|
|
2564
|
+
const transformers = opts.transformers ?? await loadTransformers();
|
|
1848
2565
|
const cacheDir = opts.cacheDir ?? process.env.GEZEL_HF_CACHE_DIR;
|
|
1849
2566
|
if (cacheDir) {
|
|
1850
2567
|
transformers.env.cacheDir = cacheDir;
|
|
1851
2568
|
transformers.env.useFSCache = true;
|
|
1852
2569
|
transformers.env.allowRemoteModels = true;
|
|
1853
2570
|
}
|
|
1854
|
-
const modelOpts = {
|
|
1855
|
-
revision: profile.model.revision,
|
|
1856
|
-
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
|
|
1857
|
-
};
|
|
1858
2571
|
const [pipe, tokenizer] = await Promise.all([
|
|
1859
|
-
transformers.pipeline("feature-extraction", profile.model.repo,
|
|
1860
|
-
|
|
2572
|
+
transformers.pipeline("feature-extraction", profile.model.repo, {
|
|
2573
|
+
...modelOptions,
|
|
2574
|
+
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
|
|
2575
|
+
}),
|
|
2576
|
+
transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
|
|
2577
|
+
revision: modelOptions.revision
|
|
2578
|
+
})
|
|
1861
2579
|
]);
|
|
2580
|
+
let verification;
|
|
2581
|
+
try {
|
|
2582
|
+
verification = await verifyProfileArtifacts(profile, {
|
|
2583
|
+
cacheDir: cacheDir ?? transformers.env.cacheDir ?? ""
|
|
2584
|
+
});
|
|
2585
|
+
} catch (err) {
|
|
2586
|
+
await pipe.dispose?.().catch(() => {
|
|
2587
|
+
});
|
|
2588
|
+
throw err;
|
|
2589
|
+
}
|
|
1862
2590
|
const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
|
|
1863
2591
|
const embed = async (texts) => {
|
|
1864
2592
|
const out = [];
|
|
1865
2593
|
for (let i = 0; i < texts.length; i += MAX_BATCH) {
|
|
1866
2594
|
const slice = texts.slice(i, i + MAX_BATCH).map(cap);
|
|
1867
|
-
const result = await pipe(slice, { pooling
|
|
2595
|
+
const result = await pipe(slice, { pooling, normalize: true });
|
|
1868
2596
|
out.push(...result.tolist());
|
|
1869
2597
|
}
|
|
1870
2598
|
return out;
|
|
1871
2599
|
};
|
|
1872
2600
|
return {
|
|
1873
2601
|
profile,
|
|
2602
|
+
verification,
|
|
1874
2603
|
embed,
|
|
1875
2604
|
embedQuery: async (text) => {
|
|
1876
2605
|
const [vector] = await embed([`${profile.queryInstruction}${text}`]);
|
|
@@ -1884,9 +2613,9 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
1884
2613
|
}
|
|
1885
2614
|
|
|
1886
2615
|
// src/markdown-adapter/load.ts
|
|
1887
|
-
import { readFile as
|
|
1888
|
-
import { join as
|
|
1889
|
-
import { CatalogDocumentSchema } from "@bendyline/
|
|
2616
|
+
import { readFile as readFile3, readdir } from "fs/promises";
|
|
2617
|
+
import { join as join6, relative as relative3, sep } from "path";
|
|
2618
|
+
import { CatalogDocumentSchema } from "@bendyline/gezk";
|
|
1890
2619
|
var ROOT_TOPIC_ID = "general";
|
|
1891
2620
|
var SUMMARY_MAX_CHARS = 280;
|
|
1892
2621
|
function parseFrontMatter(markdown) {
|
|
@@ -1940,7 +2669,7 @@ async function walkMarkdownFiles(root) {
|
|
|
1940
2669
|
);
|
|
1941
2670
|
for (const entry of entries) {
|
|
1942
2671
|
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
1943
|
-
const abs =
|
|
2672
|
+
const abs = join6(dir, entry.name);
|
|
1944
2673
|
if (entry.isDirectory()) await visit(abs);
|
|
1945
2674
|
else if (entry.isFile() && /\.(md|markdown)$/i.test(entry.name)) out.push(abs);
|
|
1946
2675
|
}
|
|
@@ -1991,7 +2720,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
1991
2720
|
const id = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
|
|
1992
2721
|
if (seenIds.has(id)) throw new Error(`duplicate document id from path: ${rel}`);
|
|
1993
2722
|
seenIds.add(id);
|
|
1994
|
-
const raw = (await
|
|
2723
|
+
const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
|
|
1995
2724
|
const fm = parseFrontMatter(raw);
|
|
1996
2725
|
const stem = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
|
|
1997
2726
|
const title = fm.title ?? firstHeading(fm.body) ?? stem;
|
|
@@ -2021,131 +2750,18 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
2021
2750
|
}
|
|
2022
2751
|
|
|
2023
2752
|
// src/registry-client/fetch.ts
|
|
2024
|
-
import {
|
|
2025
|
-
import { KnowledgeRegistryIndexSchema } from "@bendyline/gezel";
|
|
2753
|
+
import { KnowledgeRegistryIndexSchema } from "@bendyline/gezk";
|
|
2026
2754
|
|
|
2027
2755
|
// src/signatures/signing.ts
|
|
2028
2756
|
import {
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
// src/signatures/jcs.ts
|
|
2038
|
-
function canonicalizeJson(value) {
|
|
2039
|
-
return serialize(value, "$");
|
|
2040
|
-
}
|
|
2041
|
-
function serialize(value, path) {
|
|
2042
|
-
if (value === null) return "null";
|
|
2043
|
-
switch (typeof value) {
|
|
2044
|
-
case "boolean":
|
|
2045
|
-
return value ? "true" : "false";
|
|
2046
|
-
case "number":
|
|
2047
|
-
if (!Number.isFinite(value)) throw new Error(`non-finite number at ${path}`);
|
|
2048
|
-
return JSON.stringify(value);
|
|
2049
|
-
case "string":
|
|
2050
|
-
return JSON.stringify(value);
|
|
2051
|
-
case "object":
|
|
2052
|
-
break;
|
|
2053
|
-
default:
|
|
2054
|
-
throw new Error(`value of type ${typeof value} at ${path} has no JSON identity`);
|
|
2055
|
-
}
|
|
2056
|
-
if (Array.isArray(value)) {
|
|
2057
|
-
return `[${value.map((item, i) => serialize(item ?? null, `${path}[${i}]`)).join(",")}]`;
|
|
2058
|
-
}
|
|
2059
|
-
const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => compareUtf16(a, b));
|
|
2060
|
-
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${serialize(v, `${path}.${k}`)}`).join(",")}}`;
|
|
2061
|
-
}
|
|
2062
|
-
function compareUtf16(a, b) {
|
|
2063
|
-
const n = Math.min(a.length, b.length);
|
|
2064
|
-
for (let i = 0; i < n; i++) {
|
|
2065
|
-
const d = a.charCodeAt(i) - b.charCodeAt(i);
|
|
2066
|
-
if (d !== 0) return d;
|
|
2067
|
-
}
|
|
2068
|
-
return a.length - b.length;
|
|
2069
|
-
}
|
|
2070
|
-
|
|
2071
|
-
// src/signatures/signing.ts
|
|
2072
|
-
function knowledgeKeyId(publicKeyPem) {
|
|
2073
|
-
const der = createPublicKey(publicKeyPem).export({ type: "spki", format: "der" });
|
|
2074
|
-
return createHash5("sha256").update(der).digest("hex").slice(0, 16);
|
|
2075
|
-
}
|
|
2076
|
-
function generateKnowledgeSigningKeyPair() {
|
|
2077
|
-
const { publicKey, privateKey } = generateKeyPairSync("ed25519");
|
|
2078
|
-
const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }).toString();
|
|
2079
|
-
return {
|
|
2080
|
-
publicKeyPem,
|
|
2081
|
-
privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
|
|
2082
|
-
keyId: knowledgeKeyId(publicKeyPem)
|
|
2083
|
-
};
|
|
2084
|
-
}
|
|
2085
|
-
function manifestSigningPayload(manifest) {
|
|
2086
|
-
const { signature: _omitted, ...unsigned } = manifest;
|
|
2087
|
-
return Buffer.from(canonicalizeJson(unsigned), "utf8");
|
|
2088
|
-
}
|
|
2089
|
-
function signManifest(manifest, privateKeyPem) {
|
|
2090
|
-
const key = createPrivateKey(privateKeyPem);
|
|
2091
|
-
const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
|
|
2092
|
-
const value = cryptoSign(null, manifestSigningPayload(manifest), key).toString("base64");
|
|
2093
|
-
return {
|
|
2094
|
-
...manifest,
|
|
2095
|
-
signature: {
|
|
2096
|
-
algorithm: "ed25519",
|
|
2097
|
-
keyId: knowledgeKeyId(publicKeyPem),
|
|
2098
|
-
canonicalization: "rfc8785",
|
|
2099
|
-
value
|
|
2100
|
-
}
|
|
2101
|
-
};
|
|
2102
|
-
}
|
|
2103
|
-
function verifyManifestSignature(manifest, anchors) {
|
|
2104
|
-
return verifySignedDocument(manifest, anchors);
|
|
2105
|
-
}
|
|
2106
|
-
function signRegistryIndex(index, privateKeyPem) {
|
|
2107
|
-
const key = createPrivateKey(privateKeyPem);
|
|
2108
|
-
const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
|
|
2109
|
-
const { signature: _omitted, ...unsigned } = index;
|
|
2110
|
-
const value = cryptoSign(null, Buffer.from(canonicalizeJson(unsigned), "utf8"), key).toString(
|
|
2111
|
-
"base64"
|
|
2112
|
-
);
|
|
2113
|
-
return {
|
|
2114
|
-
...index,
|
|
2115
|
-
signature: {
|
|
2116
|
-
algorithm: "ed25519",
|
|
2117
|
-
keyId: knowledgeKeyId(publicKeyPem),
|
|
2118
|
-
canonicalization: "rfc8785",
|
|
2119
|
-
value
|
|
2120
|
-
}
|
|
2121
|
-
};
|
|
2122
|
-
}
|
|
2123
|
-
function verifyRegistryIndex(index, anchors) {
|
|
2124
|
-
return verifySignedDocument(index, anchors);
|
|
2125
|
-
}
|
|
2126
|
-
function verifySignedDocument(document, anchors) {
|
|
2127
|
-
const signature = document.signature;
|
|
2128
|
-
if (!signature) return { ok: false, reason: "unsigned" };
|
|
2129
|
-
const anchor = anchors.find((a) => a.keyId === signature.keyId);
|
|
2130
|
-
if (!anchor) {
|
|
2131
|
-
return { ok: false, reason: "unknown-key", detail: `no trust anchor for ${signature.keyId}` };
|
|
2132
|
-
}
|
|
2133
|
-
try {
|
|
2134
|
-
if (knowledgeKeyId(anchor.publicKeyPem) !== signature.keyId) {
|
|
2135
|
-
return { ok: false, reason: "unknown-key", detail: "anchor keyId does not match its key" };
|
|
2136
|
-
}
|
|
2137
|
-
const { signature: _omitted, ...unsigned } = document;
|
|
2138
|
-
const ok = cryptoVerify(
|
|
2139
|
-
null,
|
|
2140
|
-
Buffer.from(canonicalizeJson(unsigned), "utf8"),
|
|
2141
|
-
createPublicKey(anchor.publicKeyPem),
|
|
2142
|
-
Buffer.from(signature.value, "base64")
|
|
2143
|
-
);
|
|
2144
|
-
return ok ? { ok: true, keyId: signature.keyId } : { ok: false, reason: "bad-signature" };
|
|
2145
|
-
} catch (err) {
|
|
2146
|
-
return { ok: false, reason: "error", detail: err instanceof Error ? err.message : String(err) };
|
|
2147
|
-
}
|
|
2148
|
-
}
|
|
2757
|
+
generateKnowledgeSigningKeyPair,
|
|
2758
|
+
knowledgeKeyId,
|
|
2759
|
+
manifestSigningPayload,
|
|
2760
|
+
signManifest,
|
|
2761
|
+
signRegistryIndex,
|
|
2762
|
+
verifyManifestSignature,
|
|
2763
|
+
verifyRegistryIndex
|
|
2764
|
+
} from "@bendyline/gezk/node";
|
|
2149
2765
|
|
|
2150
2766
|
// src/registry-client/fetch.ts
|
|
2151
2767
|
var DEFAULT_MAX_REGISTRY_BYTES = 4 * 1024 * 1024;
|
|
@@ -2164,7 +2780,7 @@ async function fetchKnowledgeRegistry(url, opts) {
|
|
|
2164
2780
|
let response;
|
|
2165
2781
|
try {
|
|
2166
2782
|
response = await fetchImpl(url, {
|
|
2167
|
-
signal:
|
|
2783
|
+
signal: opts.signal ?? AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
|
2168
2784
|
redirect: "follow",
|
|
2169
2785
|
headers: { accept: "application/json" }
|
|
2170
2786
|
});
|
|
@@ -2197,7 +2813,7 @@ async function fetchKnowledgeRegistry(url, opts) {
|
|
|
2197
2813
|
} catch (err) {
|
|
2198
2814
|
throw new KnowledgeRegistryFetchError(
|
|
2199
2815
|
"invalid",
|
|
2200
|
-
`registry is not a valid
|
|
2816
|
+
`registry is not a valid gezk-registry document: ${err instanceof Error ? err.message : String(err)}`
|
|
2201
2817
|
);
|
|
2202
2818
|
}
|
|
2203
2819
|
const verdict = verifyRegistryIndex(registry, opts.anchors);
|
|
@@ -2243,7 +2859,11 @@ function newerRegistryEntries(registry, installed) {
|
|
|
2243
2859
|
|
|
2244
2860
|
// src/signatures/anchors.ts
|
|
2245
2861
|
var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
|
|
2862
|
+
|
|
2863
|
+
// src/signatures/jcs.ts
|
|
2864
|
+
import { canonicalizeJson } from "@bendyline/gezk";
|
|
2246
2865
|
export {
|
|
2866
|
+
BGE_SMALL_EN_V15_1,
|
|
2247
2867
|
BODY_CODEC_MIN_BYTES,
|
|
2248
2868
|
BUILTIN_KNOWLEDGE_TRUST_ANCHORS,
|
|
2249
2869
|
CENTROID_CHUNKS_PER,
|
|
@@ -2252,6 +2872,8 @@ export {
|
|
|
2252
2872
|
CENTROID_SAMPLE_MAX,
|
|
2253
2873
|
CatalogHandle,
|
|
2254
2874
|
CatalogOpenError,
|
|
2875
|
+
DuckdbError,
|
|
2876
|
+
EmbedderArtifactError,
|
|
2255
2877
|
EmbedderUnavailableError,
|
|
2256
2878
|
GEZEL_BGE_SMALL_EN_V15_1,
|
|
2257
2879
|
GEZEL_MARKDOWN_CHUNKS_2,
|
|
@@ -2260,11 +2882,20 @@ export {
|
|
|
2260
2882
|
GEZK_ARCHIVE_LIMITS,
|
|
2261
2883
|
GEZK_FORMAT_VERSION,
|
|
2262
2884
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
2885
|
+
GEZK_MIME_TYPE,
|
|
2263
2886
|
GezkArchiveError,
|
|
2264
2887
|
KNOWLEDGE_EMBEDDING_PROFILES,
|
|
2888
|
+
KNOWLEDGE_TOOLCHAIN,
|
|
2265
2889
|
KnowledgeRegistryFetchError,
|
|
2890
|
+
LICENSE_NOTICE_PATH,
|
|
2266
2891
|
MANIFEST_PATH,
|
|
2892
|
+
MARKDOWN_CHUNKS_2,
|
|
2267
2893
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
2894
|
+
MIMETYPE_PATH,
|
|
2895
|
+
MULTILINGUAL_E5_SMALL_1,
|
|
2896
|
+
PARQUET_EXPORT_VERSION,
|
|
2897
|
+
PARQUET_REPORT_PATH,
|
|
2898
|
+
README_PATH,
|
|
2268
2899
|
RERANK_FINAL_K,
|
|
2269
2900
|
ROUTER_DB_PATH,
|
|
2270
2901
|
ROUTER_DDL,
|
|
@@ -2273,7 +2904,9 @@ export {
|
|
|
2273
2904
|
SHARD_DDL,
|
|
2274
2905
|
SHARD_MAX_CHUNKS,
|
|
2275
2906
|
SHARD_TARGET_CHUNKS,
|
|
2907
|
+
SOURCE_NOTICES_PATH,
|
|
2276
2908
|
ZIP_FIXED_MTIME,
|
|
2909
|
+
assertDuckdbCli,
|
|
2277
2910
|
canonicalizeJson,
|
|
2278
2911
|
chunkContentHash,
|
|
2279
2912
|
chunkMarkdownProfile,
|
|
@@ -2282,10 +2915,14 @@ export {
|
|
|
2282
2915
|
compileKnowledgeCatalog,
|
|
2283
2916
|
createProfileEmbedder,
|
|
2284
2917
|
documentSlug,
|
|
2918
|
+
duckdbVersion,
|
|
2919
|
+
exportCatalogParquet,
|
|
2285
2920
|
extractGezkVerified,
|
|
2286
2921
|
fetchKnowledgeRegistry,
|
|
2922
|
+
findDuckdbBinary,
|
|
2287
2923
|
findRegistryEntry,
|
|
2288
2924
|
generateKnowledgeSigningKeyPair,
|
|
2925
|
+
hammingTopK,
|
|
2289
2926
|
inspectGezkArchive,
|
|
2290
2927
|
knowledgeEmbeddingProfile,
|
|
2291
2928
|
knowledgeKeyId,
|
|
@@ -2299,11 +2936,14 @@ export {
|
|
|
2299
2936
|
readGezkManifest,
|
|
2300
2937
|
rerankK,
|
|
2301
2938
|
rerankScore,
|
|
2939
|
+
resolveTransformersModelOptions,
|
|
2940
|
+
runDuckdbScript,
|
|
2302
2941
|
signManifest,
|
|
2303
2942
|
signRegistryIndex,
|
|
2943
|
+
transformersCachePath,
|
|
2304
2944
|
validateExtractedCatalog,
|
|
2305
|
-
vecChunksDdl,
|
|
2306
2945
|
verifyManifestSignature,
|
|
2946
|
+
verifyProfileArtifacts,
|
|
2307
2947
|
verifyRegistryIndex,
|
|
2308
2948
|
writeGezkArchive
|
|
2309
2949
|
};
|