@bendyline/gezk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +43 -0
- package/dist/index.d.ts +492 -0
- package/dist/index.js +548 -0
- package/dist/node.d.ts +72 -0
- package/dist/node.js +678 -0
- package/package.json +71 -0
package/dist/node.js
ADDED
|
@@ -0,0 +1,678 @@
|
|
|
1
|
+
// src/schemas/ids.ts
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
var KNOWLEDGE_ID_PATTERN = /^[a-z0-9]([a-z0-9-]{0,62}[a-z0-9])?$/;
|
|
4
|
+
var KnowledgeIdSchema = z.string().regex(KNOWLEDGE_ID_PATTERN);
|
|
5
|
+
var KNOWLEDGE_VERSION_PATTERN = /^[A-Za-z0-9](?:[A-Za-z0-9._+-]{0,126}[A-Za-z0-9])?$/;
|
|
6
|
+
var KnowledgeVersionSchema = z.string().min(1).max(128).regex(KNOWLEDGE_VERSION_PATTERN, "catalog version must be one portable path segment").refine(
|
|
7
|
+
(value) => !/^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(value),
|
|
8
|
+
"catalog version must not use a reserved Windows device name"
|
|
9
|
+
);
|
|
10
|
+
function hasControlCharacters(value) {
|
|
11
|
+
for (const ch of value) {
|
|
12
|
+
const cp = ch.codePointAt(0);
|
|
13
|
+
if (cp <= 31 || cp >= 127 && cp <= 159) return true;
|
|
14
|
+
}
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
var KnowledgeDocumentIdSchema = z.string().min(1).max(256).refine((s) => s === s.normalize("NFC"), "document id must be NFC-normalized").refine((s) => !hasControlCharacters(s), "document id must not contain controls").refine((s) => s === s.trim(), "document id must not have leading/trailing whitespace");
|
|
18
|
+
var KnowledgeChunkUidSchema = z.string().regex(/^[0-9a-f]{32}$/);
|
|
19
|
+
var Sha256HexSchema = z.string().regex(/^[0-9a-f]{64}$/);
|
|
20
|
+
|
|
21
|
+
// src/schemas/profiles.ts
|
|
22
|
+
import { z as z2 } from "zod";
|
|
23
|
+
var KnowledgeVectorEncodingSchema = z2.enum(["bit+int8"]);
|
|
24
|
+
var ArtifactDigestSchema = z2.string().regex(/^sha256:[0-9a-f]{64}$/, "artifact digest must be sha256:<64 lowercase hex>");
|
|
25
|
+
var RepoRelativePathSchema = z2.string().min(1).refine(
|
|
26
|
+
(path) => !path.startsWith("/") && !path.includes("\\") && path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== ".."),
|
|
27
|
+
"must be a repo-relative path"
|
|
28
|
+
);
|
|
29
|
+
var DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
|
|
30
|
+
var DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
|
|
31
|
+
var KnowledgeEmbeddingProfileSchema = z2.object({
|
|
32
|
+
/** e.g. `multilingual-e5-small@1`. A new model, prefix or encoding is a new id. */
|
|
33
|
+
id: z2.string().min(1),
|
|
34
|
+
model: z2.object({
|
|
35
|
+
/** Hugging Face repo id (`owner/name`). */
|
|
36
|
+
repo: z2.string().min(1),
|
|
37
|
+
/** Commit sha or tag the vectors were produced with. */
|
|
38
|
+
revision: z2.string().min(1),
|
|
39
|
+
/**
|
|
40
|
+
* Repo-relative path of the ONNX graph the vectors were produced with;
|
|
41
|
+
* defaults to `onnx/model.onnx`, the full-precision graph. The precision
|
|
42
|
+
* variant is part of the vector space: fp16 or int8 weights move the
|
|
43
|
+
* floats, and with them the sign bits and int8 codes at the margins.
|
|
44
|
+
*/
|
|
45
|
+
onnxFile: RepoRelativePathSchema.optional(),
|
|
46
|
+
/** sha256 of that file's bytes at `revision`. */
|
|
47
|
+
onnxDigest: ArtifactDigestSchema.optional()
|
|
48
|
+
}),
|
|
49
|
+
tokenizer: z2.object({
|
|
50
|
+
kind: z2.string().min(1),
|
|
51
|
+
/** Repo-relative path of the tokenizer definition; defaults to `tokenizer.json`. */
|
|
52
|
+
file: RepoRelativePathSchema.optional(),
|
|
53
|
+
/** sha256 of that file's bytes at `revision`. */
|
|
54
|
+
digest: ArtifactDigestSchema.optional()
|
|
55
|
+
}),
|
|
56
|
+
pooling: z2.enum(["mean", "cls", "last"]),
|
|
57
|
+
normalized: z2.boolean(),
|
|
58
|
+
dimensions: z2.number().int().positive(),
|
|
59
|
+
maxTokens: z2.number().int().positive(),
|
|
60
|
+
queryInstruction: z2.string(),
|
|
61
|
+
passageInstruction: z2.string(),
|
|
62
|
+
vectorEncoding: KnowledgeVectorEncodingSchema,
|
|
63
|
+
distance: z2.object({
|
|
64
|
+
stage1: z2.literal("hamming"),
|
|
65
|
+
stage2: z2.literal("cosine")
|
|
66
|
+
}),
|
|
67
|
+
quantization: z2.object({
|
|
68
|
+
int8: z2.object({
|
|
69
|
+
method: z2.literal("symmetric-linear"),
|
|
70
|
+
scale: z2.literal(127)
|
|
71
|
+
}),
|
|
72
|
+
binary: z2.object({
|
|
73
|
+
method: z2.literal("sign"),
|
|
74
|
+
threshold: z2.literal(0),
|
|
75
|
+
packing: z2.literal("lsb-first")
|
|
76
|
+
})
|
|
77
|
+
})
|
|
78
|
+
});
|
|
79
|
+
function embeddingProfileArtifacts(profile) {
|
|
80
|
+
return {
|
|
81
|
+
onnxFile: profile.model.onnxFile ?? DEFAULT_EMBEDDING_ONNX_FILE,
|
|
82
|
+
onnxDigest: profile.model.onnxDigest ?? null,
|
|
83
|
+
tokenizerFile: profile.tokenizer.file ?? DEFAULT_EMBEDDING_TOKENIZER_FILE,
|
|
84
|
+
tokenizerDigest: profile.tokenizer.digest ?? null
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
function sameVectorSpace(a, b) {
|
|
88
|
+
const aa = embeddingProfileArtifacts(a);
|
|
89
|
+
const ab = embeddingProfileArtifacts(b);
|
|
90
|
+
const digestsAgree = (x, y) => !x || !y || x === y;
|
|
91
|
+
return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale && a.quantization.binary.method === b.quantization.binary.method && a.quantization.binary.threshold === b.quantization.binary.threshold && a.quantization.binary.packing === b.quantization.binary.packing;
|
|
92
|
+
}
|
|
93
|
+
var KnowledgeChunkingProfileSchema = z2.object({
|
|
94
|
+
/** e.g. `markdown-chunks@2`. */
|
|
95
|
+
id: z2.string().min(1),
|
|
96
|
+
unit: z2.enum(["tokens", "chars"]),
|
|
97
|
+
tokenizer: z2.enum(["profile", "none"]),
|
|
98
|
+
target: z2.number().int().positive(),
|
|
99
|
+
overlap: z2.number().int().nonnegative(),
|
|
100
|
+
contextHeader: z2.object({ max: z2.number().int().nonnegative() })
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
// src/schemas/manifest.ts
|
|
104
|
+
import { z as z3 } from "zod";
|
|
105
|
+
|
|
106
|
+
// src/format/constants.ts
|
|
107
|
+
var GEZK_FORMAT_VERSION = "0.5";
|
|
108
|
+
var GEZK_INDEX_SCHEMA_VERSION = 2;
|
|
109
|
+
var GEZK_APPLICATION_ID = 1195727435;
|
|
110
|
+
var GEZK_MIME_TYPE = "application/vnd.gezk+zip";
|
|
111
|
+
var MIMETYPE_PATH = "mimetype";
|
|
112
|
+
var MANIFEST_PATH = "manifest.json";
|
|
113
|
+
var README_PATH = "README.md";
|
|
114
|
+
var LICENSE_NOTICE_PATH = "LICENSES/catalog.txt";
|
|
115
|
+
var SOURCE_NOTICES_PATH = "LICENSES/source-notices.json";
|
|
116
|
+
var ROUTER_DB_PATH = "index/router.db";
|
|
117
|
+
var BODY_CODEC_MIN_BYTES = 512;
|
|
118
|
+
var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
|
|
119
|
+
var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
|
|
120
|
+
|
|
121
|
+
// src/schemas/manifest.ts
|
|
122
|
+
var GEZK_MANIFEST_KIND = "gezk-catalog";
|
|
123
|
+
var GEZK_REGISTRY_KIND = "gezk-registry";
|
|
124
|
+
var KnowledgeManifestFileSchema = z3.object({
|
|
125
|
+
path: z3.string().min(1),
|
|
126
|
+
sizeBytes: z3.number().int().nonnegative(),
|
|
127
|
+
sha256: Sha256HexSchema
|
|
128
|
+
});
|
|
129
|
+
var KnowledgeSignatureSchema = z3.object({
|
|
130
|
+
algorithm: z3.literal("ed25519"),
|
|
131
|
+
keyId: z3.string().min(1),
|
|
132
|
+
canonicalization: z3.literal("rfc8785"),
|
|
133
|
+
value: z3.string().min(1)
|
|
134
|
+
});
|
|
135
|
+
var KnowledgeCatalogManifestSchema = z3.object({
|
|
136
|
+
kind: z3.literal(GEZK_MANIFEST_KIND),
|
|
137
|
+
formatVersion: z3.literal(GEZK_FORMAT_VERSION),
|
|
138
|
+
indexSchemaVersion: z3.literal(GEZK_INDEX_SCHEMA_VERSION),
|
|
139
|
+
id: KnowledgeIdSchema,
|
|
140
|
+
version: KnowledgeVersionSchema,
|
|
141
|
+
name: z3.string().min(1),
|
|
142
|
+
description: z3.string().optional(),
|
|
143
|
+
language: z3.string().min(2),
|
|
144
|
+
publisher: z3.object({
|
|
145
|
+
id: KnowledgeIdSchema,
|
|
146
|
+
name: z3.string().min(1),
|
|
147
|
+
url: z3.string().optional()
|
|
148
|
+
}),
|
|
149
|
+
createdAt: z3.string(),
|
|
150
|
+
sourceSnapshot: z3.object({
|
|
151
|
+
name: z3.string(),
|
|
152
|
+
date: z3.string(),
|
|
153
|
+
taxonomyVersion: z3.string().optional()
|
|
154
|
+
}).optional(),
|
|
155
|
+
license: z3.object({
|
|
156
|
+
name: z3.string().min(1),
|
|
157
|
+
/** SPDX identifier when one exists (e.g. `CC-BY-SA-4.0`). */
|
|
158
|
+
spdx: z3.string().optional(),
|
|
159
|
+
/** Archive path of the human-readable notice (`LICENSES/catalog.txt`). */
|
|
160
|
+
noticePath: z3.string().min(1),
|
|
161
|
+
attributionRequired: z3.boolean()
|
|
162
|
+
}),
|
|
163
|
+
embedding: KnowledgeEmbeddingProfileSchema,
|
|
164
|
+
chunking: KnowledgeChunkingProfileSchema,
|
|
165
|
+
topics: z3.array(z3.object({ id: KnowledgeIdSchema, name: z3.string().min(1) })).min(1, "a catalog must ship a table of contents (at least one topic)"),
|
|
166
|
+
router: z3.object({
|
|
167
|
+
shardTargetChunks: z3.number().int().positive(),
|
|
168
|
+
shards: z3.array(
|
|
169
|
+
z3.object({
|
|
170
|
+
id: z3.number().int().nonnegative(),
|
|
171
|
+
path: z3.string().min(1),
|
|
172
|
+
chunks: z3.number().int().nonnegative(),
|
|
173
|
+
documents: z3.number().int().nonnegative(),
|
|
174
|
+
centroids: z3.number().int().nonnegative(),
|
|
175
|
+
sha256: Sha256HexSchema
|
|
176
|
+
})
|
|
177
|
+
),
|
|
178
|
+
totalCentroids: z3.number().int().nonnegative()
|
|
179
|
+
}),
|
|
180
|
+
counts: z3.object({
|
|
181
|
+
documents: z3.number().int().nonnegative(),
|
|
182
|
+
chunks: z3.number().int().nonnegative(),
|
|
183
|
+
shards: z3.number().int().positive()
|
|
184
|
+
}),
|
|
185
|
+
files: z3.array(KnowledgeManifestFileSchema).min(1),
|
|
186
|
+
/** What a reader must implement to open this catalog. */
|
|
187
|
+
requires: z3.object({
|
|
188
|
+
formatVersion: z3.literal(GEZK_FORMAT_VERSION),
|
|
189
|
+
features: z3.array(z3.string()).optional()
|
|
190
|
+
}),
|
|
191
|
+
smokeQueries: z3.array(
|
|
192
|
+
z3.object({
|
|
193
|
+
query: z3.string().min(1),
|
|
194
|
+
expectedDocumentIds: z3.array(KnowledgeDocumentIdSchema).min(1)
|
|
195
|
+
})
|
|
196
|
+
).optional(),
|
|
197
|
+
/** Which tool produced the archive — provenance, never a compatibility gate. */
|
|
198
|
+
toolchain: z3.object({
|
|
199
|
+
name: z3.string().min(1),
|
|
200
|
+
version: z3.string().min(1),
|
|
201
|
+
node: z3.string().optional(),
|
|
202
|
+
sqlite: z3.string().optional(),
|
|
203
|
+
platform: z3.string().optional(),
|
|
204
|
+
modelDigest: z3.string().optional(),
|
|
205
|
+
tokenizerDigest: z3.string().optional()
|
|
206
|
+
}).optional(),
|
|
207
|
+
signature: KnowledgeSignatureSchema.optional()
|
|
208
|
+
});
|
|
209
|
+
var SourceNoticesSchema = z3.object({
|
|
210
|
+
sources: z3.array(
|
|
211
|
+
z3.object({
|
|
212
|
+
name: z3.string().min(1),
|
|
213
|
+
url: z3.string().optional(),
|
|
214
|
+
license: z3.string().min(1),
|
|
215
|
+
licenseUrl: z3.string().optional(),
|
|
216
|
+
/** Snapshot or revision the content was taken from. */
|
|
217
|
+
snapshot: z3.string().optional(),
|
|
218
|
+
/** Free-form notice text required by the source's license. */
|
|
219
|
+
notice: z3.string().optional()
|
|
220
|
+
})
|
|
221
|
+
).min(1)
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
// src/schemas/document.ts
|
|
225
|
+
import { z as z4 } from "zod";
|
|
226
|
+
var CatalogDocumentSchema = z4.object({
|
|
227
|
+
id: KnowledgeDocumentIdSchema,
|
|
228
|
+
title: z4.string().min(1),
|
|
229
|
+
slug: z4.string().min(1),
|
|
230
|
+
summary: z4.string().optional(),
|
|
231
|
+
language: z4.string().min(2),
|
|
232
|
+
/** Root→leaf topic id path; the first segment must exist in the manifest. */
|
|
233
|
+
topicPath: z4.array(KnowledgeIdSchema).min(1),
|
|
234
|
+
markdown: z4.string(),
|
|
235
|
+
sourceUrl: z4.string().optional(),
|
|
236
|
+
sourceRevision: z4.string().optional(),
|
|
237
|
+
sourceUpdatedAt: z4.string().optional(),
|
|
238
|
+
attribution: z4.record(z4.string(), z4.string()).optional(),
|
|
239
|
+
aliases: z4.array(z4.string()).optional()
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
// src/schemas/registry.ts
|
|
243
|
+
import { z as z5 } from "zod";
|
|
244
|
+
var KnowledgeRegistryEntrySchema = z5.object({
|
|
245
|
+
catalogId: KnowledgeIdSchema,
|
|
246
|
+
version: z5.string().min(1),
|
|
247
|
+
name: z5.string().min(1),
|
|
248
|
+
description: z5.string().optional(),
|
|
249
|
+
language: z5.string().min(2),
|
|
250
|
+
documents: z5.number().int().nonnegative(),
|
|
251
|
+
/** Size of the `.gezk` archive itself (download accounting/preflight). */
|
|
252
|
+
archiveBytes: z5.number().int().nonnegative(),
|
|
253
|
+
/** sha256 of the `.gezk` archive — the ref's contentDigest after install. */
|
|
254
|
+
contentDigest: Sha256HexSchema,
|
|
255
|
+
/** Absolute download URL. Uploaded before the registry that names it. */
|
|
256
|
+
url: z5.string().url(),
|
|
257
|
+
license: z5.object({ name: z5.string().min(1), attributionRequired: z5.boolean() }),
|
|
258
|
+
sourceSnapshot: z5.object({ name: z5.string(), date: z5.string(), taxonomyVersion: z5.string().optional() }).optional()
|
|
259
|
+
});
|
|
260
|
+
var KnowledgeRegistryIndexSchema = z5.object({
|
|
261
|
+
kind: z5.literal(GEZK_REGISTRY_KIND),
|
|
262
|
+
formatVersion: z5.literal(GEZK_FORMAT_VERSION),
|
|
263
|
+
publisher: z5.object({
|
|
264
|
+
id: KnowledgeIdSchema,
|
|
265
|
+
name: z5.string().min(1),
|
|
266
|
+
url: z5.string().optional()
|
|
267
|
+
}),
|
|
268
|
+
generatedAt: z5.string(),
|
|
269
|
+
catalogs: z5.array(KnowledgeRegistryEntrySchema),
|
|
270
|
+
signature: KnowledgeSignatureSchema.optional()
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
// src/uri.ts
|
|
274
|
+
var KNOWLEDGE_URI_PREFIX = "knowledge://";
|
|
275
|
+
var MAX_ENCODED_DOCUMENT_ID = 512;
|
|
276
|
+
function formatKnowledgeUri(uri) {
|
|
277
|
+
const encodedDoc = uri.documentId.split("/").map((seg) => encodeURIComponent(seg)).join("/");
|
|
278
|
+
let out = `${KNOWLEDGE_URI_PREFIX}${uri.publisherId}/${uri.catalogId}/${encodedDoc}`;
|
|
279
|
+
if (uri.fragment) {
|
|
280
|
+
out += "chunk" in uri.fragment ? `#chunk=${uri.fragment.chunk}` : `#line=${uri.fragment.lineStart}${uri.fragment.lineEnd !== void 0 ? `-${uri.fragment.lineEnd}` : ""}`;
|
|
281
|
+
}
|
|
282
|
+
return out;
|
|
283
|
+
}
|
|
284
|
+
function parseKnowledgeUri(raw) {
|
|
285
|
+
if (!raw.startsWith(KNOWLEDGE_URI_PREFIX)) return null;
|
|
286
|
+
const rest = raw.slice(KNOWLEDGE_URI_PREFIX.length);
|
|
287
|
+
const hashAt = rest.indexOf("#");
|
|
288
|
+
const body = hashAt === -1 ? rest : rest.slice(0, hashAt);
|
|
289
|
+
const fragmentRaw = hashAt === -1 ? null : rest.slice(hashAt + 1);
|
|
290
|
+
const firstSlash = body.indexOf("/");
|
|
291
|
+
if (firstSlash <= 0) return null;
|
|
292
|
+
const secondSlash = body.indexOf("/", firstSlash + 1);
|
|
293
|
+
if (secondSlash <= firstSlash + 1) return null;
|
|
294
|
+
const publisherId = body.slice(0, firstSlash);
|
|
295
|
+
const catalogId = body.slice(firstSlash + 1, secondSlash);
|
|
296
|
+
const encodedDoc = body.slice(secondSlash + 1);
|
|
297
|
+
if (!KNOWLEDGE_ID_PATTERN.test(publisherId) || !KNOWLEDGE_ID_PATTERN.test(catalogId)) {
|
|
298
|
+
return null;
|
|
299
|
+
}
|
|
300
|
+
if (!encodedDoc || encodedDoc.length > MAX_ENCODED_DOCUMENT_ID) return null;
|
|
301
|
+
let documentId;
|
|
302
|
+
try {
|
|
303
|
+
documentId = encodedDoc.split("/").map((seg) => {
|
|
304
|
+
if (!seg) throw new Error("empty segment");
|
|
305
|
+
return decodeURIComponent(seg);
|
|
306
|
+
}).join("/");
|
|
307
|
+
} catch {
|
|
308
|
+
return null;
|
|
309
|
+
}
|
|
310
|
+
if (!KnowledgeDocumentIdSchema.safeParse(documentId).success) return null;
|
|
311
|
+
const base = { publisherId, catalogId, documentId };
|
|
312
|
+
if (fragmentRaw === null) return base;
|
|
313
|
+
const chunkMatch = /^chunk=([0-9a-f]{32})$/.exec(fragmentRaw);
|
|
314
|
+
if (chunkMatch) return { ...base, fragment: { chunk: chunkMatch[1] } };
|
|
315
|
+
const lineMatch = /^line=(\d+)(?:-(\d+))?$/.exec(fragmentRaw);
|
|
316
|
+
if (lineMatch) {
|
|
317
|
+
return {
|
|
318
|
+
...base,
|
|
319
|
+
fragment: {
|
|
320
|
+
lineStart: Number.parseInt(lineMatch[1], 10),
|
|
321
|
+
...lineMatch[2] !== void 0 ? { lineEnd: Number.parseInt(lineMatch[2], 10) } : {}
|
|
322
|
+
}
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
return null;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// src/format/ddl.ts
|
|
329
|
+
var ROUTER_DDL = `
|
|
330
|
+
CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
331
|
+
|
|
332
|
+
CREATE TABLE topics(
|
|
333
|
+
id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,
|
|
334
|
+
description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL
|
|
335
|
+
);
|
|
336
|
+
|
|
337
|
+
CREATE TABLE documents(
|
|
338
|
+
id TEXT PRIMARY KEY,
|
|
339
|
+
title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
|
|
340
|
+
language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
|
|
341
|
+
shard_id INTEGER NOT NULL,
|
|
342
|
+
chunk_count INTEGER NOT NULL,
|
|
343
|
+
source_url TEXT, source_revision TEXT, source_updated_at TEXT,
|
|
344
|
+
attribution_json TEXT,
|
|
345
|
+
body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
|
|
346
|
+
body_blob BLOB NOT NULL
|
|
347
|
+
);
|
|
348
|
+
CREATE INDEX documents_topic ON documents(topic_id, slug);
|
|
349
|
+
CREATE INDEX documents_shard ON documents(shard_id);
|
|
350
|
+
|
|
351
|
+
CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
|
|
352
|
+
PRIMARY KEY (alias, document_id)) WITHOUT ROWID;
|
|
353
|
+
|
|
354
|
+
CREATE TABLE shards(
|
|
355
|
+
id INTEGER PRIMARY KEY,
|
|
356
|
+
path TEXT NOT NULL,
|
|
357
|
+
chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,
|
|
358
|
+
topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,
|
|
359
|
+
bytes INTEGER NOT NULL
|
|
360
|
+
);
|
|
361
|
+
|
|
362
|
+
CREATE TABLE route_centroids(
|
|
363
|
+
id INTEGER PRIMARY KEY,
|
|
364
|
+
shard_id INTEGER NOT NULL REFERENCES shards(id),
|
|
365
|
+
embedding BLOB NOT NULL,
|
|
366
|
+
weight INTEGER NOT NULL
|
|
367
|
+
);
|
|
368
|
+
CREATE INDEX route_centroids_shard ON route_centroids(shard_id);
|
|
369
|
+
|
|
370
|
+
CREATE VIRTUAL TABLE fts_documents USING fts5(
|
|
371
|
+
title, summary, aliases, document_id UNINDEXED,
|
|
372
|
+
tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
|
|
373
|
+
);
|
|
374
|
+
`;
|
|
375
|
+
var SHARD_DDL = `
|
|
376
|
+
CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
377
|
+
|
|
378
|
+
CREATE TABLE chunks(
|
|
379
|
+
id INTEGER PRIMARY KEY,
|
|
380
|
+
chunk_uid TEXT NOT NULL,
|
|
381
|
+
document_id TEXT NOT NULL,
|
|
382
|
+
ordinal INTEGER NOT NULL,
|
|
383
|
+
title TEXT NOT NULL,
|
|
384
|
+
heading_path TEXT NOT NULL,
|
|
385
|
+
heading_text TEXT NOT NULL,
|
|
386
|
+
line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,
|
|
387
|
+
token_count INTEGER NOT NULL,
|
|
388
|
+
content_hash TEXT NOT NULL,
|
|
389
|
+
text TEXT NOT NULL
|
|
390
|
+
);
|
|
391
|
+
CREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);
|
|
392
|
+
CREATE INDEX chunks_document ON chunks(document_id, ordinal);
|
|
393
|
+
|
|
394
|
+
CREATE VIRTUAL TABLE fts_chunks USING fts5(
|
|
395
|
+
title, heading_text, text,
|
|
396
|
+
content = 'chunks', content_rowid = 'id',
|
|
397
|
+
tokenize = 'unicode61 remove_diacritics 2'
|
|
398
|
+
);
|
|
399
|
+
|
|
400
|
+
CREATE TABLE chunk_vectors_bit(
|
|
401
|
+
chunk_id INTEGER PRIMARY KEY,
|
|
402
|
+
v BLOB NOT NULL
|
|
403
|
+
);
|
|
404
|
+
|
|
405
|
+
CREATE TABLE chunk_vectors_int8(
|
|
406
|
+
chunk_id INTEGER PRIMARY KEY,
|
|
407
|
+
v BLOB NOT NULL
|
|
408
|
+
);
|
|
409
|
+
`;
|
|
410
|
+
|
|
411
|
+
// src/format/quantize.ts
|
|
412
|
+
function quantizeInt8(unitVector) {
|
|
413
|
+
const out = new Int8Array(unitVector.length);
|
|
414
|
+
for (let i = 0; i < unitVector.length; i++) {
|
|
415
|
+
const q = Math.round(unitVector[i] * 127);
|
|
416
|
+
out[i] = q > 127 ? 127 : q < -127 ? -127 : q;
|
|
417
|
+
}
|
|
418
|
+
return out;
|
|
419
|
+
}
|
|
420
|
+
function quantizeBinary(unitVector) {
|
|
421
|
+
const bytes = Math.ceil(unitVector.length / 8);
|
|
422
|
+
const out = new Uint8Array(bytes);
|
|
423
|
+
for (let i = 0; i < unitVector.length; i++) {
|
|
424
|
+
if (unitVector[i] > 0) {
|
|
425
|
+
out[i >> 3] = out[i >> 3] | 1 << (i & 7);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
return out;
|
|
429
|
+
}
|
|
430
|
+
function rerankScore(query, passageInt8) {
|
|
431
|
+
let dot = 0;
|
|
432
|
+
const n = Math.min(query.length, passageInt8.length);
|
|
433
|
+
for (let i = 0; i < n; i++) dot += query[i] * (passageInt8[i] / 127);
|
|
434
|
+
return dot;
|
|
435
|
+
}
|
|
436
|
+
function l2Normalize(vector) {
|
|
437
|
+
let sum = 0;
|
|
438
|
+
for (let i = 0; i < vector.length; i++) {
|
|
439
|
+
const v = vector[i];
|
|
440
|
+
sum += v * v;
|
|
441
|
+
}
|
|
442
|
+
const norm = Math.sqrt(sum);
|
|
443
|
+
if (!Number.isFinite(norm) || norm === 0) throw new Error("degenerate vector (zero norm)");
|
|
444
|
+
const out = new Float32Array(vector.length);
|
|
445
|
+
for (let i = 0; i < vector.length; i++) out[i] = vector[i] / norm;
|
|
446
|
+
return out;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
// src/jcs.ts
|
|
450
|
+
function canonicalizeJson(value) {
|
|
451
|
+
return serialize(value, "$");
|
|
452
|
+
}
|
|
453
|
+
function serialize(input, path) {
|
|
454
|
+
const value = unwrapToJson(input, path);
|
|
455
|
+
if (value === null) return "null";
|
|
456
|
+
switch (typeof value) {
|
|
457
|
+
case "boolean":
|
|
458
|
+
return value ? "true" : "false";
|
|
459
|
+
case "number":
|
|
460
|
+
if (!Number.isFinite(value)) throw new Error(`non-finite number at ${path}`);
|
|
461
|
+
return JSON.stringify(value);
|
|
462
|
+
case "string":
|
|
463
|
+
return JSON.stringify(value);
|
|
464
|
+
case "object":
|
|
465
|
+
break;
|
|
466
|
+
default:
|
|
467
|
+
throw new Error(`value of type ${typeof value} at ${path} has no JSON identity`);
|
|
468
|
+
}
|
|
469
|
+
if (Array.isArray(value)) {
|
|
470
|
+
return `[${value.map((item, i) => serialize(item ?? null, `${path}[${i}]`)).join(",")}]`;
|
|
471
|
+
}
|
|
472
|
+
const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => compareUtf16(a, b));
|
|
473
|
+
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${serialize(v, `${path}.${k}`)}`).join(",")}}`;
|
|
474
|
+
}
|
|
475
|
+
function unwrapToJson(value, path) {
|
|
476
|
+
let current = value;
|
|
477
|
+
for (let depth = 0; depth < 8; depth++) {
|
|
478
|
+
if (current === null || typeof current !== "object") return current;
|
|
479
|
+
const toJson = current.toJSON;
|
|
480
|
+
if (typeof toJson !== "function") return current;
|
|
481
|
+
current = toJson.call(current);
|
|
482
|
+
}
|
|
483
|
+
throw new Error(`toJSON() chain at ${path} does not terminate`);
|
|
484
|
+
}
|
|
485
|
+
function compareUtf16(a, b) {
|
|
486
|
+
const n = Math.min(a.length, b.length);
|
|
487
|
+
for (let i = 0; i < n; i++) {
|
|
488
|
+
const d = a.charCodeAt(i) - b.charCodeAt(i);
|
|
489
|
+
if (d !== 0) return d;
|
|
490
|
+
}
|
|
491
|
+
return a.length - b.length;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
// src/slug.ts
|
|
495
|
+
function documentSlug(title) {
|
|
496
|
+
const slug = title.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
497
|
+
return slug || "untitled";
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// src/node/ids.ts
|
|
501
|
+
import { createHash } from "crypto";
|
|
502
|
+
function chunkUid(documentId, ordinal, chunkText) {
|
|
503
|
+
const textHash = createHash("sha256").update(chunkText, "utf8").digest();
|
|
504
|
+
return createHash("sha256").update(documentId, "utf8").update(Buffer.from([0])).update(String(ordinal), "utf8").update(Buffer.from([0])).update(textHash).digest("hex").slice(0, 32);
|
|
505
|
+
}
|
|
506
|
+
function chunkContentHash(chunkText) {
|
|
507
|
+
return createHash("sha256").update(chunkText, "utf8").digest("hex");
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// src/node/file-hash.ts
|
|
511
|
+
import { createHash as createHash2 } from "crypto";
|
|
512
|
+
import { createReadStream } from "fs";
|
|
513
|
+
async function hashFileStreaming(path, maxBytes = Number.MAX_SAFE_INTEGER) {
|
|
514
|
+
const hasher = createHash2("sha256");
|
|
515
|
+
let sizeBytes = 0;
|
|
516
|
+
await new Promise((resolve, reject) => {
|
|
517
|
+
const stream = createReadStream(path, { highWaterMark: 4 * 1024 * 1024 });
|
|
518
|
+
stream.on("data", (chunk) => {
|
|
519
|
+
const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
|
|
520
|
+
sizeBytes += bytes.byteLength;
|
|
521
|
+
if (sizeBytes > maxBytes) {
|
|
522
|
+
stream.destroy(new Error(`file exceeds ${maxBytes} byte limit`));
|
|
523
|
+
return;
|
|
524
|
+
}
|
|
525
|
+
hasher.update(bytes);
|
|
526
|
+
});
|
|
527
|
+
stream.on("error", reject);
|
|
528
|
+
stream.on("end", resolve);
|
|
529
|
+
});
|
|
530
|
+
return { sha256: hasher.digest("hex"), sizeBytes };
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
// src/node/signing.ts
|
|
534
|
+
import {
|
|
535
|
+
createHash as createHash3,
|
|
536
|
+
createPrivateKey,
|
|
537
|
+
createPublicKey,
|
|
538
|
+
sign as cryptoSign,
|
|
539
|
+
verify as cryptoVerify,
|
|
540
|
+
generateKeyPairSync
|
|
541
|
+
} from "crypto";
|
|
542
|
+
function knowledgeKeyId(publicKeyPem) {
|
|
543
|
+
const der = createPublicKey(publicKeyPem).export({ type: "spki", format: "der" });
|
|
544
|
+
return createHash3("sha256").update(der).digest("hex").slice(0, 16);
|
|
545
|
+
}
|
|
546
|
+
function generateKnowledgeSigningKeyPair() {
|
|
547
|
+
const { publicKey, privateKey } = generateKeyPairSync("ed25519");
|
|
548
|
+
const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }).toString();
|
|
549
|
+
return {
|
|
550
|
+
publicKeyPem,
|
|
551
|
+
privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
|
|
552
|
+
keyId: knowledgeKeyId(publicKeyPem)
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
function manifestSigningPayload(manifest) {
|
|
556
|
+
const { signature: _omitted, ...unsigned } = manifest;
|
|
557
|
+
return Buffer.from(canonicalizeJson(unsigned), "utf8");
|
|
558
|
+
}
|
|
559
|
+
function signManifest(manifest, privateKeyPem) {
|
|
560
|
+
const key = createPrivateKey(privateKeyPem);
|
|
561
|
+
const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
|
|
562
|
+
const value = cryptoSign(null, manifestSigningPayload(manifest), key).toString("base64");
|
|
563
|
+
return {
|
|
564
|
+
...manifest,
|
|
565
|
+
signature: {
|
|
566
|
+
algorithm: "ed25519",
|
|
567
|
+
keyId: knowledgeKeyId(publicKeyPem),
|
|
568
|
+
canonicalization: "rfc8785",
|
|
569
|
+
value
|
|
570
|
+
}
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
function verifyManifestSignature(manifest, anchors) {
|
|
574
|
+
return verifySignedDocument(manifest, anchors);
|
|
575
|
+
}
|
|
576
|
+
function signRegistryIndex(index, privateKeyPem) {
|
|
577
|
+
const key = createPrivateKey(privateKeyPem);
|
|
578
|
+
const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
|
|
579
|
+
const { signature: _omitted, ...unsigned } = index;
|
|
580
|
+
const value = cryptoSign(null, Buffer.from(canonicalizeJson(unsigned), "utf8"), key).toString(
|
|
581
|
+
"base64"
|
|
582
|
+
);
|
|
583
|
+
return {
|
|
584
|
+
...index,
|
|
585
|
+
signature: {
|
|
586
|
+
algorithm: "ed25519",
|
|
587
|
+
keyId: knowledgeKeyId(publicKeyPem),
|
|
588
|
+
canonicalization: "rfc8785",
|
|
589
|
+
value
|
|
590
|
+
}
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
function verifyRegistryIndex(index, anchors) {
|
|
594
|
+
return verifySignedDocument(index, anchors);
|
|
595
|
+
}
|
|
596
|
+
function verifySignedDocument(document, anchors) {
|
|
597
|
+
const signature = document.signature;
|
|
598
|
+
if (!signature) return { ok: false, reason: "unsigned" };
|
|
599
|
+
const anchor = anchors.find((a) => a.keyId === signature.keyId);
|
|
600
|
+
if (!anchor) {
|
|
601
|
+
return { ok: false, reason: "unknown-key", detail: `no trust anchor for ${signature.keyId}` };
|
|
602
|
+
}
|
|
603
|
+
try {
|
|
604
|
+
if (knowledgeKeyId(anchor.publicKeyPem) !== signature.keyId) {
|
|
605
|
+
return { ok: false, reason: "unknown-key", detail: "anchor keyId does not match its key" };
|
|
606
|
+
}
|
|
607
|
+
const { signature: _omitted, ...unsigned } = document;
|
|
608
|
+
const ok = cryptoVerify(
|
|
609
|
+
null,
|
|
610
|
+
Buffer.from(canonicalizeJson(unsigned), "utf8"),
|
|
611
|
+
createPublicKey(anchor.publicKeyPem),
|
|
612
|
+
Buffer.from(signature.value, "base64")
|
|
613
|
+
);
|
|
614
|
+
return ok ? { ok: true, keyId: signature.keyId } : { ok: false, reason: "bad-signature" };
|
|
615
|
+
} catch (err) {
|
|
616
|
+
return { ok: false, reason: "error", detail: err instanceof Error ? err.message : String(err) };
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
export {
|
|
620
|
+
ArtifactDigestSchema,
|
|
621
|
+
BODY_CODEC_MIN_BYTES,
|
|
622
|
+
CatalogDocumentSchema,
|
|
623
|
+
DEFAULT_EMBEDDING_ONNX_FILE,
|
|
624
|
+
DEFAULT_EMBEDDING_TOKENIZER_FILE,
|
|
625
|
+
GEZK_APPLICATION_ID,
|
|
626
|
+
GEZK_FORMAT_VERSION,
|
|
627
|
+
GEZK_INDEX_SCHEMA_VERSION,
|
|
628
|
+
GEZK_MANIFEST_KIND,
|
|
629
|
+
GEZK_MIME_TYPE,
|
|
630
|
+
GEZK_REGISTRY_KIND,
|
|
631
|
+
KNOWLEDGE_ID_PATTERN,
|
|
632
|
+
KNOWLEDGE_VERSION_PATTERN,
|
|
633
|
+
KnowledgeCatalogManifestSchema,
|
|
634
|
+
KnowledgeChunkUidSchema,
|
|
635
|
+
KnowledgeChunkingProfileSchema,
|
|
636
|
+
KnowledgeDocumentIdSchema,
|
|
637
|
+
KnowledgeEmbeddingProfileSchema,
|
|
638
|
+
KnowledgeIdSchema,
|
|
639
|
+
KnowledgeManifestFileSchema,
|
|
640
|
+
KnowledgeRegistryEntrySchema,
|
|
641
|
+
KnowledgeRegistryIndexSchema,
|
|
642
|
+
KnowledgeSignatureSchema,
|
|
643
|
+
KnowledgeVectorEncodingSchema,
|
|
644
|
+
KnowledgeVersionSchema,
|
|
645
|
+
LICENSE_NOTICE_PATH,
|
|
646
|
+
MANIFEST_PATH,
|
|
647
|
+
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
648
|
+
MIMETYPE_PATH,
|
|
649
|
+
README_PATH,
|
|
650
|
+
ROUTER_DB_PATH,
|
|
651
|
+
ROUTER_DDL,
|
|
652
|
+
RepoRelativePathSchema,
|
|
653
|
+
SHARD_DDL,
|
|
654
|
+
SOURCE_NOTICES_PATH,
|
|
655
|
+
Sha256HexSchema,
|
|
656
|
+
SourceNoticesSchema,
|
|
657
|
+
ZIP_FIXED_MTIME,
|
|
658
|
+
canonicalizeJson,
|
|
659
|
+
chunkContentHash,
|
|
660
|
+
chunkUid,
|
|
661
|
+
documentSlug,
|
|
662
|
+
embeddingProfileArtifacts,
|
|
663
|
+
formatKnowledgeUri,
|
|
664
|
+
generateKnowledgeSigningKeyPair,
|
|
665
|
+
hashFileStreaming,
|
|
666
|
+
knowledgeKeyId,
|
|
667
|
+
l2Normalize,
|
|
668
|
+
manifestSigningPayload,
|
|
669
|
+
parseKnowledgeUri,
|
|
670
|
+
quantizeBinary,
|
|
671
|
+
quantizeInt8,
|
|
672
|
+
rerankScore,
|
|
673
|
+
sameVectorSpace,
|
|
674
|
+
signManifest,
|
|
675
|
+
signRegistryIndex,
|
|
676
|
+
verifyManifestSignature,
|
|
677
|
+
verifyRegistryIndex
|
|
678
|
+
};
|