@bendyline/gezk 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +223 -24
- package/dist/index.js +325 -16
- package/dist/node.d.ts +1 -1
- package/dist/node.js +324 -15
- package/package.json +1 -1
package/dist/node.js
CHANGED
|
@@ -28,7 +28,7 @@ var RepoRelativePathSchema = z2.string().min(1).refine(
|
|
|
28
28
|
);
|
|
29
29
|
var DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
|
|
30
30
|
var DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
|
|
31
|
-
var
|
|
31
|
+
var KnowledgeEmbeddingProfileObject = z2.object({
|
|
32
32
|
/** e.g. `multilingual-e5-small@1`. A new model, prefix or encoding is a new id. */
|
|
33
33
|
id: z2.string().min(1),
|
|
34
34
|
model: z2.object({
|
|
@@ -69,13 +69,59 @@ var KnowledgeEmbeddingProfileSchema = z2.object({
|
|
|
69
69
|
method: z2.literal("symmetric-linear"),
|
|
70
70
|
scale: z2.literal(127)
|
|
71
71
|
}),
|
|
72
|
+
/**
|
|
73
|
+
* The stage-1 sign bits. `sign` takes them straight from the unit
|
|
74
|
+
* vector. `centered-sign` takes them from `vector − center`, where
|
|
75
|
+
* `center` is a fixed vector the profile pins (length = `dimensions`).
|
|
76
|
+
*
|
|
77
|
+
* Why a center exists: some embedding models put most of every vector
|
|
78
|
+
* on one shared direction (multilingual-e5-small: unrelated passages
|
|
79
|
+
* sit at cosine ≈ 0.8 and the corpus mean has norm ≈ 0.87). The sign
|
|
80
|
+
* of a dimension is then decided by that shared component rather than
|
|
81
|
+
* by the passage, hamming distance stops tracking cosine, and stage 1
|
|
82
|
+
* discards the true neighbours before the int8 rerank can see them.
|
|
83
|
+
* Subtracting the center first restores the correlation. The int8
|
|
84
|
+
* vectors are untouched either way, so query vectors stay comparable
|
|
85
|
+
* across both methods; only how the bits are derived differs.
|
|
86
|
+
*/
|
|
72
87
|
binary: z2.object({
|
|
73
|
-
method: z2.
|
|
88
|
+
method: z2.enum(["sign", "centered-sign"]),
|
|
74
89
|
threshold: z2.literal(0),
|
|
75
|
-
packing: z2.literal("lsb-first")
|
|
90
|
+
packing: z2.literal("lsb-first"),
|
|
91
|
+
/** Required by `centered-sign`, refused by `sign`; one entry per dimension. */
|
|
92
|
+
center: z2.array(z2.number()).optional()
|
|
76
93
|
})
|
|
77
94
|
})
|
|
78
95
|
});
|
|
96
|
+
function embeddingProfileCenterProblem(profile) {
|
|
97
|
+
const { method, center } = profile.quantization.binary;
|
|
98
|
+
if (method === "centered-sign") {
|
|
99
|
+
if (!center) return "centered-sign requires quantization.binary.center";
|
|
100
|
+
if (center.length !== profile.dimensions) {
|
|
101
|
+
return `quantization.binary.center has ${center.length} entries, dimensions is ${profile.dimensions}`;
|
|
102
|
+
}
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
return center ? "quantization.binary.center is only valid with method centered-sign" : null;
|
|
106
|
+
}
|
|
107
|
+
function embeddingProfileCenter(profile) {
|
|
108
|
+
const problem = embeddingProfileCenterProblem(profile);
|
|
109
|
+
if (problem) throw new Error(`invalid embedding profile ${profile.id}: ${problem}`);
|
|
110
|
+
const { center } = profile.quantization.binary;
|
|
111
|
+
return center ? Float32Array.from(center) : null;
|
|
112
|
+
}
|
|
113
|
+
var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefine(
|
|
114
|
+
(profile, ctx) => {
|
|
115
|
+
const problem = embeddingProfileCenterProblem(profile);
|
|
116
|
+
if (problem) {
|
|
117
|
+
ctx.addIssue({
|
|
118
|
+
code: "custom",
|
|
119
|
+
message: problem,
|
|
120
|
+
path: ["quantization", "binary", "center"]
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
);
|
|
79
125
|
function embeddingProfileArtifacts(profile) {
|
|
80
126
|
return {
|
|
81
127
|
onnxFile: profile.model.onnxFile ?? DEFAULT_EMBEDDING_ONNX_FILE,
|
|
@@ -88,7 +134,7 @@ function sameVectorSpace(a, b) {
|
|
|
88
134
|
const aa = embeddingProfileArtifacts(a);
|
|
89
135
|
const ab = embeddingProfileArtifacts(b);
|
|
90
136
|
const digestsAgree = (x, y) => !x || !y || x === y;
|
|
91
|
-
return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale
|
|
137
|
+
return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale;
|
|
92
138
|
}
|
|
93
139
|
var KnowledgeChunkingProfileSchema = z2.object({
|
|
94
140
|
/** e.g. `markdown-chunks@2`. */
|
|
@@ -104,8 +150,17 @@ var KnowledgeChunkingProfileSchema = z2.object({
|
|
|
104
150
|
import { z as z3 } from "zod";
|
|
105
151
|
|
|
106
152
|
// src/format/constants.ts
|
|
107
|
-
var GEZK_FORMAT_VERSION = "0.
|
|
108
|
-
var GEZK_INDEX_SCHEMA_VERSION =
|
|
153
|
+
var GEZK_FORMAT_VERSION = "0.6";
|
|
154
|
+
var GEZK_INDEX_SCHEMA_VERSION = 3;
|
|
155
|
+
var GEZK_FORMAT_GENERATIONS = { "0.5": 2, "0.6": 3 };
|
|
156
|
+
var GEZK_SUPPORTED_FORMAT_VERSIONS = Object.keys(GEZK_FORMAT_GENERATIONS);
|
|
157
|
+
var GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS = Object.values(GEZK_FORMAT_GENERATIONS);
|
|
158
|
+
function isSupportedFormatVersion(value) {
|
|
159
|
+
return typeof value === "string" && Object.hasOwn(GEZK_FORMAT_GENERATIONS, value);
|
|
160
|
+
}
|
|
161
|
+
function isSupportedIndexSchemaVersion(value) {
|
|
162
|
+
return typeof value === "number" && GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.includes(value);
|
|
163
|
+
}
|
|
109
164
|
var GEZK_APPLICATION_ID = 1195727435;
|
|
110
165
|
var GEZK_MIME_TYPE = "application/vnd.gezk+zip";
|
|
111
166
|
var MIMETYPE_PATH = "mimetype";
|
|
@@ -116,6 +171,8 @@ var SOURCE_NOTICES_PATH = "LICENSES/source-notices.json";
|
|
|
116
171
|
var ROUTER_DB_PATH = "index/router.db";
|
|
117
172
|
var BODY_CODEC_MIN_BYTES = 512;
|
|
118
173
|
var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
|
|
174
|
+
var MAX_KNOWLEDGE_DOCUMENT_META_BYTES = 16 * 1024;
|
|
175
|
+
var MAX_KNOWLEDGE_TOPIC_DEPTH = 16;
|
|
119
176
|
var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
|
|
120
177
|
|
|
121
178
|
// src/schemas/manifest.ts
|
|
@@ -134,8 +191,8 @@ var KnowledgeSignatureSchema = z3.object({
|
|
|
134
191
|
});
|
|
135
192
|
var KnowledgeCatalogManifestSchema = z3.object({
|
|
136
193
|
kind: z3.literal(GEZK_MANIFEST_KIND),
|
|
137
|
-
formatVersion: z3.
|
|
138
|
-
indexSchemaVersion: z3.literal(
|
|
194
|
+
formatVersion: z3.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
195
|
+
indexSchemaVersion: z3.union([z3.literal(2), z3.literal(3)]),
|
|
139
196
|
id: KnowledgeIdSchema,
|
|
140
197
|
version: KnowledgeVersionSchema,
|
|
141
198
|
name: z3.string().min(1),
|
|
@@ -162,7 +219,17 @@ var KnowledgeCatalogManifestSchema = z3.object({
|
|
|
162
219
|
}),
|
|
163
220
|
embedding: KnowledgeEmbeddingProfileSchema,
|
|
164
221
|
chunking: KnowledgeChunkingProfileSchema,
|
|
165
|
-
topics: z3.array(
|
|
222
|
+
topics: z3.array(
|
|
223
|
+
z3.object({
|
|
224
|
+
id: KnowledgeIdSchema,
|
|
225
|
+
name: z3.string().min(1),
|
|
226
|
+
/** Absent for a root topic; names another entry of this list. */
|
|
227
|
+
parentId: KnowledgeIdSchema.optional(),
|
|
228
|
+
/** Sibling ordering key, identical to `topics.sort_key` in the router. */
|
|
229
|
+
sortKey: z3.string().min(1).optional(),
|
|
230
|
+
description: z3.string().optional()
|
|
231
|
+
})
|
|
232
|
+
).min(1, "a catalog must ship a table of contents (at least one topic)"),
|
|
166
233
|
router: z3.object({
|
|
167
234
|
shardTargetChunks: z3.number().int().positive(),
|
|
168
235
|
shards: z3.array(
|
|
@@ -180,12 +247,14 @@ var KnowledgeCatalogManifestSchema = z3.object({
|
|
|
180
247
|
counts: z3.object({
|
|
181
248
|
documents: z3.number().int().nonnegative(),
|
|
182
249
|
chunks: z3.number().int().nonnegative(),
|
|
183
|
-
shards: z3.number().int().positive()
|
|
250
|
+
shards: z3.number().int().positive(),
|
|
251
|
+
/** Files under `assets/` (0.6+). */
|
|
252
|
+
assets: z3.number().int().nonnegative().optional()
|
|
184
253
|
}),
|
|
185
254
|
files: z3.array(KnowledgeManifestFileSchema).min(1),
|
|
186
255
|
/** What a reader must implement to open this catalog. */
|
|
187
256
|
requires: z3.object({
|
|
188
|
-
formatVersion: z3.
|
|
257
|
+
formatVersion: z3.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
189
258
|
features: z3.array(z3.string()).optional()
|
|
190
259
|
}),
|
|
191
260
|
smokeQueries: z3.array(
|
|
@@ -205,7 +274,50 @@ var KnowledgeCatalogManifestSchema = z3.object({
|
|
|
205
274
|
tokenizerDigest: z3.string().optional()
|
|
206
275
|
}).optional(),
|
|
207
276
|
signature: KnowledgeSignatureSchema.optional()
|
|
277
|
+
}).superRefine((manifest, ctx) => {
|
|
278
|
+
if (GEZK_FORMAT_GENERATIONS[manifest.formatVersion] !== manifest.indexSchemaVersion) {
|
|
279
|
+
ctx.addIssue({
|
|
280
|
+
code: "custom",
|
|
281
|
+
path: ["indexSchemaVersion"],
|
|
282
|
+
message: `format ${manifest.formatVersion} pairs with index schema ${GEZK_FORMAT_GENERATIONS[manifest.formatVersion]}, not ${manifest.indexSchemaVersion}`
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
if (manifest.requires.formatVersion !== manifest.formatVersion) {
|
|
286
|
+
ctx.addIssue({
|
|
287
|
+
code: "custom",
|
|
288
|
+
path: ["requires", "formatVersion"],
|
|
289
|
+
message: "requires.formatVersion must equal formatVersion"
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
if (manifest.formatVersion === "0.5" && manifest.counts.assets !== void 0) {
|
|
293
|
+
ctx.addIssue({
|
|
294
|
+
code: "custom",
|
|
295
|
+
path: ["counts", "assets"],
|
|
296
|
+
message: "counts.assets is a 0.6 field"
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
const topicIds = /* @__PURE__ */ new Set();
|
|
300
|
+
manifest.topics.forEach((topic, index) => {
|
|
301
|
+
if (topicIds.has(topic.id)) {
|
|
302
|
+
ctx.addIssue({
|
|
303
|
+
code: "custom",
|
|
304
|
+
path: ["topics", index, "id"],
|
|
305
|
+
message: `duplicate topic id '${topic.id}'`
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
topicIds.add(topic.id);
|
|
309
|
+
});
|
|
310
|
+
manifest.topics.forEach((topic, index) => {
|
|
311
|
+
if (topic.parentId !== void 0 && !topicIds.has(topic.parentId)) {
|
|
312
|
+
ctx.addIssue({
|
|
313
|
+
code: "custom",
|
|
314
|
+
path: ["topics", index, "parentId"],
|
|
315
|
+
message: `topic '${topic.id}' names an undeclared parent '${topic.parentId}'`
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
});
|
|
208
319
|
});
|
|
320
|
+
var KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS = GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS;
|
|
209
321
|
var SourceNoticesSchema = z3.object({
|
|
210
322
|
sources: z3.array(
|
|
211
323
|
z3.object({
|
|
@@ -223,20 +335,28 @@ var SourceNoticesSchema = z3.object({
|
|
|
223
335
|
|
|
224
336
|
// src/schemas/document.ts
|
|
225
337
|
import { z as z4 } from "zod";
|
|
338
|
+
var KnowledgeOrdinalSchema = z4.number().int().min(-2147483648).max(2147483647);
|
|
339
|
+
var KnowledgeDocumentMetaSchema = z4.record(z4.string(), z4.unknown());
|
|
226
340
|
var CatalogDocumentSchema = z4.object({
|
|
227
341
|
id: KnowledgeDocumentIdSchema,
|
|
228
342
|
title: z4.string().min(1),
|
|
229
343
|
slug: z4.string().min(1),
|
|
230
344
|
summary: z4.string().optional(),
|
|
231
345
|
language: z4.string().min(2),
|
|
232
|
-
/**
|
|
346
|
+
/**
|
|
347
|
+
* Root→leaf topic id path. Every segment must be a declared topic and each
|
|
348
|
+
* must be the parent of the next; the document is filed at the last one.
|
|
349
|
+
*/
|
|
233
350
|
topicPath: z4.array(KnowledgeIdSchema).min(1),
|
|
234
351
|
markdown: z4.string(),
|
|
235
352
|
sourceUrl: z4.string().optional(),
|
|
236
353
|
sourceRevision: z4.string().optional(),
|
|
237
354
|
sourceUpdatedAt: z4.string().optional(),
|
|
238
355
|
attribution: z4.record(z4.string(), z4.string()).optional(),
|
|
239
|
-
aliases: z4.array(z4.string()).optional()
|
|
356
|
+
aliases: z4.array(z4.string()).optional(),
|
|
357
|
+
/** Explicit position among the documents of its topic in a listing. */
|
|
358
|
+
ordinal: KnowledgeOrdinalSchema.optional(),
|
|
359
|
+
meta: KnowledgeDocumentMetaSchema.optional()
|
|
240
360
|
});
|
|
241
361
|
|
|
242
362
|
// src/schemas/registry.ts
|
|
@@ -259,7 +379,7 @@ var KnowledgeRegistryEntrySchema = z5.object({
|
|
|
259
379
|
});
|
|
260
380
|
var KnowledgeRegistryIndexSchema = z5.object({
|
|
261
381
|
kind: z5.literal(GEZK_REGISTRY_KIND),
|
|
262
|
-
formatVersion: z5.
|
|
382
|
+
formatVersion: z5.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
263
383
|
publisher: z5.object({
|
|
264
384
|
id: KnowledgeIdSchema,
|
|
265
385
|
name: z5.string().min(1),
|
|
@@ -338,14 +458,16 @@ CREATE TABLE documents(
|
|
|
338
458
|
id TEXT PRIMARY KEY,
|
|
339
459
|
title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
|
|
340
460
|
language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
|
|
461
|
+
ordinal INTEGER,
|
|
341
462
|
shard_id INTEGER NOT NULL,
|
|
342
463
|
chunk_count INTEGER NOT NULL,
|
|
343
464
|
source_url TEXT, source_revision TEXT, source_updated_at TEXT,
|
|
344
465
|
attribution_json TEXT,
|
|
466
|
+
meta_json TEXT,
|
|
345
467
|
body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
|
|
346
468
|
body_blob BLOB NOT NULL
|
|
347
469
|
);
|
|
348
|
-
CREATE INDEX documents_topic ON documents(topic_id, slug);
|
|
470
|
+
CREATE INDEX documents_topic ON documents(topic_id, ordinal, slug);
|
|
349
471
|
CREATE INDEX documents_shard ON documents(shard_id);
|
|
350
472
|
|
|
351
473
|
CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
|
|
@@ -408,6 +530,152 @@ CREATE TABLE chunk_vectors_int8(
|
|
|
408
530
|
);
|
|
409
531
|
`;
|
|
410
532
|
|
|
533
|
+
// src/format/assets.ts
|
|
534
|
+
import { z as z6 } from "zod";
|
|
535
|
+
var ASSETS_PREFIX = "assets/";
|
|
536
|
+
var KNOWLEDGE_ASSET_TYPES = {
|
|
537
|
+
png: "image/png",
|
|
538
|
+
jpg: "image/jpeg",
|
|
539
|
+
jpeg: "image/jpeg",
|
|
540
|
+
gif: "image/gif",
|
|
541
|
+
webp: "image/webp",
|
|
542
|
+
svg: "image/svg+xml"
|
|
543
|
+
};
|
|
544
|
+
var MAX_KNOWLEDGE_ASSET_BYTES = 8 * 1024 * 1024;
|
|
545
|
+
var MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES = 256 * 1024 * 1024;
|
|
546
|
+
var MAX_KNOWLEDGE_ASSET_COUNT = 8192;
|
|
547
|
+
var MAX_KNOWLEDGE_ASSET_PATH_LENGTH = 512;
|
|
548
|
+
var KNOWLEDGE_ASSET_PATH_PATTERN = /^assets\/(?:[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\/){0,15}[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\.(?:png|jpe?g|gif|webp|svg)$/i;
|
|
549
|
+
var KnowledgeAssetPathSchema = z6.string().max(MAX_KNOWLEDGE_ASSET_PATH_LENGTH).regex(KNOWLEDGE_ASSET_PATH_PATTERN, "not a valid asset path");
|
|
550
|
+
function isKnowledgeAssetPath(path) {
|
|
551
|
+
return path.length <= MAX_KNOWLEDGE_ASSET_PATH_LENGTH && KNOWLEDGE_ASSET_PATH_PATTERN.test(path);
|
|
552
|
+
}
|
|
553
|
+
function assetExtension(path) {
|
|
554
|
+
const dot = path.lastIndexOf(".");
|
|
555
|
+
if (dot < 0) return null;
|
|
556
|
+
const ext = path.slice(dot + 1).toLowerCase();
|
|
557
|
+
return Object.hasOwn(KNOWLEDGE_ASSET_TYPES, ext) ? ext : null;
|
|
558
|
+
}
|
|
559
|
+
function assetContentType(path) {
|
|
560
|
+
const ext = assetExtension(path);
|
|
561
|
+
return ext ? KNOWLEDGE_ASSET_TYPES[ext] : null;
|
|
562
|
+
}
|
|
563
|
+
function assetKindForExtension(ext) {
|
|
564
|
+
return ext === "jpg" ? "jpeg" : ext;
|
|
565
|
+
}
|
|
566
|
+
var PNG_MAGIC = [137, 80, 78, 71, 13, 10, 26, 10];
|
|
567
|
+
var JPEG_MAGIC = [255, 216, 255];
|
|
568
|
+
var GIF87 = [71, 73, 70, 56, 55, 97];
|
|
569
|
+
var GIF89 = [71, 73, 70, 56, 57, 97];
|
|
570
|
+
var RIFF = [82, 73, 70, 70];
|
|
571
|
+
var WEBP = [87, 69, 66, 80];
|
|
572
|
+
function startsWith(bytes, magic, offset = 0) {
|
|
573
|
+
if (bytes.length < offset + magic.length) return false;
|
|
574
|
+
for (let i = 0; i < magic.length; i++) {
|
|
575
|
+
if (bytes[offset + i] !== magic[i]) return false;
|
|
576
|
+
}
|
|
577
|
+
return true;
|
|
578
|
+
}
|
|
579
|
+
function sniffAssetType(bytes) {
|
|
580
|
+
if (startsWith(bytes, PNG_MAGIC)) return "png";
|
|
581
|
+
if (startsWith(bytes, JPEG_MAGIC)) return "jpeg";
|
|
582
|
+
if (startsWith(bytes, GIF87) || startsWith(bytes, GIF89)) return "gif";
|
|
583
|
+
if (startsWith(bytes, RIFF) && startsWith(bytes, WEBP, 8)) return "webp";
|
|
584
|
+
return looksLikeSvg(decodeText(bytes)) ? "svg" : null;
|
|
585
|
+
}
|
|
586
|
+
function decodeText(bytes) {
|
|
587
|
+
try {
|
|
588
|
+
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes);
|
|
589
|
+
} catch {
|
|
590
|
+
return "";
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
function looksLikeSvg(text) {
|
|
594
|
+
let i = 0;
|
|
595
|
+
const n = text.length;
|
|
596
|
+
while (i < n) {
|
|
597
|
+
const ch = text.charCodeAt(i);
|
|
598
|
+
if (ch === 32 || ch === 9 || ch === 10 || ch === 13 || ch === 65279) {
|
|
599
|
+
i++;
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
if (text.startsWith("<?", i)) {
|
|
603
|
+
const end = text.indexOf("?>", i);
|
|
604
|
+
if (end < 0) return false;
|
|
605
|
+
i = end + 2;
|
|
606
|
+
continue;
|
|
607
|
+
}
|
|
608
|
+
if (text.startsWith("<!--", i)) {
|
|
609
|
+
const end = text.indexOf("-->", i);
|
|
610
|
+
if (end < 0) return false;
|
|
611
|
+
i = end + 3;
|
|
612
|
+
continue;
|
|
613
|
+
}
|
|
614
|
+
if (/^<!doctype/i.test(text.slice(i, i + 9))) {
|
|
615
|
+
const bracket = text.indexOf("[", i);
|
|
616
|
+
const close = text.indexOf(">", i);
|
|
617
|
+
if (close < 0) return false;
|
|
618
|
+
if (bracket >= 0 && bracket < close) {
|
|
619
|
+
const end = text.indexOf("]>", bracket);
|
|
620
|
+
if (end < 0) return false;
|
|
621
|
+
i = end + 2;
|
|
622
|
+
} else {
|
|
623
|
+
i = close + 1;
|
|
624
|
+
}
|
|
625
|
+
continue;
|
|
626
|
+
}
|
|
627
|
+
return /^<svg[\s>/]/i.test(text.slice(i, i + 5));
|
|
628
|
+
}
|
|
629
|
+
return false;
|
|
630
|
+
}
|
|
631
|
+
var SVG_ACTIVE_PATTERNS = [
|
|
632
|
+
[/<script[\s>/]/i, "a <script> element"],
|
|
633
|
+
[/<foreignobject[\s>/]/i, "a <foreignObject> element"],
|
|
634
|
+
[/<!entity/i, "an entity declaration"],
|
|
635
|
+
[/\son[a-z]+\s*=/i, "an event-handler attribute"],
|
|
636
|
+
[/javascript\s*:/i, "a javascript: reference"],
|
|
637
|
+
[/data\s*:\s*text\/html/i, "a data:text/html reference"],
|
|
638
|
+
[/@import/i, "a CSS @import"]
|
|
639
|
+
];
|
|
640
|
+
var HREF_ATTRIBUTE = /(?:xlink:)?href\s*=\s*["']\s*([^"']*)["']/gi;
|
|
641
|
+
var CSS_URL = /url\(\s*["']?\s*([^"')]*)/gi;
|
|
642
|
+
function externalReferenceProblem(target) {
|
|
643
|
+
const value = target.trim();
|
|
644
|
+
if (value === "" || value.startsWith("#")) return null;
|
|
645
|
+
const scheme = /^([a-z][a-z0-9+.-]*):/i.exec(value);
|
|
646
|
+
if (!scheme) return null;
|
|
647
|
+
if (/^data:image\/(?:png|jpeg|gif|webp)[;,]/i.test(value)) return null;
|
|
648
|
+
return `a ${scheme[1]}: reference`;
|
|
649
|
+
}
|
|
650
|
+
function svgInertnessProblem(bytes) {
|
|
651
|
+
const text = decodeText(bytes);
|
|
652
|
+
if (text === "") return "not valid UTF-8";
|
|
653
|
+
if (!looksLikeSvg(text)) return "does not start with an <svg> element";
|
|
654
|
+
for (const [pattern, why] of SVG_ACTIVE_PATTERNS) {
|
|
655
|
+
if (pattern.test(text)) return `contains ${why}`;
|
|
656
|
+
}
|
|
657
|
+
for (const match of text.matchAll(HREF_ATTRIBUTE)) {
|
|
658
|
+
const problem = externalReferenceProblem(match[1] ?? "");
|
|
659
|
+
if (problem) return `contains ${problem} in an href`;
|
|
660
|
+
}
|
|
661
|
+
for (const match of text.matchAll(CSS_URL)) {
|
|
662
|
+
const problem = externalReferenceProblem(match[1] ?? "");
|
|
663
|
+
if (problem) return `contains ${problem} in a CSS url()`;
|
|
664
|
+
}
|
|
665
|
+
return null;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// src/format/sort-key.ts
|
|
669
|
+
var INT32_MIN = -2147483648;
|
|
670
|
+
var INT32_MAX = 2147483647;
|
|
671
|
+
var OFFSET = 2147483648;
|
|
672
|
+
function topicSortKeyForOrder(order) {
|
|
673
|
+
if (!Number.isInteger(order) || order < INT32_MIN || order > INT32_MAX) {
|
|
674
|
+
throw new RangeError(`topic order must be an int32: ${String(order)}`);
|
|
675
|
+
}
|
|
676
|
+
return String(order + OFFSET).padStart(10, "0");
|
|
677
|
+
}
|
|
678
|
+
|
|
411
679
|
// src/format/quantize.ts
|
|
412
680
|
function quantizeInt8(unitVector) {
|
|
413
681
|
const out = new Int8Array(unitVector.length);
|
|
@@ -417,6 +685,18 @@ function quantizeInt8(unitVector) {
|
|
|
417
685
|
}
|
|
418
686
|
return out;
|
|
419
687
|
}
|
|
688
|
+
function centerVector(vector, center) {
|
|
689
|
+
if (vector.length !== center.length) {
|
|
690
|
+
throw new Error(`vector has ${vector.length} dimensions, center has ${center.length}`);
|
|
691
|
+
}
|
|
692
|
+
const out = new Float32Array(vector.length);
|
|
693
|
+
for (let i = 0; i < vector.length; i++) out[i] = vector[i] - center[i];
|
|
694
|
+
return out;
|
|
695
|
+
}
|
|
696
|
+
function quantizeBinaryForProfile(profile, unitVector) {
|
|
697
|
+
const center = embeddingProfileCenter(profile);
|
|
698
|
+
return quantizeBinary(center ? centerVector(unitVector, center) : unitVector);
|
|
699
|
+
}
|
|
420
700
|
function quantizeBinary(unitVector) {
|
|
421
701
|
const bytes = Math.ceil(unitVector.length / 8);
|
|
422
702
|
const out = new Uint8Array(bytes);
|
|
@@ -617,26 +897,36 @@ function verifySignedDocument(document, anchors) {
|
|
|
617
897
|
}
|
|
618
898
|
}
|
|
619
899
|
export {
|
|
900
|
+
ASSETS_PREFIX,
|
|
620
901
|
ArtifactDigestSchema,
|
|
621
902
|
BODY_CODEC_MIN_BYTES,
|
|
622
903
|
CatalogDocumentSchema,
|
|
623
904
|
DEFAULT_EMBEDDING_ONNX_FILE,
|
|
624
905
|
DEFAULT_EMBEDDING_TOKENIZER_FILE,
|
|
625
906
|
GEZK_APPLICATION_ID,
|
|
907
|
+
GEZK_FORMAT_GENERATIONS,
|
|
626
908
|
GEZK_FORMAT_VERSION,
|
|
627
909
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
628
910
|
GEZK_MANIFEST_KIND,
|
|
629
911
|
GEZK_MIME_TYPE,
|
|
630
912
|
GEZK_REGISTRY_KIND,
|
|
913
|
+
GEZK_SUPPORTED_FORMAT_VERSIONS,
|
|
914
|
+
GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
|
|
915
|
+
KNOWLEDGE_ASSET_PATH_PATTERN,
|
|
916
|
+
KNOWLEDGE_ASSET_TYPES,
|
|
631
917
|
KNOWLEDGE_ID_PATTERN,
|
|
918
|
+
KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS,
|
|
632
919
|
KNOWLEDGE_VERSION_PATTERN,
|
|
920
|
+
KnowledgeAssetPathSchema,
|
|
633
921
|
KnowledgeCatalogManifestSchema,
|
|
634
922
|
KnowledgeChunkUidSchema,
|
|
635
923
|
KnowledgeChunkingProfileSchema,
|
|
636
924
|
KnowledgeDocumentIdSchema,
|
|
925
|
+
KnowledgeDocumentMetaSchema,
|
|
637
926
|
KnowledgeEmbeddingProfileSchema,
|
|
638
927
|
KnowledgeIdSchema,
|
|
639
928
|
KnowledgeManifestFileSchema,
|
|
929
|
+
KnowledgeOrdinalSchema,
|
|
640
930
|
KnowledgeRegistryEntrySchema,
|
|
641
931
|
KnowledgeRegistryIndexSchema,
|
|
642
932
|
KnowledgeSignatureSchema,
|
|
@@ -644,7 +934,13 @@ export {
|
|
|
644
934
|
KnowledgeVersionSchema,
|
|
645
935
|
LICENSE_NOTICE_PATH,
|
|
646
936
|
MANIFEST_PATH,
|
|
937
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
|
|
938
|
+
MAX_KNOWLEDGE_ASSET_BYTES,
|
|
939
|
+
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
940
|
+
MAX_KNOWLEDGE_ASSET_PATH_LENGTH,
|
|
647
941
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
942
|
+
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
943
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
648
944
|
MIMETYPE_PATH,
|
|
649
945
|
README_PATH,
|
|
650
946
|
ROUTER_DB_PATH,
|
|
@@ -655,24 +951,37 @@ export {
|
|
|
655
951
|
Sha256HexSchema,
|
|
656
952
|
SourceNoticesSchema,
|
|
657
953
|
ZIP_FIXED_MTIME,
|
|
954
|
+
assetContentType,
|
|
955
|
+
assetExtension,
|
|
956
|
+
assetKindForExtension,
|
|
658
957
|
canonicalizeJson,
|
|
958
|
+
centerVector,
|
|
659
959
|
chunkContentHash,
|
|
660
960
|
chunkUid,
|
|
661
961
|
documentSlug,
|
|
662
962
|
embeddingProfileArtifacts,
|
|
963
|
+
embeddingProfileCenter,
|
|
964
|
+
embeddingProfileCenterProblem,
|
|
663
965
|
formatKnowledgeUri,
|
|
664
966
|
generateKnowledgeSigningKeyPair,
|
|
665
967
|
hashFileStreaming,
|
|
968
|
+
isKnowledgeAssetPath,
|
|
969
|
+
isSupportedFormatVersion,
|
|
970
|
+
isSupportedIndexSchemaVersion,
|
|
666
971
|
knowledgeKeyId,
|
|
667
972
|
l2Normalize,
|
|
668
973
|
manifestSigningPayload,
|
|
669
974
|
parseKnowledgeUri,
|
|
670
975
|
quantizeBinary,
|
|
976
|
+
quantizeBinaryForProfile,
|
|
671
977
|
quantizeInt8,
|
|
672
978
|
rerankScore,
|
|
673
979
|
sameVectorSpace,
|
|
674
980
|
signManifest,
|
|
675
981
|
signRegistryIndex,
|
|
982
|
+
sniffAssetType,
|
|
983
|
+
svgInertnessProblem,
|
|
984
|
+
topicSortKeyForOrder,
|
|
676
985
|
verifyManifestSignature,
|
|
677
986
|
verifyRegistryIndex
|
|
678
987
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/gezk",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.2",
|
|
4
4
|
"description": "The gezk knowledge-catalog format: manifest and registry schemas, id grammars, knowledge:// references, vector quantization, SQLite DDL, canonical JSON, and Ed25519 manifest signatures.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|