@bendyline/gezk 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +213 -21
- package/dist/index.js +368 -118
- package/dist/node.d.ts +1 -1
- package/dist/node.js +368 -118
- package/package.json +1 -1
package/dist/node.js
CHANGED
|
@@ -28,7 +28,7 @@ var RepoRelativePathSchema = z2.string().min(1).refine(
|
|
|
28
28
|
);
|
|
29
29
|
var DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
|
|
30
30
|
var DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
|
|
31
|
-
var
|
|
31
|
+
var KnowledgeEmbeddingProfileObject = z2.object({
|
|
32
32
|
/** e.g. `multilingual-e5-small@1`. A new model, prefix or encoding is a new id. */
|
|
33
33
|
id: z2.string().min(1),
|
|
34
34
|
model: z2.object({
|
|
@@ -69,13 +69,59 @@ var KnowledgeEmbeddingProfileSchema = z2.object({
|
|
|
69
69
|
method: z2.literal("symmetric-linear"),
|
|
70
70
|
scale: z2.literal(127)
|
|
71
71
|
}),
|
|
72
|
+
/**
|
|
73
|
+
* The stage-1 sign bits. `sign` takes them straight from the unit
|
|
74
|
+
* vector. `centered-sign` takes them from `vector − center`, where
|
|
75
|
+
* `center` is a fixed vector the profile pins (length = `dimensions`).
|
|
76
|
+
*
|
|
77
|
+
* Why a center exists: some embedding models put most of every vector
|
|
78
|
+
* on one shared direction (multilingual-e5-small: unrelated passages
|
|
79
|
+
* sit at cosine ≈ 0.8 and the corpus mean has norm ≈ 0.87). The sign
|
|
80
|
+
* of a dimension is then decided by that shared component rather than
|
|
81
|
+
* by the passage, hamming distance stops tracking cosine, and stage 1
|
|
82
|
+
* discards the true neighbours before the int8 rerank can see them.
|
|
83
|
+
* Subtracting the center first restores the correlation. The int8
|
|
84
|
+
* vectors are untouched either way, so query vectors stay comparable
|
|
85
|
+
* across both methods; only how the bits are derived differs.
|
|
86
|
+
*/
|
|
72
87
|
binary: z2.object({
|
|
73
|
-
method: z2.
|
|
88
|
+
method: z2.enum(["sign", "centered-sign"]),
|
|
74
89
|
threshold: z2.literal(0),
|
|
75
|
-
packing: z2.literal("lsb-first")
|
|
90
|
+
packing: z2.literal("lsb-first"),
|
|
91
|
+
/** Required by `centered-sign`, refused by `sign`; one entry per dimension. */
|
|
92
|
+
center: z2.array(z2.number()).optional()
|
|
76
93
|
})
|
|
77
94
|
})
|
|
78
95
|
});
|
|
96
|
+
function embeddingProfileCenterProblem(profile) {
|
|
97
|
+
const { method, center } = profile.quantization.binary;
|
|
98
|
+
if (method === "centered-sign") {
|
|
99
|
+
if (!center) return "centered-sign requires quantization.binary.center";
|
|
100
|
+
if (center.length !== profile.dimensions) {
|
|
101
|
+
return `quantization.binary.center has ${center.length} entries, dimensions is ${profile.dimensions}`;
|
|
102
|
+
}
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
return center ? "quantization.binary.center is only valid with method centered-sign" : null;
|
|
106
|
+
}
|
|
107
|
+
function embeddingProfileCenter(profile) {
|
|
108
|
+
const problem = embeddingProfileCenterProblem(profile);
|
|
109
|
+
if (problem) throw new Error(`invalid embedding profile ${profile.id}: ${problem}`);
|
|
110
|
+
const { center } = profile.quantization.binary;
|
|
111
|
+
return center ? Float32Array.from(center) : null;
|
|
112
|
+
}
|
|
113
|
+
var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefine(
|
|
114
|
+
(profile, ctx) => {
|
|
115
|
+
const problem = embeddingProfileCenterProblem(profile);
|
|
116
|
+
if (problem) {
|
|
117
|
+
ctx.addIssue({
|
|
118
|
+
code: "custom",
|
|
119
|
+
message: problem,
|
|
120
|
+
path: ["quantization", "binary", "center"]
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
);
|
|
79
125
|
function embeddingProfileArtifacts(profile) {
|
|
80
126
|
return {
|
|
81
127
|
onnxFile: profile.model.onnxFile ?? DEFAULT_EMBEDDING_ONNX_FILE,
|
|
@@ -88,7 +134,7 @@ function sameVectorSpace(a, b) {
|
|
|
88
134
|
const aa = embeddingProfileArtifacts(a);
|
|
89
135
|
const ab = embeddingProfileArtifacts(b);
|
|
90
136
|
const digestsAgree = (x, y) => !x || !y || x === y;
|
|
91
|
-
return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale
|
|
137
|
+
return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale;
|
|
92
138
|
}
|
|
93
139
|
var KnowledgeChunkingProfileSchema = z2.object({
|
|
94
140
|
/** e.g. `markdown-chunks@2`. */
|
|
@@ -101,12 +147,12 @@ var KnowledgeChunkingProfileSchema = z2.object({
|
|
|
101
147
|
});
|
|
102
148
|
|
|
103
149
|
// src/schemas/manifest.ts
|
|
104
|
-
import { z as
|
|
150
|
+
import { z as z4 } from "zod";
|
|
105
151
|
|
|
106
152
|
// src/format/constants.ts
|
|
107
|
-
var GEZK_FORMAT_VERSION = "0.
|
|
108
|
-
var GEZK_INDEX_SCHEMA_VERSION =
|
|
109
|
-
var GEZK_FORMAT_GENERATIONS = { "0.5": 2, "0.6": 3 };
|
|
153
|
+
var GEZK_FORMAT_VERSION = "0.7";
|
|
154
|
+
var GEZK_INDEX_SCHEMA_VERSION = 4;
|
|
155
|
+
var GEZK_FORMAT_GENERATIONS = { "0.5": 2, "0.6": 3, "0.7": 4 };
|
|
110
156
|
var GEZK_SUPPORTED_FORMAT_VERSIONS = Object.keys(GEZK_FORMAT_GENERATIONS);
|
|
111
157
|
var GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS = Object.values(GEZK_FORMAT_GENERATIONS);
|
|
112
158
|
function isSupportedFormatVersion(value) {
|
|
@@ -129,106 +175,250 @@ var MAX_KNOWLEDGE_DOCUMENT_META_BYTES = 16 * 1024;
|
|
|
129
175
|
var MAX_KNOWLEDGE_TOPIC_DEPTH = 16;
|
|
130
176
|
var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
|
|
131
177
|
|
|
178
|
+
// src/spatial.ts
|
|
179
|
+
import { z as z3 } from "zod";
|
|
180
|
+
var GEZK_EARTH_RADIUS_METERS = 6371e3;
|
|
181
|
+
var MAX_DOCUMENT_LOCATIONS = 256;
|
|
182
|
+
var KnowledgeLocationSchema = z3.object({
|
|
183
|
+
id: KnowledgeDocumentIdSchema,
|
|
184
|
+
latitude: z3.number().finite().min(-90).max(90),
|
|
185
|
+
longitude: z3.number().finite().min(-180).max(180),
|
|
186
|
+
role: z3.enum(["subject", "associated"]),
|
|
187
|
+
provenance: z3.record(z3.string(), z3.unknown()).optional()
|
|
188
|
+
}).strict();
|
|
189
|
+
var KnowledgeLocationsSchema = z3.array(KnowledgeLocationSchema).max(MAX_DOCUMENT_LOCATIONS).superRefine((locations, ctx) => {
|
|
190
|
+
const ids = /* @__PURE__ */ new Set();
|
|
191
|
+
for (const [index, location] of locations.entries()) {
|
|
192
|
+
if (ids.has(location.id))
|
|
193
|
+
ctx.addIssue({ code: "custom", path: [index, "id"], message: "duplicate location id" });
|
|
194
|
+
ids.add(location.id);
|
|
195
|
+
}
|
|
196
|
+
});
|
|
197
|
+
var KnowledgeRadiusSchema = z3.object({
|
|
198
|
+
latitude: z3.number().finite().min(-90).max(90),
|
|
199
|
+
longitude: z3.number().finite().min(-180).max(180),
|
|
200
|
+
radiusMeters: z3.number().finite().nonnegative()
|
|
201
|
+
}).strict();
|
|
202
|
+
var KnowledgeBoundsSchema = z3.object({
|
|
203
|
+
west: z3.number().finite().min(-180).max(180),
|
|
204
|
+
south: z3.number().finite().min(-90).max(90),
|
|
205
|
+
east: z3.number().finite().min(-180).max(180),
|
|
206
|
+
north: z3.number().finite().min(-90).max(90)
|
|
207
|
+
}).strict().refine((b) => b.west <= b.east && b.south <= b.north, "bounds must not wrap");
|
|
208
|
+
var KnowledgeSpatialManifestSchema = z3.object({
|
|
209
|
+
schema: z3.literal("document-points@1"),
|
|
210
|
+
crs: z3.literal("EPSG:4326"),
|
|
211
|
+
distanceModel: z3.literal("sphere-6371000"),
|
|
212
|
+
locationCount: z3.number().int().nonnegative(),
|
|
213
|
+
locatedDocuments: z3.number().int().nonnegative(),
|
|
214
|
+
coverage: z3.array(KnowledgeBoundsSchema).max(2)
|
|
215
|
+
}).strict().refine((s) => s.locatedDocuments <= s.locationCount, "located documents exceed locations");
|
|
216
|
+
function normalizeLongitude(longitude) {
|
|
217
|
+
return longitude === 180 ? -180 : longitude === 0 ? 0 : longitude;
|
|
218
|
+
}
|
|
219
|
+
function locationDistanceMeters(from, to) {
|
|
220
|
+
if (from.latitude === to.latitude && (Math.abs(from.latitude) === 90 || normalizeLongitude(from.longitude) === normalizeLongitude(to.longitude)))
|
|
221
|
+
return 0;
|
|
222
|
+
const radians = Math.PI / 180;
|
|
223
|
+
const dlat = (to.latitude - from.latitude) * radians;
|
|
224
|
+
const dlon = (normalizeLongitude(to.longitude) - normalizeLongitude(from.longitude)) * radians;
|
|
225
|
+
const a = Math.sin(dlat / 2) ** 2 + Math.cos(from.latitude * radians) * Math.cos(to.latitude * radians) * Math.sin(dlon / 2) ** 2;
|
|
226
|
+
const clamped = Math.max(0, Math.min(1, a));
|
|
227
|
+
return 2 * GEZK_EARTH_RADIUS_METERS * Math.atan2(Math.sqrt(clamped), Math.sqrt(1 - clamped));
|
|
228
|
+
}
|
|
229
|
+
function radiusBounds(raw) {
|
|
230
|
+
const radius = KnowledgeRadiusSchema.parse(raw);
|
|
231
|
+
if (radius.radiusMeters === 0) {
|
|
232
|
+
const longitude = normalizeLongitude(radius.longitude);
|
|
233
|
+
return [
|
|
234
|
+
{
|
|
235
|
+
west: Math.abs(radius.latitude) === 90 ? -180 : longitude,
|
|
236
|
+
east: Math.abs(radius.latitude) === 90 ? 180 : longitude,
|
|
237
|
+
south: radius.latitude,
|
|
238
|
+
north: radius.latitude
|
|
239
|
+
}
|
|
240
|
+
];
|
|
241
|
+
}
|
|
242
|
+
const radians = Math.PI / 180;
|
|
243
|
+
const delta = Math.min(Math.PI, radius.radiusMeters / GEZK_EARTH_RADIUS_METERS);
|
|
244
|
+
const lat = radius.latitude * radians;
|
|
245
|
+
const padding = radius.radiusMeters > 0 ? 1e-10 : 0;
|
|
246
|
+
const south = Math.max(-90, (lat - delta) / radians - padding);
|
|
247
|
+
const north = Math.min(90, (lat + delta) / radians + padding);
|
|
248
|
+
if (south <= -90 || north >= 90) return [{ west: -180, east: 180, south, north }];
|
|
249
|
+
const width = Math.asin(Math.min(1, Math.sin(delta) / Math.cos(lat))) / radians + padding;
|
|
250
|
+
const lon = normalizeLongitude(radius.longitude);
|
|
251
|
+
const west = lon - width;
|
|
252
|
+
const east = lon + width;
|
|
253
|
+
if (west < -180)
|
|
254
|
+
return [
|
|
255
|
+
{ west: west + 360, east: 180, south, north },
|
|
256
|
+
{ west: -180, east, south, north }
|
|
257
|
+
];
|
|
258
|
+
if (east >= 180)
|
|
259
|
+
return [
|
|
260
|
+
{ west, east: 180, south, north },
|
|
261
|
+
{ west: -180, east: east - 360, south, north }
|
|
262
|
+
];
|
|
263
|
+
return [{ west, east, south, north }];
|
|
264
|
+
}
|
|
265
|
+
function locationInBounds(location, bounds) {
|
|
266
|
+
const longitude = normalizeLongitude(location.longitude);
|
|
267
|
+
return location.latitude >= bounds.south && location.latitude <= bounds.north && longitude >= bounds.west && longitude <= bounds.east;
|
|
268
|
+
}
|
|
269
|
+
function spatialManifest(locations) {
|
|
270
|
+
const subjects = locations.filter((r) => r.location.role === "subject");
|
|
271
|
+
const coverage = [];
|
|
272
|
+
if (subjects.length) {
|
|
273
|
+
const latitudes = subjects.map((r) => r.location.latitude);
|
|
274
|
+
const longitudes = [
|
|
275
|
+
...new Set(subjects.map((r) => normalizeLongitude(r.location.longitude)))
|
|
276
|
+
].sort((a, b) => a - b);
|
|
277
|
+
let gap = -1;
|
|
278
|
+
let startIndex = 0;
|
|
279
|
+
for (let i = 0; i < longitudes.length; i++) {
|
|
280
|
+
const next = longitudes[(i + 1) % longitudes.length] + (i === longitudes.length - 1 ? 360 : 0);
|
|
281
|
+
if (next - longitudes[i] > gap) {
|
|
282
|
+
gap = next - longitudes[i];
|
|
283
|
+
startIndex = (i + 1) % longitudes.length;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
let south = 90;
|
|
287
|
+
let north = -90;
|
|
288
|
+
for (const latitude of latitudes) {
|
|
289
|
+
south = Math.min(south, latitude);
|
|
290
|
+
north = Math.max(north, latitude);
|
|
291
|
+
}
|
|
292
|
+
const west = longitudes[startIndex];
|
|
293
|
+
const east = longitudes[(startIndex + longitudes.length - 1) % longitudes.length];
|
|
294
|
+
if (west <= east) coverage.push({ west, east, south, north });
|
|
295
|
+
else coverage.push({ west, east: 180, south, north }, { west: -180, east, south, north });
|
|
296
|
+
}
|
|
297
|
+
return {
|
|
298
|
+
schema: "document-points@1",
|
|
299
|
+
crs: "EPSG:4326",
|
|
300
|
+
distanceModel: "sphere-6371000",
|
|
301
|
+
locationCount: locations.length,
|
|
302
|
+
locatedDocuments: new Set(locations.map((r) => r.documentId)).size,
|
|
303
|
+
coverage
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
|
|
132
307
|
// src/schemas/manifest.ts
|
|
133
308
|
var GEZK_MANIFEST_KIND = "gezk-catalog";
|
|
134
309
|
var GEZK_REGISTRY_KIND = "gezk-registry";
|
|
135
|
-
var KnowledgeManifestFileSchema =
|
|
136
|
-
path:
|
|
137
|
-
sizeBytes:
|
|
310
|
+
var KnowledgeManifestFileSchema = z4.object({
|
|
311
|
+
path: z4.string().min(1),
|
|
312
|
+
sizeBytes: z4.number().int().nonnegative(),
|
|
138
313
|
sha256: Sha256HexSchema
|
|
139
314
|
});
|
|
140
|
-
var KnowledgeSignatureSchema =
|
|
141
|
-
algorithm:
|
|
142
|
-
keyId:
|
|
143
|
-
canonicalization:
|
|
144
|
-
value:
|
|
315
|
+
var KnowledgeSignatureSchema = z4.object({
|
|
316
|
+
algorithm: z4.literal("ed25519"),
|
|
317
|
+
keyId: z4.string().min(1),
|
|
318
|
+
canonicalization: z4.literal("rfc8785"),
|
|
319
|
+
value: z4.string().min(1)
|
|
145
320
|
});
|
|
146
|
-
var KnowledgeCatalogManifestSchema =
|
|
147
|
-
kind:
|
|
148
|
-
formatVersion:
|
|
149
|
-
indexSchemaVersion:
|
|
321
|
+
var KnowledgeCatalogManifestSchema = z4.object({
|
|
322
|
+
kind: z4.literal(GEZK_MANIFEST_KIND),
|
|
323
|
+
formatVersion: z4.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
324
|
+
indexSchemaVersion: z4.union([z4.literal(2), z4.literal(3), z4.literal(4)]),
|
|
150
325
|
id: KnowledgeIdSchema,
|
|
151
326
|
version: KnowledgeVersionSchema,
|
|
152
|
-
name:
|
|
153
|
-
description:
|
|
154
|
-
language:
|
|
155
|
-
publisher:
|
|
327
|
+
name: z4.string().min(1),
|
|
328
|
+
description: z4.string().optional(),
|
|
329
|
+
language: z4.string().min(2),
|
|
330
|
+
publisher: z4.object({
|
|
156
331
|
id: KnowledgeIdSchema,
|
|
157
|
-
name:
|
|
158
|
-
url:
|
|
332
|
+
name: z4.string().min(1),
|
|
333
|
+
url: z4.string().optional()
|
|
159
334
|
}),
|
|
160
|
-
createdAt:
|
|
161
|
-
sourceSnapshot:
|
|
162
|
-
name:
|
|
163
|
-
date:
|
|
164
|
-
taxonomyVersion:
|
|
335
|
+
createdAt: z4.string(),
|
|
336
|
+
sourceSnapshot: z4.object({
|
|
337
|
+
name: z4.string(),
|
|
338
|
+
date: z4.string(),
|
|
339
|
+
taxonomyVersion: z4.string().optional()
|
|
165
340
|
}).optional(),
|
|
166
|
-
license:
|
|
167
|
-
name:
|
|
341
|
+
license: z4.object({
|
|
342
|
+
name: z4.string().min(1),
|
|
168
343
|
/** SPDX identifier when one exists (e.g. `CC-BY-SA-4.0`). */
|
|
169
|
-
spdx:
|
|
344
|
+
spdx: z4.string().optional(),
|
|
170
345
|
/** Archive path of the human-readable notice (`LICENSES/catalog.txt`). */
|
|
171
|
-
noticePath:
|
|
172
|
-
attributionRequired:
|
|
346
|
+
noticePath: z4.string().min(1),
|
|
347
|
+
attributionRequired: z4.boolean()
|
|
173
348
|
}),
|
|
174
349
|
embedding: KnowledgeEmbeddingProfileSchema,
|
|
175
350
|
chunking: KnowledgeChunkingProfileSchema,
|
|
176
|
-
topics:
|
|
177
|
-
|
|
351
|
+
topics: z4.array(
|
|
352
|
+
z4.object({
|
|
178
353
|
id: KnowledgeIdSchema,
|
|
179
|
-
name:
|
|
354
|
+
name: z4.string().min(1),
|
|
180
355
|
/** Absent for a root topic; names another entry of this list. */
|
|
181
356
|
parentId: KnowledgeIdSchema.optional(),
|
|
182
357
|
/** Sibling ordering key, identical to `topics.sort_key` in the router. */
|
|
183
|
-
sortKey:
|
|
184
|
-
description:
|
|
358
|
+
sortKey: z4.string().min(1).optional(),
|
|
359
|
+
description: z4.string().optional()
|
|
185
360
|
})
|
|
186
361
|
).min(1, "a catalog must ship a table of contents (at least one topic)"),
|
|
187
|
-
router:
|
|
188
|
-
shardTargetChunks:
|
|
189
|
-
shards:
|
|
190
|
-
|
|
191
|
-
id:
|
|
192
|
-
path:
|
|
193
|
-
chunks:
|
|
194
|
-
documents:
|
|
195
|
-
centroids:
|
|
362
|
+
router: z4.object({
|
|
363
|
+
shardTargetChunks: z4.number().int().positive(),
|
|
364
|
+
shards: z4.array(
|
|
365
|
+
z4.object({
|
|
366
|
+
id: z4.number().int().nonnegative(),
|
|
367
|
+
path: z4.string().min(1),
|
|
368
|
+
chunks: z4.number().int().nonnegative(),
|
|
369
|
+
documents: z4.number().int().nonnegative(),
|
|
370
|
+
centroids: z4.number().int().nonnegative(),
|
|
196
371
|
sha256: Sha256HexSchema
|
|
197
372
|
})
|
|
198
373
|
),
|
|
199
|
-
totalCentroids:
|
|
374
|
+
totalCentroids: z4.number().int().nonnegative()
|
|
200
375
|
}),
|
|
201
|
-
counts:
|
|
202
|
-
documents:
|
|
203
|
-
chunks:
|
|
204
|
-
shards:
|
|
376
|
+
counts: z4.object({
|
|
377
|
+
documents: z4.number().int().nonnegative(),
|
|
378
|
+
chunks: z4.number().int().nonnegative(),
|
|
379
|
+
shards: z4.number().int().positive(),
|
|
205
380
|
/** Files under `assets/` (0.6+). */
|
|
206
|
-
assets:
|
|
381
|
+
assets: z4.number().int().nonnegative().optional()
|
|
207
382
|
}),
|
|
208
|
-
files:
|
|
383
|
+
files: z4.array(KnowledgeManifestFileSchema).min(1),
|
|
384
|
+
spatial: KnowledgeSpatialManifestSchema.optional(),
|
|
209
385
|
/** What a reader must implement to open this catalog. */
|
|
210
|
-
requires:
|
|
211
|
-
formatVersion:
|
|
212
|
-
features:
|
|
386
|
+
requires: z4.object({
|
|
387
|
+
formatVersion: z4.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
388
|
+
features: z4.array(z4.string()).optional()
|
|
213
389
|
}),
|
|
214
|
-
smokeQueries:
|
|
215
|
-
|
|
216
|
-
query:
|
|
217
|
-
expectedDocumentIds:
|
|
390
|
+
smokeQueries: z4.array(
|
|
391
|
+
z4.object({
|
|
392
|
+
query: z4.string().min(1),
|
|
393
|
+
expectedDocumentIds: z4.array(KnowledgeDocumentIdSchema).min(1)
|
|
218
394
|
})
|
|
219
395
|
).optional(),
|
|
220
396
|
/** Which tool produced the archive — provenance, never a compatibility gate. */
|
|
221
|
-
toolchain:
|
|
222
|
-
name:
|
|
223
|
-
version:
|
|
224
|
-
node:
|
|
225
|
-
sqlite:
|
|
226
|
-
platform:
|
|
227
|
-
modelDigest:
|
|
228
|
-
tokenizerDigest:
|
|
397
|
+
toolchain: z4.object({
|
|
398
|
+
name: z4.string().min(1),
|
|
399
|
+
version: z4.string().min(1),
|
|
400
|
+
node: z4.string().optional(),
|
|
401
|
+
sqlite: z4.string().optional(),
|
|
402
|
+
platform: z4.string().optional(),
|
|
403
|
+
modelDigest: z4.string().optional(),
|
|
404
|
+
tokenizerDigest: z4.string().optional()
|
|
229
405
|
}).optional(),
|
|
230
406
|
signature: KnowledgeSignatureSchema.optional()
|
|
231
407
|
}).superRefine((manifest, ctx) => {
|
|
408
|
+
if (manifest.formatVersion === "0.7" && !manifest.spatial) {
|
|
409
|
+
ctx.addIssue({
|
|
410
|
+
code: "custom",
|
|
411
|
+
path: ["spatial"],
|
|
412
|
+
message: "0.7 requires a spatial summary, including catalogs with zero locations"
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
if (manifest.spatial && (manifest.formatVersion !== "0.7" || manifest.spatial.locatedDocuments > manifest.counts.documents)) {
|
|
416
|
+
ctx.addIssue({
|
|
417
|
+
code: "custom",
|
|
418
|
+
path: ["spatial"],
|
|
419
|
+
message: "spatial requires 0.7 and valid document counts"
|
|
420
|
+
});
|
|
421
|
+
}
|
|
232
422
|
if (GEZK_FORMAT_GENERATIONS[manifest.formatVersion] !== manifest.indexSchemaVersion) {
|
|
233
423
|
ctx.addIssue({
|
|
234
424
|
code: "custom",
|
|
@@ -272,75 +462,82 @@ var KnowledgeCatalogManifestSchema = z3.object({
|
|
|
272
462
|
});
|
|
273
463
|
});
|
|
274
464
|
var KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS = GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS;
|
|
275
|
-
var SourceNoticesSchema =
|
|
276
|
-
sources:
|
|
277
|
-
|
|
278
|
-
name:
|
|
279
|
-
url:
|
|
280
|
-
license:
|
|
281
|
-
licenseUrl:
|
|
465
|
+
var SourceNoticesSchema = z4.object({
|
|
466
|
+
sources: z4.array(
|
|
467
|
+
z4.object({
|
|
468
|
+
name: z4.string().min(1),
|
|
469
|
+
url: z4.string().optional(),
|
|
470
|
+
license: z4.string().min(1),
|
|
471
|
+
licenseUrl: z4.string().optional(),
|
|
282
472
|
/** Snapshot or revision the content was taken from. */
|
|
283
|
-
snapshot:
|
|
473
|
+
snapshot: z4.string().optional(),
|
|
284
474
|
/** Free-form notice text required by the source's license. */
|
|
285
|
-
notice:
|
|
475
|
+
notice: z4.string().optional()
|
|
286
476
|
})
|
|
287
477
|
).min(1)
|
|
288
478
|
});
|
|
289
479
|
|
|
290
480
|
// src/schemas/document.ts
|
|
291
|
-
import { z as
|
|
292
|
-
var KnowledgeOrdinalSchema =
|
|
293
|
-
var KnowledgeDocumentMetaSchema =
|
|
294
|
-
var
|
|
481
|
+
import { z as z5 } from "zod";
|
|
482
|
+
var KnowledgeOrdinalSchema = z5.number().int().min(-2147483648).max(2147483647);
|
|
483
|
+
var KnowledgeDocumentMetaSchema = z5.record(z5.string(), z5.unknown());
|
|
484
|
+
var KnowledgeTocReferenceSchema = z5.object({
|
|
485
|
+
topicPath: z5.array(KnowledgeIdSchema).min(1),
|
|
486
|
+
ordinal: KnowledgeOrdinalSchema.optional()
|
|
487
|
+
}).strict();
|
|
488
|
+
var CatalogDocumentSchema = z5.object({
|
|
295
489
|
id: KnowledgeDocumentIdSchema,
|
|
296
|
-
title:
|
|
297
|
-
slug:
|
|
298
|
-
summary:
|
|
299
|
-
language:
|
|
490
|
+
title: z5.string().min(1),
|
|
491
|
+
slug: z5.string().min(1),
|
|
492
|
+
summary: z5.string().optional(),
|
|
493
|
+
language: z5.string().min(2),
|
|
300
494
|
/**
|
|
301
495
|
* Root→leaf topic id path. Every segment must be a declared topic and each
|
|
302
496
|
* must be the parent of the next; the document is filed at the last one.
|
|
303
497
|
*/
|
|
304
|
-
topicPath:
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
498
|
+
topicPath: z5.array(KnowledgeIdSchema).min(1),
|
|
499
|
+
/** Additional root-to-leaf paths. Each leaf may list this document only once. */
|
|
500
|
+
tocReferences: z5.array(KnowledgeTocReferenceSchema).optional(),
|
|
501
|
+
markdown: z5.string(),
|
|
502
|
+
sourceUrl: z5.string().optional(),
|
|
503
|
+
sourceRevision: z5.string().optional(),
|
|
504
|
+
sourceUpdatedAt: z5.string().optional(),
|
|
505
|
+
attribution: z5.record(z5.string(), z5.string()).optional(),
|
|
506
|
+
aliases: z5.array(z5.string()).optional(),
|
|
311
507
|
/** Explicit position among the documents of its topic in a listing. */
|
|
312
508
|
ordinal: KnowledgeOrdinalSchema.optional(),
|
|
313
|
-
meta: KnowledgeDocumentMetaSchema.optional()
|
|
509
|
+
meta: KnowledgeDocumentMetaSchema.optional(),
|
|
510
|
+
locations: KnowledgeLocationsSchema.optional()
|
|
314
511
|
});
|
|
315
512
|
|
|
316
513
|
// src/schemas/registry.ts
|
|
317
|
-
import { z as
|
|
318
|
-
var KnowledgeRegistryEntrySchema =
|
|
514
|
+
import { z as z6 } from "zod";
|
|
515
|
+
var KnowledgeRegistryEntrySchema = z6.object({
|
|
319
516
|
catalogId: KnowledgeIdSchema,
|
|
320
|
-
version:
|
|
321
|
-
name:
|
|
322
|
-
description:
|
|
323
|
-
language:
|
|
324
|
-
documents:
|
|
517
|
+
version: z6.string().min(1),
|
|
518
|
+
name: z6.string().min(1),
|
|
519
|
+
description: z6.string().optional(),
|
|
520
|
+
language: z6.string().min(2),
|
|
521
|
+
documents: z6.number().int().nonnegative(),
|
|
325
522
|
/** Size of the `.gezk` archive itself (download accounting/preflight). */
|
|
326
|
-
archiveBytes:
|
|
523
|
+
archiveBytes: z6.number().int().nonnegative(),
|
|
327
524
|
/** sha256 of the `.gezk` archive — the ref's contentDigest after install. */
|
|
328
525
|
contentDigest: Sha256HexSchema,
|
|
329
526
|
/** Absolute download URL. Uploaded before the registry that names it. */
|
|
330
|
-
url:
|
|
331
|
-
license:
|
|
332
|
-
sourceSnapshot:
|
|
527
|
+
url: z6.string().url(),
|
|
528
|
+
license: z6.object({ name: z6.string().min(1), attributionRequired: z6.boolean() }),
|
|
529
|
+
sourceSnapshot: z6.object({ name: z6.string(), date: z6.string(), taxonomyVersion: z6.string().optional() }).optional()
|
|
333
530
|
});
|
|
334
|
-
var KnowledgeRegistryIndexSchema =
|
|
335
|
-
kind:
|
|
336
|
-
formatVersion:
|
|
337
|
-
publisher:
|
|
531
|
+
var KnowledgeRegistryIndexSchema = z6.object({
|
|
532
|
+
kind: z6.literal(GEZK_REGISTRY_KIND),
|
|
533
|
+
formatVersion: z6.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
534
|
+
publisher: z6.object({
|
|
338
535
|
id: KnowledgeIdSchema,
|
|
339
|
-
name:
|
|
340
|
-
url:
|
|
536
|
+
name: z6.string().min(1),
|
|
537
|
+
url: z6.string().optional()
|
|
341
538
|
}),
|
|
342
|
-
generatedAt:
|
|
343
|
-
catalogs:
|
|
539
|
+
generatedAt: z6.string(),
|
|
540
|
+
catalogs: z6.array(KnowledgeRegistryEntrySchema),
|
|
344
541
|
signature: KnowledgeSignatureSchema.optional()
|
|
345
542
|
});
|
|
346
543
|
|
|
@@ -424,6 +621,30 @@ CREATE TABLE documents(
|
|
|
424
621
|
CREATE INDEX documents_topic ON documents(topic_id, ordinal, slug);
|
|
425
622
|
CREATE INDEX documents_shard ON documents(shard_id);
|
|
426
623
|
|
|
624
|
+
CREATE TABLE document_locations(
|
|
625
|
+
document_id TEXT NOT NULL REFERENCES documents(id),
|
|
626
|
+
location_id TEXT NOT NULL,
|
|
627
|
+
latitude REAL NOT NULL CHECK (latitude BETWEEN -90 AND 90),
|
|
628
|
+
longitude REAL NOT NULL CHECK (longitude >= -180 AND longitude < 180),
|
|
629
|
+
role TEXT NOT NULL CHECK (role IN ('subject','associated')),
|
|
630
|
+
provenance_json TEXT,
|
|
631
|
+
PRIMARY KEY (document_id, location_id)
|
|
632
|
+
) WITHOUT ROWID;
|
|
633
|
+
CREATE INDEX document_locations_latitude ON document_locations(latitude, longitude, document_id);
|
|
634
|
+
CREATE INDEX document_locations_longitude ON document_locations(longitude, latitude, document_id);
|
|
635
|
+
|
|
636
|
+
-- Includes the primary placement in documents.topic_id/ordinal as well as
|
|
637
|
+
-- shared TOC references. Bodies, chunks and vectors remain keyed by document id.
|
|
638
|
+
CREATE TABLE topic_documents(
|
|
639
|
+
topic_id TEXT NOT NULL REFERENCES topics(id),
|
|
640
|
+
document_id TEXT NOT NULL REFERENCES documents(id),
|
|
641
|
+
ordinal INTEGER CHECK (ordinal IS NULL OR
|
|
642
|
+
(typeof(ordinal) = 'integer' AND ordinal BETWEEN -2147483648 AND 2147483647)),
|
|
643
|
+
PRIMARY KEY (topic_id, document_id)
|
|
644
|
+
) WITHOUT ROWID;
|
|
645
|
+
CREATE INDEX topic_documents_document ON topic_documents(document_id);
|
|
646
|
+
CREATE INDEX topic_documents_order ON topic_documents(topic_id, ordinal, document_id);
|
|
647
|
+
|
|
427
648
|
CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
|
|
428
649
|
PRIMARY KEY (alias, document_id)) WITHOUT ROWID;
|
|
429
650
|
|
|
@@ -485,7 +706,7 @@ CREATE TABLE chunk_vectors_int8(
|
|
|
485
706
|
`;
|
|
486
707
|
|
|
487
708
|
// src/format/assets.ts
|
|
488
|
-
import { z as
|
|
709
|
+
import { z as z7 } from "zod";
|
|
489
710
|
var ASSETS_PREFIX = "assets/";
|
|
490
711
|
var KNOWLEDGE_ASSET_TYPES = {
|
|
491
712
|
png: "image/png",
|
|
@@ -500,7 +721,7 @@ var MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES = 256 * 1024 * 1024;
|
|
|
500
721
|
var MAX_KNOWLEDGE_ASSET_COUNT = 8192;
|
|
501
722
|
var MAX_KNOWLEDGE_ASSET_PATH_LENGTH = 512;
|
|
502
723
|
var KNOWLEDGE_ASSET_PATH_PATTERN = /^assets\/(?:[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\/){0,15}[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\.(?:png|jpe?g|gif|webp|svg)$/i;
|
|
503
|
-
var KnowledgeAssetPathSchema =
|
|
724
|
+
var KnowledgeAssetPathSchema = z7.string().max(MAX_KNOWLEDGE_ASSET_PATH_LENGTH).regex(KNOWLEDGE_ASSET_PATH_PATTERN, "not a valid asset path");
|
|
504
725
|
function isKnowledgeAssetPath(path) {
|
|
505
726
|
return path.length <= MAX_KNOWLEDGE_ASSET_PATH_LENGTH && KNOWLEDGE_ASSET_PATH_PATTERN.test(path);
|
|
506
727
|
}
|
|
@@ -639,6 +860,18 @@ function quantizeInt8(unitVector) {
|
|
|
639
860
|
}
|
|
640
861
|
return out;
|
|
641
862
|
}
|
|
863
|
+
function centerVector(vector, center) {
|
|
864
|
+
if (vector.length !== center.length) {
|
|
865
|
+
throw new Error(`vector has ${vector.length} dimensions, center has ${center.length}`);
|
|
866
|
+
}
|
|
867
|
+
const out = new Float32Array(vector.length);
|
|
868
|
+
for (let i = 0; i < vector.length; i++) out[i] = vector[i] - center[i];
|
|
869
|
+
return out;
|
|
870
|
+
}
|
|
871
|
+
function quantizeBinaryForProfile(profile, unitVector) {
|
|
872
|
+
const center = embeddingProfileCenter(profile);
|
|
873
|
+
return quantizeBinary(center ? centerVector(unitVector, center) : unitVector);
|
|
874
|
+
}
|
|
642
875
|
function quantizeBinary(unitVector) {
|
|
643
876
|
const bytes = Math.ceil(unitVector.length / 8);
|
|
644
877
|
const out = new Uint8Array(bytes);
|
|
@@ -846,6 +1079,7 @@ export {
|
|
|
846
1079
|
DEFAULT_EMBEDDING_ONNX_FILE,
|
|
847
1080
|
DEFAULT_EMBEDDING_TOKENIZER_FILE,
|
|
848
1081
|
GEZK_APPLICATION_ID,
|
|
1082
|
+
GEZK_EARTH_RADIUS_METERS,
|
|
849
1083
|
GEZK_FORMAT_GENERATIONS,
|
|
850
1084
|
GEZK_FORMAT_VERSION,
|
|
851
1085
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
@@ -860,6 +1094,7 @@ export {
|
|
|
860
1094
|
KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS,
|
|
861
1095
|
KNOWLEDGE_VERSION_PATTERN,
|
|
862
1096
|
KnowledgeAssetPathSchema,
|
|
1097
|
+
KnowledgeBoundsSchema,
|
|
863
1098
|
KnowledgeCatalogManifestSchema,
|
|
864
1099
|
KnowledgeChunkUidSchema,
|
|
865
1100
|
KnowledgeChunkingProfileSchema,
|
|
@@ -867,15 +1102,21 @@ export {
|
|
|
867
1102
|
KnowledgeDocumentMetaSchema,
|
|
868
1103
|
KnowledgeEmbeddingProfileSchema,
|
|
869
1104
|
KnowledgeIdSchema,
|
|
1105
|
+
KnowledgeLocationSchema,
|
|
1106
|
+
KnowledgeLocationsSchema,
|
|
870
1107
|
KnowledgeManifestFileSchema,
|
|
871
1108
|
KnowledgeOrdinalSchema,
|
|
1109
|
+
KnowledgeRadiusSchema,
|
|
872
1110
|
KnowledgeRegistryEntrySchema,
|
|
873
1111
|
KnowledgeRegistryIndexSchema,
|
|
874
1112
|
KnowledgeSignatureSchema,
|
|
1113
|
+
KnowledgeSpatialManifestSchema,
|
|
1114
|
+
KnowledgeTocReferenceSchema,
|
|
875
1115
|
KnowledgeVectorEncodingSchema,
|
|
876
1116
|
KnowledgeVersionSchema,
|
|
877
1117
|
LICENSE_NOTICE_PATH,
|
|
878
1118
|
MANIFEST_PATH,
|
|
1119
|
+
MAX_DOCUMENT_LOCATIONS,
|
|
879
1120
|
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
|
|
880
1121
|
MAX_KNOWLEDGE_ASSET_BYTES,
|
|
881
1122
|
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
@@ -897,10 +1138,13 @@ export {
|
|
|
897
1138
|
assetExtension,
|
|
898
1139
|
assetKindForExtension,
|
|
899
1140
|
canonicalizeJson,
|
|
1141
|
+
centerVector,
|
|
900
1142
|
chunkContentHash,
|
|
901
1143
|
chunkUid,
|
|
902
1144
|
documentSlug,
|
|
903
1145
|
embeddingProfileArtifacts,
|
|
1146
|
+
embeddingProfileCenter,
|
|
1147
|
+
embeddingProfileCenterProblem,
|
|
904
1148
|
formatKnowledgeUri,
|
|
905
1149
|
generateKnowledgeSigningKeyPair,
|
|
906
1150
|
hashFileStreaming,
|
|
@@ -909,15 +1153,21 @@ export {
|
|
|
909
1153
|
isSupportedIndexSchemaVersion,
|
|
910
1154
|
knowledgeKeyId,
|
|
911
1155
|
l2Normalize,
|
|
1156
|
+
locationDistanceMeters,
|
|
1157
|
+
locationInBounds,
|
|
912
1158
|
manifestSigningPayload,
|
|
1159
|
+
normalizeLongitude,
|
|
913
1160
|
parseKnowledgeUri,
|
|
914
1161
|
quantizeBinary,
|
|
1162
|
+
quantizeBinaryForProfile,
|
|
915
1163
|
quantizeInt8,
|
|
1164
|
+
radiusBounds,
|
|
916
1165
|
rerankScore,
|
|
917
1166
|
sameVectorSpace,
|
|
918
1167
|
signManifest,
|
|
919
1168
|
signRegistryIndex,
|
|
920
1169
|
sniffAssetType,
|
|
1170
|
+
spatialManifest,
|
|
921
1171
|
svgInertnessProblem,
|
|
922
1172
|
topicSortKeyForOrder,
|
|
923
1173
|
verifyManifestSignature,
|