@bendyline/gezk 1.0.2 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -28,6 +28,43 @@ var RepoRelativePathSchema = z2.string().min(1).refine(
28
28
  );
29
29
  var DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
30
30
  var DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
31
+ var EmbeddingModelFileSchema = z2.object({
32
+ path: RepoRelativePathSchema,
33
+ digest: ArtifactDigestSchema.optional()
34
+ });
35
+ var EmbeddingMediaEncoderSchema = z2.object({
36
+ onnxFile: RepoRelativePathSchema,
37
+ onnxDigest: ArtifactDigestSchema.optional(),
38
+ files: z2.array(EmbeddingModelFileSchema).optional()
39
+ });
40
+ var EmbeddingMediaSchema = z2.object({
41
+ image: z2.object({
42
+ encoder: EmbeddingMediaEncoderSchema,
43
+ /** Vision tokens per image; changing it moves every image vector. */
44
+ tokenBudget: z2.number().int().positive(),
45
+ resample: z2.enum(["bilinear", "bicubic"]),
46
+ /** Transparent pixels are composited onto white before encoding. */
47
+ alpha: z2.literal("composite-white")
48
+ }).optional(),
49
+ /** Video frames go through the image encoder, so `video` requires `image`. */
50
+ video: z2.object({
51
+ framesPerSecond: z2.number().positive(),
52
+ maxFrames: z2.number().int().positive(),
53
+ tokenBudgetPerFrame: z2.number().int().positive()
54
+ }).optional(),
55
+ audio: z2.object({
56
+ encoder: EmbeddingMediaEncoderSchema,
57
+ sampleRate: z2.number().int().positive(),
58
+ channels: z2.literal(1),
59
+ /** Longest window one audio row may cover. */
60
+ maxWindowMs: z2.number().int().positive()
61
+ }).optional()
62
+ }).refine((media) => Boolean(media.image || media.video || media.audio), {
63
+ message: "media must describe at least one modality"
64
+ }).refine((media) => !media.video || Boolean(media.image), {
65
+ message: "media.video requires media.image (frames go through the image encoder)",
66
+ path: ["video"]
67
+ });
31
68
  var KnowledgeEmbeddingProfileObject = z2.object({
32
69
  /** e.g. `multilingual-e5-small@1`. A new model, prefix or encoding is a new id. */
33
70
  id: z2.string().min(1),
@@ -44,7 +81,9 @@ var KnowledgeEmbeddingProfileObject = z2.object({
44
81
  */
45
82
  onnxFile: RepoRelativePathSchema.optional(),
46
83
  /** sha256 of that file's bytes at `revision`. */
47
- onnxDigest: ArtifactDigestSchema.optional()
84
+ onnxDigest: ArtifactDigestSchema.optional(),
85
+ /** Every further file the runtime loads: weight sidecars, configs (0.8). */
86
+ files: z2.array(EmbeddingModelFileSchema).optional()
48
87
  }),
49
88
  tokenizer: z2.object({
50
89
  kind: z2.string().min(1),
@@ -55,7 +94,17 @@ var KnowledgeEmbeddingProfileObject = z2.object({
55
94
  }),
56
95
  pooling: z2.enum(["mean", "cls", "last"]),
57
96
  normalized: z2.boolean(),
97
+ /** The stored vector width — after truncation, when the profile truncates. */
58
98
  dimensions: z2.number().int().positive(),
99
+ /**
100
+ * Matryoshka truncation (0.8): the model emits `sourceDimensions` values,
101
+ * the first `dimensions` are kept, and the result is L2-normalized again.
102
+ * One rule for passages, queries and media; `profileUnitVector` owns it.
103
+ */
104
+ truncation: z2.object({
105
+ method: z2.literal("prefix"),
106
+ sourceDimensions: z2.number().int().positive()
107
+ }).optional(),
59
108
  maxTokens: z2.number().int().positive(),
60
109
  queryInstruction: z2.string(),
61
110
  passageInstruction: z2.string(),
@@ -91,7 +140,8 @@ var KnowledgeEmbeddingProfileObject = z2.object({
91
140
  /** Required by `centered-sign`, refused by `sign`; one entry per dimension. */
92
141
  center: z2.array(z2.number()).optional()
93
142
  })
94
- })
143
+ }),
144
+ media: EmbeddingMediaSchema.optional()
95
145
  });
96
146
  function embeddingProfileCenterProblem(profile) {
97
147
  const { method, center } = profile.quantization.binary;
@@ -110,6 +160,19 @@ function embeddingProfileCenter(profile) {
110
160
  const { center } = profile.quantization.binary;
111
161
  return center ? Float32Array.from(center) : null;
112
162
  }
163
+ function embeddingProfileTruncationProblem(profile) {
164
+ if (!profile.truncation) return null;
165
+ if (profile.truncation.sourceDimensions <= profile.dimensions) {
166
+ return `truncation.sourceDimensions (${profile.truncation.sourceDimensions}) must exceed dimensions (${profile.dimensions})`;
167
+ }
168
+ return profile.normalized ? null : "truncation requires a normalized profile";
169
+ }
170
+ function embeddingProfileSourceDimensions(profile) {
171
+ return profile.truncation?.sourceDimensions ?? profile.dimensions;
172
+ }
173
+ function embeddingProfileMinimumFormat(profile) {
174
+ return profile.truncation || profile.model.files || profile.media ? "0.8" : "0.5";
175
+ }
113
176
  var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefine(
114
177
  (profile, ctx) => {
115
178
  const problem = embeddingProfileCenterProblem(profile);
@@ -120,6 +183,10 @@ var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefin
120
183
  path: ["quantization", "binary", "center"]
121
184
  });
122
185
  }
186
+ const truncation = embeddingProfileTruncationProblem(profile);
187
+ if (truncation) {
188
+ ctx.addIssue({ code: "custom", message: truncation, path: ["truncation"] });
189
+ }
123
190
  }
124
191
  );
125
192
  function embeddingProfileArtifacts(profile) {
@@ -127,14 +194,19 @@ function embeddingProfileArtifacts(profile) {
127
194
  onnxFile: profile.model.onnxFile ?? DEFAULT_EMBEDDING_ONNX_FILE,
128
195
  onnxDigest: profile.model.onnxDigest ?? null,
129
196
  tokenizerFile: profile.tokenizer.file ?? DEFAULT_EMBEDDING_TOKENIZER_FILE,
130
- tokenizerDigest: profile.tokenizer.digest ?? null
197
+ tokenizerDigest: profile.tokenizer.digest ?? null,
198
+ files: (profile.model.files ?? []).map((f) => ({ path: f.path, digest: f.digest ?? null }))
131
199
  };
132
200
  }
133
201
  function sameVectorSpace(a, b) {
134
202
  const aa = embeddingProfileArtifacts(a);
135
203
  const ab = embeddingProfileArtifacts(b);
136
204
  const digestsAgree = (x, y) => !x || !y || x === y;
137
- return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale;
205
+ const bFiles = new Map(ab.files.map((f) => [f.path, f.digest]));
206
+ const filesAgree = aa.files.every(
207
+ (f) => !bFiles.has(f.path) || digestsAgree(f.digest, bFiles.get(f.path) ?? null)
208
+ );
209
+ return filesAgree && a.truncation?.method === b.truncation?.method && embeddingProfileSourceDimensions(a) === embeddingProfileSourceDimensions(b) && a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale;
138
210
  }
139
211
  var KnowledgeChunkingProfileSchema = z2.object({
140
212
  /** e.g. `markdown-chunks@2`. */
@@ -147,17 +219,20 @@ var KnowledgeChunkingProfileSchema = z2.object({
147
219
  });
148
220
 
149
221
  // src/schemas/manifest.ts
150
- import { z as z3 } from "zod";
222
+ import { z as z4 } from "zod";
151
223
 
152
224
  // src/format/constants.ts
153
- var GEZK_FORMAT_VERSION = "0.6";
154
- var GEZK_INDEX_SCHEMA_VERSION = 3;
155
- var GEZK_FORMAT_GENERATIONS = { "0.5": 2, "0.6": 3 };
225
+ var GEZK_FORMAT_VERSION = "0.8";
226
+ var GEZK_INDEX_SCHEMA_VERSION = 5;
227
+ var GEZK_FORMAT_GENERATIONS = { "0.5": 2, "0.6": 3, "0.7": 4, "0.8": 5 };
156
228
  var GEZK_SUPPORTED_FORMAT_VERSIONS = Object.keys(GEZK_FORMAT_GENERATIONS);
157
229
  var GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS = Object.values(GEZK_FORMAT_GENERATIONS);
158
230
  function isSupportedFormatVersion(value) {
159
231
  return typeof value === "string" && Object.hasOwn(GEZK_FORMAT_GENERATIONS, value);
160
232
  }
233
+ function formatAtLeast(version, minimum) {
234
+ return GEZK_FORMAT_GENERATIONS[version] >= GEZK_FORMAT_GENERATIONS[minimum];
235
+ }
161
236
  function isSupportedIndexSchemaVersion(value) {
162
237
  return typeof value === "number" && GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.includes(value);
163
238
  }
@@ -175,106 +250,302 @@ var MAX_KNOWLEDGE_DOCUMENT_META_BYTES = 16 * 1024;
175
250
  var MAX_KNOWLEDGE_TOPIC_DEPTH = 16;
176
251
  var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
177
252
 
253
+ // src/spatial.ts
254
+ import { z as z3 } from "zod";
255
+ var GEZK_EARTH_RADIUS_METERS = 6371e3;
256
+ var MAX_DOCUMENT_LOCATIONS = 256;
257
+ var KnowledgeLocationSchema = z3.object({
258
+ id: KnowledgeDocumentIdSchema,
259
+ latitude: z3.number().finite().min(-90).max(90),
260
+ longitude: z3.number().finite().min(-180).max(180),
261
+ role: z3.enum(["subject", "associated"]),
262
+ provenance: z3.record(z3.string(), z3.unknown()).optional()
263
+ }).strict();
264
+ var KnowledgeLocationsSchema = z3.array(KnowledgeLocationSchema).max(MAX_DOCUMENT_LOCATIONS).superRefine((locations, ctx) => {
265
+ const ids = /* @__PURE__ */ new Set();
266
+ for (const [index, location] of locations.entries()) {
267
+ if (ids.has(location.id))
268
+ ctx.addIssue({ code: "custom", path: [index, "id"], message: "duplicate location id" });
269
+ ids.add(location.id);
270
+ }
271
+ });
272
+ var KnowledgeRadiusSchema = z3.object({
273
+ latitude: z3.number().finite().min(-90).max(90),
274
+ longitude: z3.number().finite().min(-180).max(180),
275
+ radiusMeters: z3.number().finite().nonnegative()
276
+ }).strict();
277
+ var KnowledgeBoundsSchema = z3.object({
278
+ west: z3.number().finite().min(-180).max(180),
279
+ south: z3.number().finite().min(-90).max(90),
280
+ east: z3.number().finite().min(-180).max(180),
281
+ north: z3.number().finite().min(-90).max(90)
282
+ }).strict().refine((b) => b.west <= b.east && b.south <= b.north, "bounds must not wrap");
283
+ var KnowledgeSpatialManifestSchema = z3.object({
284
+ schema: z3.literal("document-points@1"),
285
+ crs: z3.literal("EPSG:4326"),
286
+ distanceModel: z3.literal("sphere-6371000"),
287
+ locationCount: z3.number().int().nonnegative(),
288
+ locatedDocuments: z3.number().int().nonnegative(),
289
+ coverage: z3.array(KnowledgeBoundsSchema).max(2)
290
+ }).strict().refine((s) => s.locatedDocuments <= s.locationCount, "located documents exceed locations");
291
+ function normalizeLongitude(longitude) {
292
+ return longitude === 180 ? -180 : longitude === 0 ? 0 : longitude;
293
+ }
294
+ function locationDistanceMeters(from, to) {
295
+ if (from.latitude === to.latitude && (Math.abs(from.latitude) === 90 || normalizeLongitude(from.longitude) === normalizeLongitude(to.longitude)))
296
+ return 0;
297
+ const radians = Math.PI / 180;
298
+ const dlat = (to.latitude - from.latitude) * radians;
299
+ const dlon = (normalizeLongitude(to.longitude) - normalizeLongitude(from.longitude)) * radians;
300
+ const a = Math.sin(dlat / 2) ** 2 + Math.cos(from.latitude * radians) * Math.cos(to.latitude * radians) * Math.sin(dlon / 2) ** 2;
301
+ const clamped = Math.max(0, Math.min(1, a));
302
+ return 2 * GEZK_EARTH_RADIUS_METERS * Math.atan2(Math.sqrt(clamped), Math.sqrt(1 - clamped));
303
+ }
304
+ function radiusBounds(raw) {
305
+ const radius = KnowledgeRadiusSchema.parse(raw);
306
+ if (radius.radiusMeters === 0) {
307
+ const longitude = normalizeLongitude(radius.longitude);
308
+ return [
309
+ {
310
+ west: Math.abs(radius.latitude) === 90 ? -180 : longitude,
311
+ east: Math.abs(radius.latitude) === 90 ? 180 : longitude,
312
+ south: radius.latitude,
313
+ north: radius.latitude
314
+ }
315
+ ];
316
+ }
317
+ const radians = Math.PI / 180;
318
+ const delta = Math.min(Math.PI, radius.radiusMeters / GEZK_EARTH_RADIUS_METERS);
319
+ const lat = radius.latitude * radians;
320
+ const padding = radius.radiusMeters > 0 ? 1e-10 : 0;
321
+ const south = Math.max(-90, (lat - delta) / radians - padding);
322
+ const north = Math.min(90, (lat + delta) / radians + padding);
323
+ if (south <= -90 || north >= 90) return [{ west: -180, east: 180, south, north }];
324
+ const width = Math.asin(Math.min(1, Math.sin(delta) / Math.cos(lat))) / radians + padding;
325
+ const lon = normalizeLongitude(radius.longitude);
326
+ const west = lon - width;
327
+ const east = lon + width;
328
+ if (west < -180)
329
+ return [
330
+ { west: west + 360, east: 180, south, north },
331
+ { west: -180, east, south, north }
332
+ ];
333
+ if (east >= 180)
334
+ return [
335
+ { west, east: 180, south, north },
336
+ { west: -180, east: east - 360, south, north }
337
+ ];
338
+ return [{ west, east, south, north }];
339
+ }
340
+ function locationInBounds(location, bounds) {
341
+ const longitude = normalizeLongitude(location.longitude);
342
+ return location.latitude >= bounds.south && location.latitude <= bounds.north && longitude >= bounds.west && longitude <= bounds.east;
343
+ }
344
+ function spatialManifest(locations) {
345
+ const subjects = locations.filter((r) => r.location.role === "subject");
346
+ const coverage = [];
347
+ if (subjects.length) {
348
+ const latitudes = subjects.map((r) => r.location.latitude);
349
+ const longitudes = [
350
+ ...new Set(subjects.map((r) => normalizeLongitude(r.location.longitude)))
351
+ ].sort((a, b) => a - b);
352
+ let gap = -1;
353
+ let startIndex = 0;
354
+ for (let i = 0; i < longitudes.length; i++) {
355
+ const next = longitudes[(i + 1) % longitudes.length] + (i === longitudes.length - 1 ? 360 : 0);
356
+ if (next - longitudes[i] > gap) {
357
+ gap = next - longitudes[i];
358
+ startIndex = (i + 1) % longitudes.length;
359
+ }
360
+ }
361
+ let south = 90;
362
+ let north = -90;
363
+ for (const latitude of latitudes) {
364
+ south = Math.min(south, latitude);
365
+ north = Math.max(north, latitude);
366
+ }
367
+ const west = longitudes[startIndex];
368
+ const east = longitudes[(startIndex + longitudes.length - 1) % longitudes.length];
369
+ if (west <= east) coverage.push({ west, east, south, north });
370
+ else coverage.push({ west, east: 180, south, north }, { west: -180, east, south, north });
371
+ }
372
+ return {
373
+ schema: "document-points@1",
374
+ crs: "EPSG:4326",
375
+ distanceModel: "sphere-6371000",
376
+ locationCount: locations.length,
377
+ locatedDocuments: new Set(locations.map((r) => r.documentId)).size,
378
+ coverage
379
+ };
380
+ }
381
+
178
382
  // src/schemas/manifest.ts
179
383
  var GEZK_MANIFEST_KIND = "gezk-catalog";
180
384
  var GEZK_REGISTRY_KIND = "gezk-registry";
181
- var KnowledgeManifestFileSchema = z3.object({
182
- path: z3.string().min(1),
183
- sizeBytes: z3.number().int().nonnegative(),
385
+ var KnowledgeManifestFileSchema = z4.object({
386
+ path: z4.string().min(1),
387
+ sizeBytes: z4.number().int().nonnegative(),
184
388
  sha256: Sha256HexSchema
185
389
  });
186
- var KnowledgeSignatureSchema = z3.object({
187
- algorithm: z3.literal("ed25519"),
188
- keyId: z3.string().min(1),
189
- canonicalization: z3.literal("rfc8785"),
190
- value: z3.string().min(1)
390
+ var KnowledgeSignatureSchema = z4.object({
391
+ algorithm: z4.literal("ed25519"),
392
+ keyId: z4.string().min(1),
393
+ canonicalization: z4.literal("rfc8785"),
394
+ value: z4.string().min(1)
191
395
  });
192
- var KnowledgeCatalogManifestSchema = z3.object({
193
- kind: z3.literal(GEZK_MANIFEST_KIND),
194
- formatVersion: z3.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
195
- indexSchemaVersion: z3.union([z3.literal(2), z3.literal(3)]),
396
+ var KnowledgeCatalogManifestSchema = z4.object({
397
+ kind: z4.literal(GEZK_MANIFEST_KIND),
398
+ formatVersion: z4.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
399
+ indexSchemaVersion: z4.literal(GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS),
196
400
  id: KnowledgeIdSchema,
197
401
  version: KnowledgeVersionSchema,
198
- name: z3.string().min(1),
199
- description: z3.string().optional(),
200
- language: z3.string().min(2),
201
- publisher: z3.object({
402
+ name: z4.string().min(1),
403
+ description: z4.string().optional(),
404
+ language: z4.string().min(2),
405
+ publisher: z4.object({
202
406
  id: KnowledgeIdSchema,
203
- name: z3.string().min(1),
204
- url: z3.string().optional()
407
+ name: z4.string().min(1),
408
+ url: z4.string().optional()
205
409
  }),
206
- createdAt: z3.string(),
207
- sourceSnapshot: z3.object({
208
- name: z3.string(),
209
- date: z3.string(),
210
- taxonomyVersion: z3.string().optional()
410
+ createdAt: z4.string(),
411
+ sourceSnapshot: z4.object({
412
+ name: z4.string(),
413
+ date: z4.string(),
414
+ taxonomyVersion: z4.string().optional()
211
415
  }).optional(),
212
- license: z3.object({
213
- name: z3.string().min(1),
416
+ license: z4.object({
417
+ name: z4.string().min(1),
214
418
  /** SPDX identifier when one exists (e.g. `CC-BY-SA-4.0`). */
215
- spdx: z3.string().optional(),
419
+ spdx: z4.string().optional(),
216
420
  /** Archive path of the human-readable notice (`LICENSES/catalog.txt`). */
217
- noticePath: z3.string().min(1),
218
- attributionRequired: z3.boolean()
421
+ noticePath: z4.string().min(1),
422
+ attributionRequired: z4.boolean()
219
423
  }),
220
424
  embedding: KnowledgeEmbeddingProfileSchema,
221
425
  chunking: KnowledgeChunkingProfileSchema,
222
- topics: z3.array(
223
- z3.object({
426
+ topics: z4.array(
427
+ z4.object({
224
428
  id: KnowledgeIdSchema,
225
- name: z3.string().min(1),
429
+ name: z4.string().min(1),
226
430
  /** Absent for a root topic; names another entry of this list. */
227
431
  parentId: KnowledgeIdSchema.optional(),
228
432
  /** Sibling ordering key, identical to `topics.sort_key` in the router. */
229
- sortKey: z3.string().min(1).optional(),
230
- description: z3.string().optional()
433
+ sortKey: z4.string().min(1).optional(),
434
+ description: z4.string().optional()
231
435
  })
232
436
  ).min(1, "a catalog must ship a table of contents (at least one topic)"),
233
- router: z3.object({
234
- shardTargetChunks: z3.number().int().positive(),
235
- shards: z3.array(
236
- z3.object({
237
- id: z3.number().int().nonnegative(),
238
- path: z3.string().min(1),
239
- chunks: z3.number().int().nonnegative(),
240
- documents: z3.number().int().nonnegative(),
241
- centroids: z3.number().int().nonnegative(),
437
+ router: z4.object({
438
+ shardTargetChunks: z4.number().int().positive(),
439
+ shards: z4.array(
440
+ z4.object({
441
+ id: z4.number().int().nonnegative(),
442
+ path: z4.string().min(1),
443
+ chunks: z4.number().int().nonnegative(),
444
+ documents: z4.number().int().nonnegative(),
445
+ centroids: z4.number().int().nonnegative(),
242
446
  sha256: Sha256HexSchema
243
447
  })
244
448
  ),
245
- totalCentroids: z3.number().int().nonnegative()
449
+ totalCentroids: z4.number().int().nonnegative()
246
450
  }),
247
- counts: z3.object({
248
- documents: z3.number().int().nonnegative(),
249
- chunks: z3.number().int().nonnegative(),
250
- shards: z3.number().int().positive(),
451
+ counts: z4.object({
452
+ documents: z4.number().int().nonnegative(),
453
+ chunks: z4.number().int().nonnegative(),
454
+ shards: z4.number().int().positive(),
251
455
  /** Files under `assets/` (0.6+). */
252
- assets: z3.number().int().nonnegative().optional()
456
+ assets: z4.number().int().nonnegative().optional(),
457
+ /**
458
+ * Media rows by modality (0.8, required there). They are counted in
459
+ * `chunks` too: every row of the shard `chunks` table is a chunk.
460
+ */
461
+ media: z4.object({
462
+ image: z4.number().int().nonnegative(),
463
+ video: z4.number().int().nonnegative(),
464
+ audio: z4.number().int().nonnegative()
465
+ }).optional()
253
466
  }),
254
- files: z3.array(KnowledgeManifestFileSchema).min(1),
467
+ files: z4.array(KnowledgeManifestFileSchema).min(1),
468
+ spatial: KnowledgeSpatialManifestSchema.optional(),
255
469
  /** What a reader must implement to open this catalog. */
256
- requires: z3.object({
257
- formatVersion: z3.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
258
- features: z3.array(z3.string()).optional()
470
+ requires: z4.object({
471
+ formatVersion: z4.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
472
+ features: z4.array(z4.string()).optional()
259
473
  }),
260
- smokeQueries: z3.array(
261
- z3.object({
262
- query: z3.string().min(1),
263
- expectedDocumentIds: z3.array(KnowledgeDocumentIdSchema).min(1)
474
+ smokeQueries: z4.array(
475
+ z4.object({
476
+ query: z4.string().min(1),
477
+ expectedDocumentIds: z4.array(KnowledgeDocumentIdSchema).min(1)
264
478
  })
265
479
  ).optional(),
266
480
  /** Which tool produced the archive — provenance, never a compatibility gate. */
267
- toolchain: z3.object({
268
- name: z3.string().min(1),
269
- version: z3.string().min(1),
270
- node: z3.string().optional(),
271
- sqlite: z3.string().optional(),
272
- platform: z3.string().optional(),
273
- modelDigest: z3.string().optional(),
274
- tokenizerDigest: z3.string().optional()
481
+ toolchain: z4.object({
482
+ name: z4.string().min(1),
483
+ version: z4.string().min(1),
484
+ node: z4.string().optional(),
485
+ sqlite: z4.string().optional(),
486
+ platform: z4.string().optional(),
487
+ modelDigest: z4.string().optional(),
488
+ tokenizerDigest: z4.string().optional()
275
489
  }).optional(),
276
490
  signature: KnowledgeSignatureSchema.optional()
277
491
  }).superRefine((manifest, ctx) => {
492
+ const spatialEra = formatAtLeast(manifest.formatVersion, "0.7");
493
+ if (spatialEra && !manifest.spatial) {
494
+ ctx.addIssue({
495
+ code: "custom",
496
+ path: ["spatial"],
497
+ message: "0.7+ requires a spatial summary, including catalogs with zero locations"
498
+ });
499
+ }
500
+ if (manifest.spatial && (!spatialEra || manifest.spatial.locatedDocuments > manifest.counts.documents)) {
501
+ ctx.addIssue({
502
+ code: "custom",
503
+ path: ["spatial"],
504
+ message: "spatial requires 0.7+ and valid document counts"
505
+ });
506
+ }
507
+ const mediaEra = formatAtLeast(manifest.formatVersion, "0.8");
508
+ const media = manifest.counts.media;
509
+ if (mediaEra && !media) {
510
+ ctx.addIssue({
511
+ code: "custom",
512
+ path: ["counts", "media"],
513
+ message: "0.8 requires counts.media, including catalogs with no media rows"
514
+ });
515
+ }
516
+ if (media && !mediaEra) {
517
+ ctx.addIssue({
518
+ code: "custom",
519
+ path: ["counts", "media"],
520
+ message: "counts.media is a 0.8 field"
521
+ });
522
+ }
523
+ if (media && media.image + media.video + media.audio > manifest.counts.chunks) {
524
+ ctx.addIssue({
525
+ code: "custom",
526
+ path: ["counts", "media"],
527
+ message: "media rows are chunks, so counts.media cannot exceed counts.chunks"
528
+ });
529
+ }
530
+ if (media) {
531
+ for (const modality of ["image", "video", "audio"]) {
532
+ if (media[modality] > 0 && !manifest.embedding.media?.[modality]) {
533
+ ctx.addIssue({
534
+ code: "custom",
535
+ path: ["embedding", "media", modality],
536
+ message: `${media[modality]} ${modality} rows need embedding.media.${modality}`
537
+ });
538
+ }
539
+ }
540
+ }
541
+ const profileFormat = embeddingProfileMinimumFormat(manifest.embedding);
542
+ if (!formatAtLeast(manifest.formatVersion, profileFormat)) {
543
+ ctx.addIssue({
544
+ code: "custom",
545
+ path: ["embedding"],
546
+ message: `embedding profile ${manifest.embedding.id} uses ${profileFormat} vocabulary (truncation, model.files or media), which ${manifest.formatVersion} cannot carry`
547
+ });
548
+ }
278
549
  if (GEZK_FORMAT_GENERATIONS[manifest.formatVersion] !== manifest.indexSchemaVersion) {
279
550
  ctx.addIssue({
280
551
  code: "custom",
@@ -318,75 +589,82 @@ var KnowledgeCatalogManifestSchema = z3.object({
318
589
  });
319
590
  });
320
591
  var KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS = GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS;
321
- var SourceNoticesSchema = z3.object({
322
- sources: z3.array(
323
- z3.object({
324
- name: z3.string().min(1),
325
- url: z3.string().optional(),
326
- license: z3.string().min(1),
327
- licenseUrl: z3.string().optional(),
592
+ var SourceNoticesSchema = z4.object({
593
+ sources: z4.array(
594
+ z4.object({
595
+ name: z4.string().min(1),
596
+ url: z4.string().optional(),
597
+ license: z4.string().min(1),
598
+ licenseUrl: z4.string().optional(),
328
599
  /** Snapshot or revision the content was taken from. */
329
- snapshot: z3.string().optional(),
600
+ snapshot: z4.string().optional(),
330
601
  /** Free-form notice text required by the source's license. */
331
- notice: z3.string().optional()
602
+ notice: z4.string().optional()
332
603
  })
333
604
  ).min(1)
334
605
  });
335
606
 
336
607
  // src/schemas/document.ts
337
- import { z as z4 } from "zod";
338
- var KnowledgeOrdinalSchema = z4.number().int().min(-2147483648).max(2147483647);
339
- var KnowledgeDocumentMetaSchema = z4.record(z4.string(), z4.unknown());
340
- var CatalogDocumentSchema = z4.object({
608
+ import { z as z5 } from "zod";
609
+ var KnowledgeOrdinalSchema = z5.number().int().min(-2147483648).max(2147483647);
610
+ var KnowledgeDocumentMetaSchema = z5.record(z5.string(), z5.unknown());
611
+ var KnowledgeTocReferenceSchema = z5.object({
612
+ topicPath: z5.array(KnowledgeIdSchema).min(1),
613
+ ordinal: KnowledgeOrdinalSchema.optional()
614
+ }).strict();
615
+ var CatalogDocumentSchema = z5.object({
341
616
  id: KnowledgeDocumentIdSchema,
342
- title: z4.string().min(1),
343
- slug: z4.string().min(1),
344
- summary: z4.string().optional(),
345
- language: z4.string().min(2),
617
+ title: z5.string().min(1),
618
+ slug: z5.string().min(1),
619
+ summary: z5.string().optional(),
620
+ language: z5.string().min(2),
346
621
  /**
347
622
  * Root→leaf topic id path. Every segment must be a declared topic and each
348
623
  * must be the parent of the next; the document is filed at the last one.
349
624
  */
350
- topicPath: z4.array(KnowledgeIdSchema).min(1),
351
- markdown: z4.string(),
352
- sourceUrl: z4.string().optional(),
353
- sourceRevision: z4.string().optional(),
354
- sourceUpdatedAt: z4.string().optional(),
355
- attribution: z4.record(z4.string(), z4.string()).optional(),
356
- aliases: z4.array(z4.string()).optional(),
625
+ topicPath: z5.array(KnowledgeIdSchema).min(1),
626
+ /** Additional root-to-leaf paths. Each leaf may list this document only once. */
627
+ tocReferences: z5.array(KnowledgeTocReferenceSchema).optional(),
628
+ markdown: z5.string(),
629
+ sourceUrl: z5.string().optional(),
630
+ sourceRevision: z5.string().optional(),
631
+ sourceUpdatedAt: z5.string().optional(),
632
+ attribution: z5.record(z5.string(), z5.string()).optional(),
633
+ aliases: z5.array(z5.string()).optional(),
357
634
  /** Explicit position among the documents of its topic in a listing. */
358
635
  ordinal: KnowledgeOrdinalSchema.optional(),
359
- meta: KnowledgeDocumentMetaSchema.optional()
636
+ meta: KnowledgeDocumentMetaSchema.optional(),
637
+ locations: KnowledgeLocationsSchema.optional()
360
638
  });
361
639
 
362
640
  // src/schemas/registry.ts
363
- import { z as z5 } from "zod";
364
- var KnowledgeRegistryEntrySchema = z5.object({
641
+ import { z as z6 } from "zod";
642
+ var KnowledgeRegistryEntrySchema = z6.object({
365
643
  catalogId: KnowledgeIdSchema,
366
- version: z5.string().min(1),
367
- name: z5.string().min(1),
368
- description: z5.string().optional(),
369
- language: z5.string().min(2),
370
- documents: z5.number().int().nonnegative(),
644
+ version: z6.string().min(1),
645
+ name: z6.string().min(1),
646
+ description: z6.string().optional(),
647
+ language: z6.string().min(2),
648
+ documents: z6.number().int().nonnegative(),
371
649
  /** Size of the `.gezk` archive itself (download accounting/preflight). */
372
- archiveBytes: z5.number().int().nonnegative(),
650
+ archiveBytes: z6.number().int().nonnegative(),
373
651
  /** sha256 of the `.gezk` archive — the ref's contentDigest after install. */
374
652
  contentDigest: Sha256HexSchema,
375
653
  /** Absolute download URL. Uploaded before the registry that names it. */
376
- url: z5.string().url(),
377
- license: z5.object({ name: z5.string().min(1), attributionRequired: z5.boolean() }),
378
- sourceSnapshot: z5.object({ name: z5.string(), date: z5.string(), taxonomyVersion: z5.string().optional() }).optional()
654
+ url: z6.string().url(),
655
+ license: z6.object({ name: z6.string().min(1), attributionRequired: z6.boolean() }),
656
+ sourceSnapshot: z6.object({ name: z6.string(), date: z6.string(), taxonomyVersion: z6.string().optional() }).optional()
379
657
  });
380
- var KnowledgeRegistryIndexSchema = z5.object({
381
- kind: z5.literal(GEZK_REGISTRY_KIND),
382
- formatVersion: z5.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
383
- publisher: z5.object({
658
+ var KnowledgeRegistryIndexSchema = z6.object({
659
+ kind: z6.literal(GEZK_REGISTRY_KIND),
660
+ formatVersion: z6.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
661
+ publisher: z6.object({
384
662
  id: KnowledgeIdSchema,
385
- name: z5.string().min(1),
386
- url: z5.string().optional()
663
+ name: z6.string().min(1),
664
+ url: z6.string().optional()
387
665
  }),
388
- generatedAt: z5.string(),
389
- catalogs: z5.array(KnowledgeRegistryEntrySchema),
666
+ generatedAt: z6.string(),
667
+ catalogs: z6.array(KnowledgeRegistryEntrySchema),
390
668
  signature: KnowledgeSignatureSchema.optional()
391
669
  });
392
670
 
@@ -470,6 +748,30 @@ CREATE TABLE documents(
470
748
  CREATE INDEX documents_topic ON documents(topic_id, ordinal, slug);
471
749
  CREATE INDEX documents_shard ON documents(shard_id);
472
750
 
751
+ CREATE TABLE document_locations(
752
+ document_id TEXT NOT NULL REFERENCES documents(id),
753
+ location_id TEXT NOT NULL,
754
+ latitude REAL NOT NULL CHECK (latitude BETWEEN -90 AND 90),
755
+ longitude REAL NOT NULL CHECK (longitude >= -180 AND longitude < 180),
756
+ role TEXT NOT NULL CHECK (role IN ('subject','associated')),
757
+ provenance_json TEXT,
758
+ PRIMARY KEY (document_id, location_id)
759
+ ) WITHOUT ROWID;
760
+ CREATE INDEX document_locations_latitude ON document_locations(latitude, longitude, document_id);
761
+ CREATE INDEX document_locations_longitude ON document_locations(longitude, latitude, document_id);
762
+
763
+ -- Includes the primary placement in documents.topic_id/ordinal as well as
764
+ -- shared TOC references. Bodies, chunks and vectors remain keyed by document id.
765
+ CREATE TABLE topic_documents(
766
+ topic_id TEXT NOT NULL REFERENCES topics(id),
767
+ document_id TEXT NOT NULL REFERENCES documents(id),
768
+ ordinal INTEGER CHECK (ordinal IS NULL OR
769
+ (typeof(ordinal) = 'integer' AND ordinal BETWEEN -2147483648 AND 2147483647)),
770
+ PRIMARY KEY (topic_id, document_id)
771
+ ) WITHOUT ROWID;
772
+ CREATE INDEX topic_documents_document ON topic_documents(document_id);
773
+ CREATE INDEX topic_documents_order ON topic_documents(topic_id, ordinal, document_id);
774
+
473
775
  CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
474
776
  PRIMARY KEY (alias, document_id)) WITHOUT ROWID;
475
777
 
@@ -494,7 +796,8 @@ CREATE VIRTUAL TABLE fts_documents USING fts5(
494
796
  tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
495
797
  );
496
798
  `;
497
- var SHARD_DDL = `
799
+ function shardDdl(mediaColumns, mediaIndex) {
800
+ return `
498
801
  CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
499
802
 
500
803
  CREATE TABLE chunks(
@@ -508,10 +811,10 @@ CREATE TABLE chunks(
508
811
  line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,
509
812
  token_count INTEGER NOT NULL,
510
813
  content_hash TEXT NOT NULL,
511
- text TEXT NOT NULL
814
+ text TEXT NOT NULL${mediaColumns}
512
815
  );
513
816
  CREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);
514
- CREATE INDEX chunks_document ON chunks(document_id, ordinal);
817
+ CREATE INDEX chunks_document ON chunks(document_id, ordinal);${mediaIndex}
515
818
 
516
819
  CREATE VIRTUAL TABLE fts_chunks USING fts5(
517
820
  title, heading_text, text,
@@ -529,9 +832,33 @@ CREATE TABLE chunk_vectors_int8(
529
832
  v BLOB NOT NULL
530
833
  );
531
834
  `;
835
+ }
836
+ var MEDIA_COLUMNS = `,
837
+ modality TEXT NOT NULL DEFAULT 'text'
838
+ CHECK (modality IN ('text','image','video','audio')),
839
+ asset_path TEXT,
840
+ mime_type TEXT,
841
+ width INTEGER, height INTEGER,
842
+ start_ms INTEGER, end_ms INTEGER,
843
+ thumbnail_path TEXT,
844
+ attribution_json TEXT,
845
+ CHECK (modality <> 'text' OR (asset_path IS NULL AND mime_type IS NULL
846
+ AND width IS NULL AND height IS NULL AND start_ms IS NULL AND end_ms IS NULL
847
+ AND thumbnail_path IS NULL AND attribution_json IS NULL)),
848
+ CHECK (modality = 'text' OR (asset_path IS NOT NULL AND mime_type IS NOT NULL)),
849
+ CHECK (modality <> 'image' OR (start_ms IS NULL AND end_ms IS NULL)),
850
+ CHECK (modality NOT IN ('video','audio') OR (start_ms >= 0 AND end_ms > start_ms))`;
851
+ var MEDIA_INDEX = `
852
+ CREATE INDEX chunks_media ON chunks(modality, id) WHERE modality <> 'text';`;
853
+ var SHARD_DDL_V4 = shardDdl("", "");
854
+ var SHARD_DDL_V5 = shardDdl(MEDIA_COLUMNS, MEDIA_INDEX);
855
+ var SHARD_DDL = SHARD_DDL_V5;
856
+ function shardDdlFor(indexSchemaVersion) {
857
+ return indexSchemaVersion === 5 ? SHARD_DDL_V5 : SHARD_DDL_V4;
858
+ }
532
859
 
533
860
  // src/format/assets.ts
534
- import { z as z6 } from "zod";
861
+ import { z as z7 } from "zod";
535
862
  var ASSETS_PREFIX = "assets/";
536
863
  var KNOWLEDGE_ASSET_TYPES = {
537
864
  png: "image/png",
@@ -541,12 +868,24 @@ var KNOWLEDGE_ASSET_TYPES = {
541
868
  webp: "image/webp",
542
869
  svg: "image/svg+xml"
543
870
  };
871
+ var KNOWLEDGE_MEDIA_ASSET_TYPES = {
872
+ mp4: "video/mp4",
873
+ webm: "video/webm",
874
+ mp3: "audio/mpeg",
875
+ m4a: "audio/mp4",
876
+ ogg: "audio/ogg",
877
+ opus: "audio/ogg",
878
+ wav: "audio/wav",
879
+ flac: "audio/flac"
880
+ };
544
881
  var MAX_KNOWLEDGE_ASSET_BYTES = 8 * 1024 * 1024;
545
882
  var MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES = 256 * 1024 * 1024;
883
+ var MAX_KNOWLEDGE_MEDIA_ASSET_BYTES = 512 * 1024 * 1024;
884
+ var MAX_KNOWLEDGE_MEDIA_ASSETS_TOTAL_BYTES = 8 * 1024 * 1024 * 1024;
546
885
  var MAX_KNOWLEDGE_ASSET_COUNT = 8192;
547
886
  var MAX_KNOWLEDGE_ASSET_PATH_LENGTH = 512;
548
- var KNOWLEDGE_ASSET_PATH_PATTERN = /^assets\/(?:[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\/){0,15}[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\.(?:png|jpe?g|gif|webp|svg)$/i;
549
- var KnowledgeAssetPathSchema = z6.string().max(MAX_KNOWLEDGE_ASSET_PATH_LENGTH).regex(KNOWLEDGE_ASSET_PATH_PATTERN, "not a valid asset path");
887
+ var KNOWLEDGE_ASSET_PATH_PATTERN = /^assets\/(?:[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\/){0,15}[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\.(?:png|jpe?g|gif|webp|svg|mp4|webm|mp3|m4a|ogg|opus|wav|flac)$/i;
888
+ var KnowledgeAssetPathSchema = z7.string().max(MAX_KNOWLEDGE_ASSET_PATH_LENGTH).regex(KNOWLEDGE_ASSET_PATH_PATTERN, "not a valid asset path");
550
889
  function isKnowledgeAssetPath(path) {
551
890
  return path.length <= MAX_KNOWLEDGE_ASSET_PATH_LENGTH && KNOWLEDGE_ASSET_PATH_PATTERN.test(path);
552
891
  }
@@ -554,14 +893,61 @@ function assetExtension(path) {
554
893
  const dot = path.lastIndexOf(".");
555
894
  if (dot < 0) return null;
556
895
  const ext = path.slice(dot + 1).toLowerCase();
557
- return Object.hasOwn(KNOWLEDGE_ASSET_TYPES, ext) ? ext : null;
896
+ if (Object.hasOwn(KNOWLEDGE_ASSET_TYPES, ext)) return ext;
897
+ if (Object.hasOwn(KNOWLEDGE_MEDIA_ASSET_TYPES, ext)) return ext;
898
+ return null;
558
899
  }
559
900
  function assetContentType(path) {
560
901
  const ext = assetExtension(path);
561
- return ext ? KNOWLEDGE_ASSET_TYPES[ext] : null;
902
+ if (!ext) return null;
903
+ return ext in KNOWLEDGE_ASSET_TYPES ? KNOWLEDGE_ASSET_TYPES[ext] : KNOWLEDGE_MEDIA_ASSET_TYPES[ext];
904
+ }
905
+ function isKnowledgeMediaAssetPath(path) {
906
+ const ext = assetExtension(path);
907
+ return ext !== null && Object.hasOwn(KNOWLEDGE_MEDIA_ASSET_TYPES, ext);
908
+ }
909
+ function assetModality(path) {
910
+ const ext = assetExtension(path);
911
+ if (!ext) return null;
912
+ if (ext === "mp4" || ext === "webm") return "video";
913
+ return Object.hasOwn(KNOWLEDGE_MEDIA_ASSET_TYPES, ext) ? "audio" : "image";
914
+ }
915
+ function maxKnowledgeAssetBytes(path) {
916
+ return isKnowledgeMediaAssetPath(path) ? MAX_KNOWLEDGE_MEDIA_ASSET_BYTES : MAX_KNOWLEDGE_ASSET_BYTES;
917
+ }
918
+ function knowledgeAssetLimitsProblem(files) {
919
+ if (files.length > MAX_KNOWLEDGE_ASSET_COUNT) {
920
+ return `${files.length} assets exceed the ${MAX_KNOWLEDGE_ASSET_COUNT}-file limit`;
921
+ }
922
+ let imageBytes = 0;
923
+ let mediaBytes = 0;
924
+ for (const file of files) {
925
+ if (file.sizeBytes > maxKnowledgeAssetBytes(file.path)) {
926
+ return `${file.path} is ${file.sizeBytes} bytes, over its ${maxKnowledgeAssetBytes(file.path)}-byte limit`;
927
+ }
928
+ if (isKnowledgeMediaAssetPath(file.path)) mediaBytes += file.sizeBytes;
929
+ else imageBytes += file.sizeBytes;
930
+ }
931
+ if (imageBytes > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
932
+ return `image assets total ${imageBytes} bytes, over the ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES}-byte limit`;
933
+ }
934
+ if (mediaBytes > MAX_KNOWLEDGE_MEDIA_ASSETS_TOTAL_BYTES) {
935
+ return `audio and video assets total ${mediaBytes} bytes, over the ${MAX_KNOWLEDGE_MEDIA_ASSETS_TOTAL_BYTES}-byte limit`;
936
+ }
937
+ return null;
562
938
  }
563
939
  function assetKindForExtension(ext) {
564
- return ext === "jpg" ? "jpeg" : ext;
940
+ switch (ext) {
941
+ case "jpg":
942
+ return "jpeg";
943
+ case "mp4":
944
+ case "m4a":
945
+ return "isobmff";
946
+ case "opus":
947
+ return "ogg";
948
+ default:
949
+ return ext;
950
+ }
565
951
  }
566
952
  var PNG_MAGIC = [137, 80, 78, 71, 13, 10, 26, 10];
567
953
  var JPEG_MAGIC = [255, 216, 255];
@@ -569,6 +955,12 @@ var GIF87 = [71, 73, 70, 56, 55, 97];
569
955
  var GIF89 = [71, 73, 70, 56, 57, 97];
570
956
  var RIFF = [82, 73, 70, 70];
571
957
  var WEBP = [87, 69, 66, 80];
958
+ var WAVE = [87, 65, 86, 69];
959
+ var FTYP = [102, 116, 121, 112];
960
+ var EBML = [26, 69, 223, 163];
961
+ var OGGS = [79, 103, 103, 83];
962
+ var FLAC = [102, 76, 97, 67];
963
+ var ID3 = [73, 68, 51];
572
964
  function startsWith(bytes, magic, offset = 0) {
573
965
  if (bytes.length < offset + magic.length) return false;
574
966
  for (let i = 0; i < magic.length; i++) {
@@ -581,8 +973,20 @@ function sniffAssetType(bytes) {
581
973
  if (startsWith(bytes, JPEG_MAGIC)) return "jpeg";
582
974
  if (startsWith(bytes, GIF87) || startsWith(bytes, GIF89)) return "gif";
583
975
  if (startsWith(bytes, RIFF) && startsWith(bytes, WEBP, 8)) return "webp";
976
+ if (startsWith(bytes, RIFF) && startsWith(bytes, WAVE, 8)) return "wav";
977
+ if (startsWith(bytes, FTYP, 4)) return "isobmff";
978
+ if (startsWith(bytes, EBML)) return "webm";
979
+ if (startsWith(bytes, OGGS)) return "ogg";
980
+ if (startsWith(bytes, FLAC)) return "flac";
981
+ if (startsWith(bytes, ID3) || isMpegAudioFrame(bytes)) return "mp3";
584
982
  return looksLikeSvg(decodeText(bytes)) ? "svg" : null;
585
983
  }
984
+ function isMpegAudioFrame(bytes) {
985
+ if (bytes.length < 2) return false;
986
+ const b0 = bytes[0];
987
+ const b1 = bytes[1];
988
+ return b0 === 255 && (b1 & 224) === 224 && (b1 & 6) !== 0;
989
+ }
586
990
  function decodeText(bytes) {
587
991
  try {
588
992
  return new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes);
@@ -677,6 +1081,16 @@ function topicSortKeyForOrder(order) {
677
1081
  }
678
1082
 
679
1083
  // src/format/quantize.ts
1084
+ function profileUnitVector(profile, raw) {
1085
+ const source = embeddingProfileSourceDimensions(profile);
1086
+ if (raw.length !== source) {
1087
+ throw new Error(
1088
+ `embedding has ${raw.length} dimensions, profile ${profile.id} expects ${source}`
1089
+ );
1090
+ }
1091
+ if (source === profile.dimensions) return l2Normalize(raw);
1092
+ return l2Normalize(Array.prototype.slice.call(raw, 0, profile.dimensions));
1093
+ }
680
1094
  function quantizeInt8(unitVector) {
681
1095
  const out = new Int8Array(unitVector.length);
682
1096
  for (let i = 0; i < unitVector.length; i++) {
@@ -708,9 +1122,13 @@ function quantizeBinary(unitVector) {
708
1122
  return out;
709
1123
  }
710
1124
  function rerankScore(query, passageInt8) {
1125
+ if (query.length !== passageInt8.length) {
1126
+ throw new Error(`query has ${query.length} dimensions, passage has ${passageInt8.length}`);
1127
+ }
711
1128
  let dot = 0;
712
- const n = Math.min(query.length, passageInt8.length);
713
- for (let i = 0; i < n; i++) dot += query[i] * (passageInt8[i] / 127);
1129
+ for (let i = 0; i < query.length; i++) {
1130
+ dot += query[i] * (passageInt8[i] / 127);
1131
+ }
714
1132
  return dot;
715
1133
  }
716
1134
  function l2Normalize(vector) {
@@ -776,6 +1194,178 @@ function documentSlug(title) {
776
1194
  const slug = title.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
777
1195
  return slug || "untitled";
778
1196
  }
1197
+
1198
+ // src/query-stopwords.ts
1199
+ var QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
1200
+ // articles + determiners
1201
+ "a",
1202
+ "an",
1203
+ "the",
1204
+ "this",
1205
+ "that",
1206
+ "these",
1207
+ "those",
1208
+ "some",
1209
+ "any",
1210
+ "all",
1211
+ "each",
1212
+ "both",
1213
+ "every",
1214
+ "other",
1215
+ "another",
1216
+ "same",
1217
+ "such",
1218
+ // pronouns
1219
+ "i",
1220
+ "me",
1221
+ "my",
1222
+ "we",
1223
+ "us",
1224
+ "our",
1225
+ "you",
1226
+ "your",
1227
+ "it",
1228
+ "its",
1229
+ "he",
1230
+ "him",
1231
+ "his",
1232
+ "she",
1233
+ "her",
1234
+ "they",
1235
+ "them",
1236
+ "their",
1237
+ // be / have / do
1238
+ "am",
1239
+ "is",
1240
+ "are",
1241
+ "was",
1242
+ "were",
1243
+ "be",
1244
+ "been",
1245
+ "being",
1246
+ "have",
1247
+ "has",
1248
+ "had",
1249
+ "do",
1250
+ "does",
1251
+ "did",
1252
+ "done",
1253
+ // modals
1254
+ "can",
1255
+ "could",
1256
+ "will",
1257
+ "would",
1258
+ "shall",
1259
+ "should",
1260
+ "may",
1261
+ "might",
1262
+ "must",
1263
+ // question words
1264
+ "how",
1265
+ "what",
1266
+ "when",
1267
+ "where",
1268
+ "which",
1269
+ "who",
1270
+ "whom",
1271
+ "why",
1272
+ // prepositions + conjunctions
1273
+ "about",
1274
+ "and",
1275
+ "as",
1276
+ "at",
1277
+ "but",
1278
+ "by",
1279
+ "for",
1280
+ "from",
1281
+ "if",
1282
+ "in",
1283
+ "into",
1284
+ "of",
1285
+ "on",
1286
+ "or",
1287
+ "so",
1288
+ "than",
1289
+ "then",
1290
+ "there",
1291
+ "to",
1292
+ "with",
1293
+ // request scaffolding + degree words
1294
+ "please",
1295
+ "help",
1296
+ "let",
1297
+ "tell",
1298
+ "want",
1299
+ "need",
1300
+ "create",
1301
+ "make",
1302
+ "give",
1303
+ "get",
1304
+ "got",
1305
+ "use",
1306
+ "using",
1307
+ "just",
1308
+ "like",
1309
+ "also",
1310
+ "very",
1311
+ "only",
1312
+ "new",
1313
+ "more",
1314
+ "most",
1315
+ "much",
1316
+ "many",
1317
+ "few",
1318
+ "own",
1319
+ // conversational scaffolding
1320
+ "hey",
1321
+ "hi",
1322
+ "hiya",
1323
+ "hello",
1324
+ "thanks",
1325
+ "thank",
1326
+ "okay",
1327
+ "ok",
1328
+ "yep",
1329
+ "yeah",
1330
+ "yes",
1331
+ "nope",
1332
+ "good",
1333
+ "great",
1334
+ "fine",
1335
+ "nice",
1336
+ "cool",
1337
+ "ready",
1338
+ "going",
1339
+ "doing",
1340
+ "sounds",
1341
+ "morning",
1342
+ "afternoon",
1343
+ "evening",
1344
+ "night",
1345
+ // Apostrophes are token boundaries, so contractions otherwise leave these
1346
+ // fragments looking like rare, highly distinctive query terms.
1347
+ "s",
1348
+ "t",
1349
+ "re",
1350
+ "ve",
1351
+ "ll",
1352
+ "d",
1353
+ "m",
1354
+ "isn",
1355
+ "aren",
1356
+ "wasn",
1357
+ "weren",
1358
+ "don",
1359
+ "doesn",
1360
+ "didn",
1361
+ "hasn",
1362
+ "haven",
1363
+ "hadn",
1364
+ "won",
1365
+ "wouldn",
1366
+ "shouldn",
1367
+ "couldn"
1368
+ ]);
779
1369
  export {
780
1370
  ASSETS_PREFIX,
781
1371
  ArtifactDigestSchema,
@@ -783,7 +1373,10 @@ export {
783
1373
  CatalogDocumentSchema,
784
1374
  DEFAULT_EMBEDDING_ONNX_FILE,
785
1375
  DEFAULT_EMBEDDING_TOKENIZER_FILE,
1376
+ EmbeddingMediaSchema,
1377
+ EmbeddingModelFileSchema,
786
1378
  GEZK_APPLICATION_ID,
1379
+ GEZK_EARTH_RADIUS_METERS,
787
1380
  GEZK_FORMAT_GENERATIONS,
788
1381
  GEZK_FORMAT_VERSION,
789
1382
  GEZK_INDEX_SCHEMA_VERSION,
@@ -796,8 +1389,10 @@ export {
796
1389
  KNOWLEDGE_ASSET_TYPES,
797
1390
  KNOWLEDGE_ID_PATTERN,
798
1391
  KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS,
1392
+ KNOWLEDGE_MEDIA_ASSET_TYPES,
799
1393
  KNOWLEDGE_VERSION_PATTERN,
800
1394
  KnowledgeAssetPathSchema,
1395
+ KnowledgeBoundsSchema,
801
1396
  KnowledgeCatalogManifestSchema,
802
1397
  KnowledgeChunkUidSchema,
803
1398
  KnowledgeChunkingProfileSchema,
@@ -805,28 +1400,39 @@ export {
805
1400
  KnowledgeDocumentMetaSchema,
806
1401
  KnowledgeEmbeddingProfileSchema,
807
1402
  KnowledgeIdSchema,
1403
+ KnowledgeLocationSchema,
1404
+ KnowledgeLocationsSchema,
808
1405
  KnowledgeManifestFileSchema,
809
1406
  KnowledgeOrdinalSchema,
1407
+ KnowledgeRadiusSchema,
810
1408
  KnowledgeRegistryEntrySchema,
811
1409
  KnowledgeRegistryIndexSchema,
812
1410
  KnowledgeSignatureSchema,
1411
+ KnowledgeSpatialManifestSchema,
1412
+ KnowledgeTocReferenceSchema,
813
1413
  KnowledgeVectorEncodingSchema,
814
1414
  KnowledgeVersionSchema,
815
1415
  LICENSE_NOTICE_PATH,
816
1416
  MANIFEST_PATH,
1417
+ MAX_DOCUMENT_LOCATIONS,
817
1418
  MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
818
1419
  MAX_KNOWLEDGE_ASSET_BYTES,
819
1420
  MAX_KNOWLEDGE_ASSET_COUNT,
820
1421
  MAX_KNOWLEDGE_ASSET_PATH_LENGTH,
821
1422
  MAX_KNOWLEDGE_DOCUMENT_BYTES,
822
1423
  MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
1424
+ MAX_KNOWLEDGE_MEDIA_ASSETS_TOTAL_BYTES,
1425
+ MAX_KNOWLEDGE_MEDIA_ASSET_BYTES,
823
1426
  MAX_KNOWLEDGE_TOPIC_DEPTH,
824
1427
  MIMETYPE_PATH,
1428
+ QUERY_STOP_WORDS,
825
1429
  README_PATH,
826
1430
  ROUTER_DB_PATH,
827
1431
  ROUTER_DDL,
828
1432
  RepoRelativePathSchema,
829
1433
  SHARD_DDL,
1434
+ SHARD_DDL_V4,
1435
+ SHARD_DDL_V5,
830
1436
  SOURCE_NOTICES_PATH,
831
1437
  Sha256HexSchema,
832
1438
  SourceNoticesSchema,
@@ -834,24 +1440,39 @@ export {
834
1440
  assetContentType,
835
1441
  assetExtension,
836
1442
  assetKindForExtension,
1443
+ assetModality,
837
1444
  canonicalizeJson,
838
1445
  centerVector,
839
1446
  documentSlug,
840
1447
  embeddingProfileArtifacts,
841
1448
  embeddingProfileCenter,
842
1449
  embeddingProfileCenterProblem,
1450
+ embeddingProfileMinimumFormat,
1451
+ embeddingProfileSourceDimensions,
1452
+ embeddingProfileTruncationProblem,
1453
+ formatAtLeast,
843
1454
  formatKnowledgeUri,
844
1455
  isKnowledgeAssetPath,
1456
+ isKnowledgeMediaAssetPath,
845
1457
  isSupportedFormatVersion,
846
1458
  isSupportedIndexSchemaVersion,
1459
+ knowledgeAssetLimitsProblem,
847
1460
  l2Normalize,
1461
+ locationDistanceMeters,
1462
+ locationInBounds,
1463
+ maxKnowledgeAssetBytes,
1464
+ normalizeLongitude,
848
1465
  parseKnowledgeUri,
1466
+ profileUnitVector,
849
1467
  quantizeBinary,
850
1468
  quantizeBinaryForProfile,
851
1469
  quantizeInt8,
1470
+ radiusBounds,
852
1471
  rerankScore,
853
1472
  sameVectorSpace,
1473
+ shardDdlFor,
854
1474
  sniffAssetType,
1475
+ spatialManifest,
855
1476
  svgInertnessProblem,
856
1477
  topicSortKeyForOrder
857
1478
  };