@bendyline/gezk 1.0.0 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -28,7 +28,7 @@ var RepoRelativePathSchema = z2.string().min(1).refine(
28
28
  );
29
29
  var DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
30
30
  var DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
31
- var KnowledgeEmbeddingProfileSchema = z2.object({
31
+ var KnowledgeEmbeddingProfileObject = z2.object({
32
32
  /** e.g. `multilingual-e5-small@1`. A new model, prefix or encoding is a new id. */
33
33
  id: z2.string().min(1),
34
34
  model: z2.object({
@@ -69,13 +69,59 @@ var KnowledgeEmbeddingProfileSchema = z2.object({
69
69
  method: z2.literal("symmetric-linear"),
70
70
  scale: z2.literal(127)
71
71
  }),
72
+ /**
73
+ * The stage-1 sign bits. `sign` takes them straight from the unit
74
+ * vector. `centered-sign` takes them from `vector − center`, where
75
+ * `center` is a fixed vector the profile pins (length = `dimensions`).
76
+ *
77
+ * Why a center exists: some embedding models put most of every vector
78
+ * on one shared direction (multilingual-e5-small: unrelated passages
79
+ * sit at cosine ≈ 0.8 and the corpus mean has norm ≈ 0.87). The sign
80
+ * of a dimension is then decided by that shared component rather than
81
+ * by the passage, hamming distance stops tracking cosine, and stage 1
82
+ * discards the true neighbours before the int8 rerank can see them.
83
+ * Subtracting the center first restores the correlation. The int8
84
+ * vectors are untouched either way, so query vectors stay comparable
85
+ * across both methods; only how the bits are derived differs.
86
+ */
72
87
  binary: z2.object({
73
- method: z2.literal("sign"),
88
+ method: z2.enum(["sign", "centered-sign"]),
74
89
  threshold: z2.literal(0),
75
- packing: z2.literal("lsb-first")
90
+ packing: z2.literal("lsb-first"),
91
+ /** Required by `centered-sign`, refused by `sign`; one entry per dimension. */
92
+ center: z2.array(z2.number()).optional()
76
93
  })
77
94
  })
78
95
  });
96
+ function embeddingProfileCenterProblem(profile) {
97
+ const { method, center } = profile.quantization.binary;
98
+ if (method === "centered-sign") {
99
+ if (!center) return "centered-sign requires quantization.binary.center";
100
+ if (center.length !== profile.dimensions) {
101
+ return `quantization.binary.center has ${center.length} entries, dimensions is ${profile.dimensions}`;
102
+ }
103
+ return null;
104
+ }
105
+ return center ? "quantization.binary.center is only valid with method centered-sign" : null;
106
+ }
107
+ function embeddingProfileCenter(profile) {
108
+ const problem = embeddingProfileCenterProblem(profile);
109
+ if (problem) throw new Error(`invalid embedding profile ${profile.id}: ${problem}`);
110
+ const { center } = profile.quantization.binary;
111
+ return center ? Float32Array.from(center) : null;
112
+ }
113
+ var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefine(
114
+ (profile, ctx) => {
115
+ const problem = embeddingProfileCenterProblem(profile);
116
+ if (problem) {
117
+ ctx.addIssue({
118
+ code: "custom",
119
+ message: problem,
120
+ path: ["quantization", "binary", "center"]
121
+ });
122
+ }
123
+ }
124
+ );
79
125
  function embeddingProfileArtifacts(profile) {
80
126
  return {
81
127
  onnxFile: profile.model.onnxFile ?? DEFAULT_EMBEDDING_ONNX_FILE,
@@ -88,7 +134,7 @@ function sameVectorSpace(a, b) {
88
134
  const aa = embeddingProfileArtifacts(a);
89
135
  const ab = embeddingProfileArtifacts(b);
90
136
  const digestsAgree = (x, y) => !x || !y || x === y;
91
- return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale && a.quantization.binary.method === b.quantization.binary.method && a.quantization.binary.threshold === b.quantization.binary.threshold && a.quantization.binary.packing === b.quantization.binary.packing;
137
+ return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale;
92
138
  }
93
139
  var KnowledgeChunkingProfileSchema = z2.object({
94
140
  /** e.g. `markdown-chunks@2`. */
@@ -104,8 +150,17 @@ var KnowledgeChunkingProfileSchema = z2.object({
104
150
  import { z as z3 } from "zod";
105
151
 
106
152
  // src/format/constants.ts
107
- var GEZK_FORMAT_VERSION = "0.5";
108
- var GEZK_INDEX_SCHEMA_VERSION = 2;
153
+ var GEZK_FORMAT_VERSION = "0.6";
154
+ var GEZK_INDEX_SCHEMA_VERSION = 3;
155
+ var GEZK_FORMAT_GENERATIONS = { "0.5": 2, "0.6": 3 };
156
+ var GEZK_SUPPORTED_FORMAT_VERSIONS = Object.keys(GEZK_FORMAT_GENERATIONS);
157
+ var GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS = Object.values(GEZK_FORMAT_GENERATIONS);
158
+ function isSupportedFormatVersion(value) {
159
+ return typeof value === "string" && Object.hasOwn(GEZK_FORMAT_GENERATIONS, value);
160
+ }
161
+ function isSupportedIndexSchemaVersion(value) {
162
+ return typeof value === "number" && GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.includes(value);
163
+ }
109
164
  var GEZK_APPLICATION_ID = 1195727435;
110
165
  var GEZK_MIME_TYPE = "application/vnd.gezk+zip";
111
166
  var MIMETYPE_PATH = "mimetype";
@@ -116,6 +171,8 @@ var SOURCE_NOTICES_PATH = "LICENSES/source-notices.json";
116
171
  var ROUTER_DB_PATH = "index/router.db";
117
172
  var BODY_CODEC_MIN_BYTES = 512;
118
173
  var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
174
+ var MAX_KNOWLEDGE_DOCUMENT_META_BYTES = 16 * 1024;
175
+ var MAX_KNOWLEDGE_TOPIC_DEPTH = 16;
119
176
  var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
120
177
 
121
178
  // src/schemas/manifest.ts
@@ -134,8 +191,8 @@ var KnowledgeSignatureSchema = z3.object({
134
191
  });
135
192
  var KnowledgeCatalogManifestSchema = z3.object({
136
193
  kind: z3.literal(GEZK_MANIFEST_KIND),
137
- formatVersion: z3.literal(GEZK_FORMAT_VERSION),
138
- indexSchemaVersion: z3.literal(GEZK_INDEX_SCHEMA_VERSION),
194
+ formatVersion: z3.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
195
+ indexSchemaVersion: z3.union([z3.literal(2), z3.literal(3)]),
139
196
  id: KnowledgeIdSchema,
140
197
  version: KnowledgeVersionSchema,
141
198
  name: z3.string().min(1),
@@ -162,7 +219,17 @@ var KnowledgeCatalogManifestSchema = z3.object({
162
219
  }),
163
220
  embedding: KnowledgeEmbeddingProfileSchema,
164
221
  chunking: KnowledgeChunkingProfileSchema,
165
- topics: z3.array(z3.object({ id: KnowledgeIdSchema, name: z3.string().min(1) })).min(1, "a catalog must ship a table of contents (at least one topic)"),
222
+ topics: z3.array(
223
+ z3.object({
224
+ id: KnowledgeIdSchema,
225
+ name: z3.string().min(1),
226
+ /** Absent for a root topic; names another entry of this list. */
227
+ parentId: KnowledgeIdSchema.optional(),
228
+ /** Sibling ordering key, identical to `topics.sort_key` in the router. */
229
+ sortKey: z3.string().min(1).optional(),
230
+ description: z3.string().optional()
231
+ })
232
+ ).min(1, "a catalog must ship a table of contents (at least one topic)"),
166
233
  router: z3.object({
167
234
  shardTargetChunks: z3.number().int().positive(),
168
235
  shards: z3.array(
@@ -180,12 +247,14 @@ var KnowledgeCatalogManifestSchema = z3.object({
180
247
  counts: z3.object({
181
248
  documents: z3.number().int().nonnegative(),
182
249
  chunks: z3.number().int().nonnegative(),
183
- shards: z3.number().int().positive()
250
+ shards: z3.number().int().positive(),
251
+ /** Files under `assets/` (0.6+). */
252
+ assets: z3.number().int().nonnegative().optional()
184
253
  }),
185
254
  files: z3.array(KnowledgeManifestFileSchema).min(1),
186
255
  /** What a reader must implement to open this catalog. */
187
256
  requires: z3.object({
188
- formatVersion: z3.literal(GEZK_FORMAT_VERSION),
257
+ formatVersion: z3.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
189
258
  features: z3.array(z3.string()).optional()
190
259
  }),
191
260
  smokeQueries: z3.array(
@@ -205,7 +274,50 @@ var KnowledgeCatalogManifestSchema = z3.object({
205
274
  tokenizerDigest: z3.string().optional()
206
275
  }).optional(),
207
276
  signature: KnowledgeSignatureSchema.optional()
277
+ }).superRefine((manifest, ctx) => {
278
+ if (GEZK_FORMAT_GENERATIONS[manifest.formatVersion] !== manifest.indexSchemaVersion) {
279
+ ctx.addIssue({
280
+ code: "custom",
281
+ path: ["indexSchemaVersion"],
282
+ message: `format ${manifest.formatVersion} pairs with index schema ${GEZK_FORMAT_GENERATIONS[manifest.formatVersion]}, not ${manifest.indexSchemaVersion}`
283
+ });
284
+ }
285
+ if (manifest.requires.formatVersion !== manifest.formatVersion) {
286
+ ctx.addIssue({
287
+ code: "custom",
288
+ path: ["requires", "formatVersion"],
289
+ message: "requires.formatVersion must equal formatVersion"
290
+ });
291
+ }
292
+ if (manifest.formatVersion === "0.5" && manifest.counts.assets !== void 0) {
293
+ ctx.addIssue({
294
+ code: "custom",
295
+ path: ["counts", "assets"],
296
+ message: "counts.assets is a 0.6 field"
297
+ });
298
+ }
299
+ const topicIds = /* @__PURE__ */ new Set();
300
+ manifest.topics.forEach((topic, index) => {
301
+ if (topicIds.has(topic.id)) {
302
+ ctx.addIssue({
303
+ code: "custom",
304
+ path: ["topics", index, "id"],
305
+ message: `duplicate topic id '${topic.id}'`
306
+ });
307
+ }
308
+ topicIds.add(topic.id);
309
+ });
310
+ manifest.topics.forEach((topic, index) => {
311
+ if (topic.parentId !== void 0 && !topicIds.has(topic.parentId)) {
312
+ ctx.addIssue({
313
+ code: "custom",
314
+ path: ["topics", index, "parentId"],
315
+ message: `topic '${topic.id}' names an undeclared parent '${topic.parentId}'`
316
+ });
317
+ }
318
+ });
208
319
  });
320
+ var KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS = GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS;
209
321
  var SourceNoticesSchema = z3.object({
210
322
  sources: z3.array(
211
323
  z3.object({
@@ -223,20 +335,28 @@ var SourceNoticesSchema = z3.object({
223
335
 
224
336
  // src/schemas/document.ts
225
337
  import { z as z4 } from "zod";
338
+ var KnowledgeOrdinalSchema = z4.number().int().min(-2147483648).max(2147483647);
339
+ var KnowledgeDocumentMetaSchema = z4.record(z4.string(), z4.unknown());
226
340
  var CatalogDocumentSchema = z4.object({
227
341
  id: KnowledgeDocumentIdSchema,
228
342
  title: z4.string().min(1),
229
343
  slug: z4.string().min(1),
230
344
  summary: z4.string().optional(),
231
345
  language: z4.string().min(2),
232
- /** Root→leaf topic id path; the first segment must exist in the manifest. */
346
+ /**
347
+ * Root→leaf topic id path. Every segment must be a declared topic and each
348
+ * must be the parent of the next; the document is filed at the last one.
349
+ */
233
350
  topicPath: z4.array(KnowledgeIdSchema).min(1),
234
351
  markdown: z4.string(),
235
352
  sourceUrl: z4.string().optional(),
236
353
  sourceRevision: z4.string().optional(),
237
354
  sourceUpdatedAt: z4.string().optional(),
238
355
  attribution: z4.record(z4.string(), z4.string()).optional(),
239
- aliases: z4.array(z4.string()).optional()
356
+ aliases: z4.array(z4.string()).optional(),
357
+ /** Explicit position among the documents of its topic in a listing. */
358
+ ordinal: KnowledgeOrdinalSchema.optional(),
359
+ meta: KnowledgeDocumentMetaSchema.optional()
240
360
  });
241
361
 
242
362
  // src/schemas/registry.ts
@@ -259,7 +379,7 @@ var KnowledgeRegistryEntrySchema = z5.object({
259
379
  });
260
380
  var KnowledgeRegistryIndexSchema = z5.object({
261
381
  kind: z5.literal(GEZK_REGISTRY_KIND),
262
- formatVersion: z5.literal(GEZK_FORMAT_VERSION),
382
+ formatVersion: z5.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
263
383
  publisher: z5.object({
264
384
  id: KnowledgeIdSchema,
265
385
  name: z5.string().min(1),
@@ -338,14 +458,16 @@ CREATE TABLE documents(
338
458
  id TEXT PRIMARY KEY,
339
459
  title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
340
460
  language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
461
+ ordinal INTEGER,
341
462
  shard_id INTEGER NOT NULL,
342
463
  chunk_count INTEGER NOT NULL,
343
464
  source_url TEXT, source_revision TEXT, source_updated_at TEXT,
344
465
  attribution_json TEXT,
466
+ meta_json TEXT,
345
467
  body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
346
468
  body_blob BLOB NOT NULL
347
469
  );
348
- CREATE INDEX documents_topic ON documents(topic_id, slug);
470
+ CREATE INDEX documents_topic ON documents(topic_id, ordinal, slug);
349
471
  CREATE INDEX documents_shard ON documents(shard_id);
350
472
 
351
473
  CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
@@ -408,6 +530,152 @@ CREATE TABLE chunk_vectors_int8(
408
530
  );
409
531
  `;
410
532
 
533
+ // src/format/assets.ts
534
+ import { z as z6 } from "zod";
535
+ var ASSETS_PREFIX = "assets/";
536
+ var KNOWLEDGE_ASSET_TYPES = {
537
+ png: "image/png",
538
+ jpg: "image/jpeg",
539
+ jpeg: "image/jpeg",
540
+ gif: "image/gif",
541
+ webp: "image/webp",
542
+ svg: "image/svg+xml"
543
+ };
544
+ var MAX_KNOWLEDGE_ASSET_BYTES = 8 * 1024 * 1024;
545
+ var MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES = 256 * 1024 * 1024;
546
+ var MAX_KNOWLEDGE_ASSET_COUNT = 8192;
547
+ var MAX_KNOWLEDGE_ASSET_PATH_LENGTH = 512;
548
+ var KNOWLEDGE_ASSET_PATH_PATTERN = /^assets\/(?:[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\/){0,15}[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\.(?:png|jpe?g|gif|webp|svg)$/i;
549
+ var KnowledgeAssetPathSchema = z6.string().max(MAX_KNOWLEDGE_ASSET_PATH_LENGTH).regex(KNOWLEDGE_ASSET_PATH_PATTERN, "not a valid asset path");
550
+ function isKnowledgeAssetPath(path) {
551
+ return path.length <= MAX_KNOWLEDGE_ASSET_PATH_LENGTH && KNOWLEDGE_ASSET_PATH_PATTERN.test(path);
552
+ }
553
+ function assetExtension(path) {
554
+ const dot = path.lastIndexOf(".");
555
+ if (dot < 0) return null;
556
+ const ext = path.slice(dot + 1).toLowerCase();
557
+ return Object.hasOwn(KNOWLEDGE_ASSET_TYPES, ext) ? ext : null;
558
+ }
559
+ function assetContentType(path) {
560
+ const ext = assetExtension(path);
561
+ return ext ? KNOWLEDGE_ASSET_TYPES[ext] : null;
562
+ }
563
+ function assetKindForExtension(ext) {
564
+ return ext === "jpg" ? "jpeg" : ext;
565
+ }
566
+ var PNG_MAGIC = [137, 80, 78, 71, 13, 10, 26, 10];
567
+ var JPEG_MAGIC = [255, 216, 255];
568
+ var GIF87 = [71, 73, 70, 56, 55, 97];
569
+ var GIF89 = [71, 73, 70, 56, 57, 97];
570
+ var RIFF = [82, 73, 70, 70];
571
+ var WEBP = [87, 69, 66, 80];
572
+ function startsWith(bytes, magic, offset = 0) {
573
+ if (bytes.length < offset + magic.length) return false;
574
+ for (let i = 0; i < magic.length; i++) {
575
+ if (bytes[offset + i] !== magic[i]) return false;
576
+ }
577
+ return true;
578
+ }
579
+ function sniffAssetType(bytes) {
580
+ if (startsWith(bytes, PNG_MAGIC)) return "png";
581
+ if (startsWith(bytes, JPEG_MAGIC)) return "jpeg";
582
+ if (startsWith(bytes, GIF87) || startsWith(bytes, GIF89)) return "gif";
583
+ if (startsWith(bytes, RIFF) && startsWith(bytes, WEBP, 8)) return "webp";
584
+ return looksLikeSvg(decodeText(bytes)) ? "svg" : null;
585
+ }
586
+ function decodeText(bytes) {
587
+ try {
588
+ return new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes);
589
+ } catch {
590
+ return "";
591
+ }
592
+ }
593
+ function looksLikeSvg(text) {
594
+ let i = 0;
595
+ const n = text.length;
596
+ while (i < n) {
597
+ const ch = text.charCodeAt(i);
598
+ if (ch === 32 || ch === 9 || ch === 10 || ch === 13 || ch === 65279) {
599
+ i++;
600
+ continue;
601
+ }
602
+ if (text.startsWith("<?", i)) {
603
+ const end = text.indexOf("?>", i);
604
+ if (end < 0) return false;
605
+ i = end + 2;
606
+ continue;
607
+ }
608
+ if (text.startsWith("<!--", i)) {
609
+ const end = text.indexOf("-->", i);
610
+ if (end < 0) return false;
611
+ i = end + 3;
612
+ continue;
613
+ }
614
+ if (/^<!doctype/i.test(text.slice(i, i + 9))) {
615
+ const bracket = text.indexOf("[", i);
616
+ const close = text.indexOf(">", i);
617
+ if (close < 0) return false;
618
+ if (bracket >= 0 && bracket < close) {
619
+ const end = text.indexOf("]>", bracket);
620
+ if (end < 0) return false;
621
+ i = end + 2;
622
+ } else {
623
+ i = close + 1;
624
+ }
625
+ continue;
626
+ }
627
+ return /^<svg[\s>/]/i.test(text.slice(i, i + 5));
628
+ }
629
+ return false;
630
+ }
631
+ var SVG_ACTIVE_PATTERNS = [
632
+ [/<script[\s>/]/i, "a <script> element"],
633
+ [/<foreignobject[\s>/]/i, "a <foreignObject> element"],
634
+ [/<!entity/i, "an entity declaration"],
635
+ [/\son[a-z]+\s*=/i, "an event-handler attribute"],
636
+ [/javascript\s*:/i, "a javascript: reference"],
637
+ [/data\s*:\s*text\/html/i, "a data:text/html reference"],
638
+ [/@import/i, "a CSS @import"]
639
+ ];
640
+ var HREF_ATTRIBUTE = /(?:xlink:)?href\s*=\s*["']\s*([^"']*)["']/gi;
641
+ var CSS_URL = /url\(\s*["']?\s*([^"')]*)/gi;
642
+ function externalReferenceProblem(target) {
643
+ const value = target.trim();
644
+ if (value === "" || value.startsWith("#")) return null;
645
+ const scheme = /^([a-z][a-z0-9+.-]*):/i.exec(value);
646
+ if (!scheme) return null;
647
+ if (/^data:image\/(?:png|jpeg|gif|webp)[;,]/i.test(value)) return null;
648
+ return `a ${scheme[1]}: reference`;
649
+ }
650
+ function svgInertnessProblem(bytes) {
651
+ const text = decodeText(bytes);
652
+ if (text === "") return "not valid UTF-8";
653
+ if (!looksLikeSvg(text)) return "does not start with an <svg> element";
654
+ for (const [pattern, why] of SVG_ACTIVE_PATTERNS) {
655
+ if (pattern.test(text)) return `contains ${why}`;
656
+ }
657
+ for (const match of text.matchAll(HREF_ATTRIBUTE)) {
658
+ const problem = externalReferenceProblem(match[1] ?? "");
659
+ if (problem) return `contains ${problem} in an href`;
660
+ }
661
+ for (const match of text.matchAll(CSS_URL)) {
662
+ const problem = externalReferenceProblem(match[1] ?? "");
663
+ if (problem) return `contains ${problem} in a CSS url()`;
664
+ }
665
+ return null;
666
+ }
667
+
668
+ // src/format/sort-key.ts
669
+ var INT32_MIN = -2147483648;
670
+ var INT32_MAX = 2147483647;
671
+ var OFFSET = 2147483648;
672
+ function topicSortKeyForOrder(order) {
673
+ if (!Number.isInteger(order) || order < INT32_MIN || order > INT32_MAX) {
674
+ throw new RangeError(`topic order must be an int32: ${String(order)}`);
675
+ }
676
+ return String(order + OFFSET).padStart(10, "0");
677
+ }
678
+
411
679
  // src/format/quantize.ts
412
680
  function quantizeInt8(unitVector) {
413
681
  const out = new Int8Array(unitVector.length);
@@ -417,6 +685,18 @@ function quantizeInt8(unitVector) {
417
685
  }
418
686
  return out;
419
687
  }
688
+ function centerVector(vector, center) {
689
+ if (vector.length !== center.length) {
690
+ throw new Error(`vector has ${vector.length} dimensions, center has ${center.length}`);
691
+ }
692
+ const out = new Float32Array(vector.length);
693
+ for (let i = 0; i < vector.length; i++) out[i] = vector[i] - center[i];
694
+ return out;
695
+ }
696
+ function quantizeBinaryForProfile(profile, unitVector) {
697
+ const center = embeddingProfileCenter(profile);
698
+ return quantizeBinary(center ? centerVector(unitVector, center) : unitVector);
699
+ }
420
700
  function quantizeBinary(unitVector) {
421
701
  const bytes = Math.ceil(unitVector.length / 8);
422
702
  const out = new Uint8Array(bytes);
@@ -497,26 +777,36 @@ function documentSlug(title) {
497
777
  return slug || "untitled";
498
778
  }
499
779
  export {
780
+ ASSETS_PREFIX,
500
781
  ArtifactDigestSchema,
501
782
  BODY_CODEC_MIN_BYTES,
502
783
  CatalogDocumentSchema,
503
784
  DEFAULT_EMBEDDING_ONNX_FILE,
504
785
  DEFAULT_EMBEDDING_TOKENIZER_FILE,
505
786
  GEZK_APPLICATION_ID,
787
+ GEZK_FORMAT_GENERATIONS,
506
788
  GEZK_FORMAT_VERSION,
507
789
  GEZK_INDEX_SCHEMA_VERSION,
508
790
  GEZK_MANIFEST_KIND,
509
791
  GEZK_MIME_TYPE,
510
792
  GEZK_REGISTRY_KIND,
793
+ GEZK_SUPPORTED_FORMAT_VERSIONS,
794
+ GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
795
+ KNOWLEDGE_ASSET_PATH_PATTERN,
796
+ KNOWLEDGE_ASSET_TYPES,
511
797
  KNOWLEDGE_ID_PATTERN,
798
+ KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS,
512
799
  KNOWLEDGE_VERSION_PATTERN,
800
+ KnowledgeAssetPathSchema,
513
801
  KnowledgeCatalogManifestSchema,
514
802
  KnowledgeChunkUidSchema,
515
803
  KnowledgeChunkingProfileSchema,
516
804
  KnowledgeDocumentIdSchema,
805
+ KnowledgeDocumentMetaSchema,
517
806
  KnowledgeEmbeddingProfileSchema,
518
807
  KnowledgeIdSchema,
519
808
  KnowledgeManifestFileSchema,
809
+ KnowledgeOrdinalSchema,
520
810
  KnowledgeRegistryEntrySchema,
521
811
  KnowledgeRegistryIndexSchema,
522
812
  KnowledgeSignatureSchema,
@@ -524,7 +814,13 @@ export {
524
814
  KnowledgeVersionSchema,
525
815
  LICENSE_NOTICE_PATH,
526
816
  MANIFEST_PATH,
817
+ MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
818
+ MAX_KNOWLEDGE_ASSET_BYTES,
819
+ MAX_KNOWLEDGE_ASSET_COUNT,
820
+ MAX_KNOWLEDGE_ASSET_PATH_LENGTH,
527
821
  MAX_KNOWLEDGE_DOCUMENT_BYTES,
822
+ MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
823
+ MAX_KNOWLEDGE_TOPIC_DEPTH,
528
824
  MIMETYPE_PATH,
529
825
  README_PATH,
530
826
  ROUTER_DB_PATH,
@@ -535,14 +831,27 @@ export {
535
831
  Sha256HexSchema,
536
832
  SourceNoticesSchema,
537
833
  ZIP_FIXED_MTIME,
834
+ assetContentType,
835
+ assetExtension,
836
+ assetKindForExtension,
538
837
  canonicalizeJson,
838
+ centerVector,
539
839
  documentSlug,
540
840
  embeddingProfileArtifacts,
841
+ embeddingProfileCenter,
842
+ embeddingProfileCenterProblem,
541
843
  formatKnowledgeUri,
844
+ isKnowledgeAssetPath,
845
+ isSupportedFormatVersion,
846
+ isSupportedIndexSchemaVersion,
542
847
  l2Normalize,
543
848
  parseKnowledgeUri,
544
849
  quantizeBinary,
850
+ quantizeBinaryForProfile,
545
851
  quantizeInt8,
546
852
  rerankScore,
547
- sameVectorSpace
853
+ sameVectorSpace,
854
+ sniffAssetType,
855
+ svgInertnessProblem,
856
+ topicSortKeyForOrder
548
857
  };
package/dist/node.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { KnowledgeCatalogManifest, KnowledgeRegistryIndex } from './index.js';
2
- export { ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_VERSION_PATTERN, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, KnowledgeRegistryIndexSchema, KnowledgeSignature, KnowledgeSignatureSchema, KnowledgeUri, KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace } from './index.js';
2
+ export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, KnowledgeAssetExtension, KnowledgeAssetKind, KnowledgeAssetPathSchema, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, KnowledgeRegistryIndexSchema, KnowledgeSignature, KnowledgeSignatureSchema, KnowledgeUri, KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, centerVector, documentSlug, embeddingProfileArtifacts, embeddingProfileCenter, embeddingProfileCenterProblem, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder } from './index.js';
3
3
  import 'zod';
4
4
 
5
5
  /**