@bendyline/gezk 1.0.1 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -48,6 +48,63 @@ declare const DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
48
48
  * name or a dimension is unsafe — two 384-dim models do not share a space,
49
49
  * and an instruction-prefix change silently changes vectors.
50
50
  */
51
+ declare const KnowledgeEmbeddingProfileObject: z.ZodObject<{
52
+ id: z.ZodString;
53
+ model: z.ZodObject<{
54
+ repo: z.ZodString;
55
+ revision: z.ZodString;
56
+ onnxFile: z.ZodOptional<z.ZodString>;
57
+ onnxDigest: z.ZodOptional<z.ZodString>;
58
+ }, z.core.$strip>;
59
+ tokenizer: z.ZodObject<{
60
+ kind: z.ZodString;
61
+ file: z.ZodOptional<z.ZodString>;
62
+ digest: z.ZodOptional<z.ZodString>;
63
+ }, z.core.$strip>;
64
+ pooling: z.ZodEnum<{
65
+ mean: "mean";
66
+ cls: "cls";
67
+ last: "last";
68
+ }>;
69
+ normalized: z.ZodBoolean;
70
+ dimensions: z.ZodNumber;
71
+ maxTokens: z.ZodNumber;
72
+ queryInstruction: z.ZodString;
73
+ passageInstruction: z.ZodString;
74
+ vectorEncoding: z.ZodEnum<{
75
+ "bit+int8": "bit+int8";
76
+ }>;
77
+ distance: z.ZodObject<{
78
+ stage1: z.ZodLiteral<"hamming">;
79
+ stage2: z.ZodLiteral<"cosine">;
80
+ }, z.core.$strip>;
81
+ quantization: z.ZodObject<{
82
+ int8: z.ZodObject<{
83
+ method: z.ZodLiteral<"symmetric-linear">;
84
+ scale: z.ZodLiteral<127>;
85
+ }, z.core.$strip>;
86
+ binary: z.ZodObject<{
87
+ method: z.ZodEnum<{
88
+ sign: "sign";
89
+ "centered-sign": "centered-sign";
90
+ }>;
91
+ threshold: z.ZodLiteral<0>;
92
+ packing: z.ZodLiteral<"lsb-first">;
93
+ center: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
94
+ }, z.core.$strip>;
95
+ }, z.core.$strip>;
96
+ }, z.core.$strip>;
97
+ type KnowledgeEmbeddingProfile = z.infer<typeof KnowledgeEmbeddingProfileObject>;
98
+ /**
99
+ * The cross-field rule the object schema cannot express on its own: a
100
+ * `centered-sign` profile must carry a center of exactly `dimensions`
101
+ * entries, and a `sign` profile must not carry one. Returns the problem, or
102
+ * null when the profile is consistent.
103
+ */
104
+ declare function embeddingProfileCenterProblem(profile: KnowledgeEmbeddingProfile): string | null;
105
+ /** The profile's binary center as a Float32Array, or null for a plain `sign` profile. */
106
+ declare function embeddingProfileCenter(profile: KnowledgeEmbeddingProfile): Float32Array | null;
107
+ /** The profile schema proper: the object shape plus the center rule above. */
51
108
  declare const KnowledgeEmbeddingProfileSchema: z.ZodObject<{
52
109
  id: z.ZodString;
53
110
  model: z.ZodObject<{
@@ -84,13 +141,16 @@ declare const KnowledgeEmbeddingProfileSchema: z.ZodObject<{
84
141
  scale: z.ZodLiteral<127>;
85
142
  }, z.core.$strip>;
86
143
  binary: z.ZodObject<{
87
- method: z.ZodLiteral<"sign">;
144
+ method: z.ZodEnum<{
145
+ sign: "sign";
146
+ "centered-sign": "centered-sign";
147
+ }>;
88
148
  threshold: z.ZodLiteral<0>;
89
149
  packing: z.ZodLiteral<"lsb-first">;
150
+ center: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
90
151
  }, z.core.$strip>;
91
152
  }, z.core.$strip>;
92
153
  }, z.core.$strip>;
93
- type KnowledgeEmbeddingProfile = z.infer<typeof KnowledgeEmbeddingProfileSchema>;
94
154
  /** The exact repo files a profile pins, with the format defaults applied. */
95
155
  declare function embeddingProfileArtifacts(profile: KnowledgeEmbeddingProfile): {
96
156
  onnxFile: string;
@@ -101,12 +161,19 @@ declare function embeddingProfileArtifacts(profile: KnowledgeEmbeddingProfile):
101
161
  /**
102
162
  * Whether two profiles describe one vector space, so vectors from either
103
163
  * may be compared: same model files at the same revision, same tokenizer,
104
- * pooling, normalization, dimensions, instruction prefixes and encoding.
105
- * The profile id and `maxTokens` are not compared — the first is a label,
106
- * the second a compile-time bound. A digest counts only when both sides
107
- * declare one: two pins at one revision that hash differently name
108
- * different artifacts, whatever the path says; an undeclared digest is
109
- * simply not a claim.
164
+ * pooling, normalization, dimensions, instruction prefixes, encoding and
165
+ * int8 quantization. The profile id and `maxTokens` are not compared — the
166
+ * first is a label, the second a compile-time bound. A digest counts only
167
+ * when both sides declare one: two pins at one revision that hash
168
+ * differently name different artifacts, whatever the path says; an
169
+ * undeclared digest is simply not a claim.
170
+ *
171
+ * The binary (stage-1) parameters are deliberately NOT compared. They
172
+ * describe how a catalog derived its own pre-filter bits, and a reader
173
+ * applies them from the catalog's profile echo; they never change what a
174
+ * query vector is. A daemon that shares a model with a `sign` profile
175
+ * therefore also shares it with the `centered-sign` revision of the same
176
+ * model — same floats in, same int8 rerank, only the bit scan differs.
110
177
  */
111
178
  declare function sameVectorSpace(a: KnowledgeEmbeddingProfile, b: KnowledgeEmbeddingProfile): boolean;
112
179
  /**
@@ -217,9 +284,13 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
217
284
  scale: z.ZodLiteral<127>;
218
285
  }, z.core.$strip>;
219
286
  binary: z.ZodObject<{
220
- method: z.ZodLiteral<"sign">;
287
+ method: z.ZodEnum<{
288
+ sign: "sign";
289
+ "centered-sign": "centered-sign";
290
+ }>;
221
291
  threshold: z.ZodLiteral<0>;
222
292
  packing: z.ZodLiteral<"lsb-first">;
293
+ center: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
223
294
  }, z.core.$strip>;
224
295
  }, z.core.$strip>;
225
296
  }, z.core.$strip>;
@@ -490,9 +561,11 @@ declare const ZIP_FIXED_MTIME: Date;
490
561
  declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n ordinal INTEGER,\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n meta_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, ordinal, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
491
562
  /**
492
563
  * Per-shard tables. Vectors are plain BLOB columns keyed by chunk id:
493
- * `chunk_vectors_bit.v` holds ceil(dim/8) bytes of sign bits (LSB-first) and
494
- * `chunk_vectors_int8.v` holds dim signed bytes (symmetric-linear, scale
495
- * 127). Two tables, not one, so a stage-1 scan pages in 48 bytes per row
564
+ * `chunk_vectors_bit.v` holds ceil(dim/8) bytes of sign bits (LSB-first; of
565
+ * the unit vector for a `sign` profile, of `vector − center` for a
566
+ * `centered-sign` profile — the profile echo in the router's meta says
567
+ * which) and `chunk_vectors_int8.v` holds dim signed bytes
568
+ * (symmetric-linear, scale 127, never centered). Two tables, not one, so a stage-1 scan pages in 48 bytes per row
496
569
  * rather than the whole record. Rowid alignment invariant:
497
570
  * chunks.id == chunk_vectors_bit.chunk_id == chunk_vectors_int8.chunk_id ==
498
571
  * fts_chunks.rowid, dense from 1.
@@ -567,9 +640,25 @@ declare function topicSortKeyForOrder(order: number): string;
567
640
  * functions, so extension-internal semantics can never drift the format:
568
641
  * both the compiler (packing) and the reader (query packing + rerank) call
569
642
  * exactly these functions.
643
+ *
644
+ * `quantizeBinary` is the raw sign packing. What a catalog actually stores
645
+ * depends on the profile's `quantization.binary.method`: `sign` packs the
646
+ * unit vector, `centered-sign` packs `vector − center` (see the profile
647
+ * schema for why). `quantizeBinaryForProfile` is the one entry point that
648
+ * applies that rule, so the compiler and every reader derive bits the same
649
+ * way.
570
650
  */
651
+
571
652
  /** int8: symmetric linear, scale 127, −128 never produced. */
572
653
  declare function quantizeInt8(unitVector: ArrayLike<number>): Int8Array;
654
+ /** `vector − center`, the input to a `centered-sign` profile's bits. */
655
+ declare function centerVector(vector: ArrayLike<number>, center: ArrayLike<number>): Float32Array;
656
+ /**
657
+ * The sign bits a profile stores for a unit vector — and, symmetrically,
658
+ * the bits a hamming reader packs for a query: centered first when the
659
+ * profile pins a center, raw otherwise.
660
+ */
661
+ declare function quantizeBinaryForProfile(profile: KnowledgeEmbeddingProfile, unitVector: ArrayLike<number>): Uint8Array;
573
662
  /** binary: sign threshold 0 (exact 0.0 → 0), packed LSB-first. */
574
663
  declare function quantizeBinary(unitVector: ArrayLike<number>): Uint8Array;
575
664
  /**
@@ -599,4 +688,4 @@ declare function canonicalizeJson(value: unknown): string;
599
688
  /** URL-ish slug for a document title (display/dedup aid, not an identity). */
600
689
  declare function documentSlug(title: string): string;
601
690
 
602
- export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, type CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, type GezkFormatVersion, type GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, type KnowledgeAssetExtension, type KnowledgeAssetKind, KnowledgeAssetPathSchema, type KnowledgeCatalogManifest, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, type KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, type KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, type KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, type KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, type KnowledgeRegistryIndex, KnowledgeRegistryIndexSchema, type KnowledgeSignature, KnowledgeSignatureSchema, type KnowledgeUri, type KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, type SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder };
691
+ export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, type CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, type GezkFormatVersion, type GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, type KnowledgeAssetExtension, type KnowledgeAssetKind, KnowledgeAssetPathSchema, type KnowledgeCatalogManifest, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, type KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, type KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, type KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, type KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, type KnowledgeRegistryIndex, KnowledgeRegistryIndexSchema, type KnowledgeSignature, KnowledgeSignatureSchema, type KnowledgeUri, type KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, type SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, centerVector, documentSlug, embeddingProfileArtifacts, embeddingProfileCenter, embeddingProfileCenterProblem, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder };
package/dist/index.js CHANGED
@@ -28,7 +28,7 @@ var RepoRelativePathSchema = z2.string().min(1).refine(
28
28
  );
29
29
  var DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
30
30
  var DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
31
- var KnowledgeEmbeddingProfileSchema = z2.object({
31
+ var KnowledgeEmbeddingProfileObject = z2.object({
32
32
  /** e.g. `multilingual-e5-small@1`. A new model, prefix or encoding is a new id. */
33
33
  id: z2.string().min(1),
34
34
  model: z2.object({
@@ -69,13 +69,59 @@ var KnowledgeEmbeddingProfileSchema = z2.object({
69
69
  method: z2.literal("symmetric-linear"),
70
70
  scale: z2.literal(127)
71
71
  }),
72
+ /**
73
+ * The stage-1 sign bits. `sign` takes them straight from the unit
74
+ * vector. `centered-sign` takes them from `vector − center`, where
75
+ * `center` is a fixed vector the profile pins (length = `dimensions`).
76
+ *
77
+ * Why a center exists: some embedding models put most of every vector
78
+ * on one shared direction (multilingual-e5-small: unrelated passages
79
+ * sit at cosine ≈ 0.8 and the corpus mean has norm ≈ 0.87). The sign
80
+ * of a dimension is then decided by that shared component rather than
81
+ * by the passage, hamming distance stops tracking cosine, and stage 1
82
+ * discards the true neighbours before the int8 rerank can see them.
83
+ * Subtracting the center first restores the correlation. The int8
84
+ * vectors are untouched either way, so query vectors stay comparable
85
+ * across both methods; only how the bits are derived differs.
86
+ */
72
87
  binary: z2.object({
73
- method: z2.literal("sign"),
88
+ method: z2.enum(["sign", "centered-sign"]),
74
89
  threshold: z2.literal(0),
75
- packing: z2.literal("lsb-first")
90
+ packing: z2.literal("lsb-first"),
91
+ /** Required by `centered-sign`, refused by `sign`; one entry per dimension. */
92
+ center: z2.array(z2.number()).optional()
76
93
  })
77
94
  })
78
95
  });
96
+ function embeddingProfileCenterProblem(profile) {
97
+ const { method, center } = profile.quantization.binary;
98
+ if (method === "centered-sign") {
99
+ if (!center) return "centered-sign requires quantization.binary.center";
100
+ if (center.length !== profile.dimensions) {
101
+ return `quantization.binary.center has ${center.length} entries, dimensions is ${profile.dimensions}`;
102
+ }
103
+ return null;
104
+ }
105
+ return center ? "quantization.binary.center is only valid with method centered-sign" : null;
106
+ }
107
+ function embeddingProfileCenter(profile) {
108
+ const problem = embeddingProfileCenterProblem(profile);
109
+ if (problem) throw new Error(`invalid embedding profile ${profile.id}: ${problem}`);
110
+ const { center } = profile.quantization.binary;
111
+ return center ? Float32Array.from(center) : null;
112
+ }
113
+ var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefine(
114
+ (profile, ctx) => {
115
+ const problem = embeddingProfileCenterProblem(profile);
116
+ if (problem) {
117
+ ctx.addIssue({
118
+ code: "custom",
119
+ message: problem,
120
+ path: ["quantization", "binary", "center"]
121
+ });
122
+ }
123
+ }
124
+ );
79
125
  function embeddingProfileArtifacts(profile) {
80
126
  return {
81
127
  onnxFile: profile.model.onnxFile ?? DEFAULT_EMBEDDING_ONNX_FILE,
@@ -88,7 +134,7 @@ function sameVectorSpace(a, b) {
88
134
  const aa = embeddingProfileArtifacts(a);
89
135
  const ab = embeddingProfileArtifacts(b);
90
136
  const digestsAgree = (x, y) => !x || !y || x === y;
91
- return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale && a.quantization.binary.method === b.quantization.binary.method && a.quantization.binary.threshold === b.quantization.binary.threshold && a.quantization.binary.packing === b.quantization.binary.packing;
137
+ return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale;
92
138
  }
93
139
  var KnowledgeChunkingProfileSchema = z2.object({
94
140
  /** e.g. `markdown-chunks@2`. */
@@ -639,6 +685,18 @@ function quantizeInt8(unitVector) {
639
685
  }
640
686
  return out;
641
687
  }
688
+ function centerVector(vector, center) {
689
+ if (vector.length !== center.length) {
690
+ throw new Error(`vector has ${vector.length} dimensions, center has ${center.length}`);
691
+ }
692
+ const out = new Float32Array(vector.length);
693
+ for (let i = 0; i < vector.length; i++) out[i] = vector[i] - center[i];
694
+ return out;
695
+ }
696
+ function quantizeBinaryForProfile(profile, unitVector) {
697
+ const center = embeddingProfileCenter(profile);
698
+ return quantizeBinary(center ? centerVector(unitVector, center) : unitVector);
699
+ }
642
700
  function quantizeBinary(unitVector) {
643
701
  const bytes = Math.ceil(unitVector.length / 8);
644
702
  const out = new Uint8Array(bytes);
@@ -777,8 +835,11 @@ export {
777
835
  assetExtension,
778
836
  assetKindForExtension,
779
837
  canonicalizeJson,
838
+ centerVector,
780
839
  documentSlug,
781
840
  embeddingProfileArtifacts,
841
+ embeddingProfileCenter,
842
+ embeddingProfileCenterProblem,
782
843
  formatKnowledgeUri,
783
844
  isKnowledgeAssetPath,
784
845
  isSupportedFormatVersion,
@@ -786,6 +847,7 @@ export {
786
847
  l2Normalize,
787
848
  parseKnowledgeUri,
788
849
  quantizeBinary,
850
+ quantizeBinaryForProfile,
789
851
  quantizeInt8,
790
852
  rerankScore,
791
853
  sameVectorSpace,
package/dist/node.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { KnowledgeCatalogManifest, KnowledgeRegistryIndex } from './index.js';
2
- export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, KnowledgeAssetExtension, KnowledgeAssetKind, KnowledgeAssetPathSchema, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, KnowledgeRegistryIndexSchema, KnowledgeSignature, KnowledgeSignatureSchema, KnowledgeUri, KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder } from './index.js';
2
+ export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, KnowledgeAssetExtension, KnowledgeAssetKind, KnowledgeAssetPathSchema, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, KnowledgeRegistryIndexSchema, KnowledgeSignature, KnowledgeSignatureSchema, KnowledgeUri, KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, centerVector, documentSlug, embeddingProfileArtifacts, embeddingProfileCenter, embeddingProfileCenterProblem, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder } from './index.js';
3
3
  import 'zod';
4
4
 
5
5
  /**
package/dist/node.js CHANGED
@@ -28,7 +28,7 @@ var RepoRelativePathSchema = z2.string().min(1).refine(
28
28
  );
29
29
  var DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
30
30
  var DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
31
- var KnowledgeEmbeddingProfileSchema = z2.object({
31
+ var KnowledgeEmbeddingProfileObject = z2.object({
32
32
  /** e.g. `multilingual-e5-small@1`. A new model, prefix or encoding is a new id. */
33
33
  id: z2.string().min(1),
34
34
  model: z2.object({
@@ -69,13 +69,59 @@ var KnowledgeEmbeddingProfileSchema = z2.object({
69
69
  method: z2.literal("symmetric-linear"),
70
70
  scale: z2.literal(127)
71
71
  }),
72
+ /**
73
+ * The stage-1 sign bits. `sign` takes them straight from the unit
74
+ * vector. `centered-sign` takes them from `vector − center`, where
75
+ * `center` is a fixed vector the profile pins (length = `dimensions`).
76
+ *
77
+ * Why a center exists: some embedding models put most of every vector
78
+ * on one shared direction (multilingual-e5-small: unrelated passages
79
+ * sit at cosine ≈ 0.8 and the corpus mean has norm ≈ 0.87). The sign
80
+ * of a dimension is then decided by that shared component rather than
81
+ * by the passage, hamming distance stops tracking cosine, and stage 1
82
+ * discards the true neighbours before the int8 rerank can see them.
83
+ * Subtracting the center first restores the correlation. The int8
84
+ * vectors are untouched either way, so query vectors stay comparable
85
+ * across both methods; only how the bits are derived differs.
86
+ */
72
87
  binary: z2.object({
73
- method: z2.literal("sign"),
88
+ method: z2.enum(["sign", "centered-sign"]),
74
89
  threshold: z2.literal(0),
75
- packing: z2.literal("lsb-first")
90
+ packing: z2.literal("lsb-first"),
91
+ /** Required by `centered-sign`, refused by `sign`; one entry per dimension. */
92
+ center: z2.array(z2.number()).optional()
76
93
  })
77
94
  })
78
95
  });
96
+ function embeddingProfileCenterProblem(profile) {
97
+ const { method, center } = profile.quantization.binary;
98
+ if (method === "centered-sign") {
99
+ if (!center) return "centered-sign requires quantization.binary.center";
100
+ if (center.length !== profile.dimensions) {
101
+ return `quantization.binary.center has ${center.length} entries, dimensions is ${profile.dimensions}`;
102
+ }
103
+ return null;
104
+ }
105
+ return center ? "quantization.binary.center is only valid with method centered-sign" : null;
106
+ }
107
+ function embeddingProfileCenter(profile) {
108
+ const problem = embeddingProfileCenterProblem(profile);
109
+ if (problem) throw new Error(`invalid embedding profile ${profile.id}: ${problem}`);
110
+ const { center } = profile.quantization.binary;
111
+ return center ? Float32Array.from(center) : null;
112
+ }
113
+ var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefine(
114
+ (profile, ctx) => {
115
+ const problem = embeddingProfileCenterProblem(profile);
116
+ if (problem) {
117
+ ctx.addIssue({
118
+ code: "custom",
119
+ message: problem,
120
+ path: ["quantization", "binary", "center"]
121
+ });
122
+ }
123
+ }
124
+ );
79
125
  function embeddingProfileArtifacts(profile) {
80
126
  return {
81
127
  onnxFile: profile.model.onnxFile ?? DEFAULT_EMBEDDING_ONNX_FILE,
@@ -88,7 +134,7 @@ function sameVectorSpace(a, b) {
88
134
  const aa = embeddingProfileArtifacts(a);
89
135
  const ab = embeddingProfileArtifacts(b);
90
136
  const digestsAgree = (x, y) => !x || !y || x === y;
91
- return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale && a.quantization.binary.method === b.quantization.binary.method && a.quantization.binary.threshold === b.quantization.binary.threshold && a.quantization.binary.packing === b.quantization.binary.packing;
137
+ return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale;
92
138
  }
93
139
  var KnowledgeChunkingProfileSchema = z2.object({
94
140
  /** e.g. `markdown-chunks@2`. */
@@ -639,6 +685,18 @@ function quantizeInt8(unitVector) {
639
685
  }
640
686
  return out;
641
687
  }
688
+ function centerVector(vector, center) {
689
+ if (vector.length !== center.length) {
690
+ throw new Error(`vector has ${vector.length} dimensions, center has ${center.length}`);
691
+ }
692
+ const out = new Float32Array(vector.length);
693
+ for (let i = 0; i < vector.length; i++) out[i] = vector[i] - center[i];
694
+ return out;
695
+ }
696
+ function quantizeBinaryForProfile(profile, unitVector) {
697
+ const center = embeddingProfileCenter(profile);
698
+ return quantizeBinary(center ? centerVector(unitVector, center) : unitVector);
699
+ }
642
700
  function quantizeBinary(unitVector) {
643
701
  const bytes = Math.ceil(unitVector.length / 8);
644
702
  const out = new Uint8Array(bytes);
@@ -897,10 +955,13 @@ export {
897
955
  assetExtension,
898
956
  assetKindForExtension,
899
957
  canonicalizeJson,
958
+ centerVector,
900
959
  chunkContentHash,
901
960
  chunkUid,
902
961
  documentSlug,
903
962
  embeddingProfileArtifacts,
963
+ embeddingProfileCenter,
964
+ embeddingProfileCenterProblem,
904
965
  formatKnowledgeUri,
905
966
  generateKnowledgeSigningKeyPair,
906
967
  hashFileStreaming,
@@ -912,6 +973,7 @@ export {
912
973
  manifestSigningPayload,
913
974
  parseKnowledgeUri,
914
975
  quantizeBinary,
976
+ quantizeBinaryForProfile,
915
977
  quantizeInt8,
916
978
  rerankScore,
917
979
  sameVectorSpace,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bendyline/gezk",
3
- "version": "1.0.1",
3
+ "version": "1.0.2",
4
4
  "description": "The gezk knowledge-catalog format: manifest and registry schemas, id grammars, knowledge:// references, vector quantization, SQLite DDL, canonical JSON, and Ed25519 manifest signatures.",
5
5
  "license": "MIT",
6
6
  "keywords": [