@bendyline/gezk 1.0.1 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +102 -13
- package/dist/index.js +66 -4
- package/dist/node.d.ts +1 -1
- package/dist/node.js +66 -4
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -48,6 +48,63 @@ declare const DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
|
|
|
48
48
|
* name or a dimension is unsafe — two 384-dim models do not share a space,
|
|
49
49
|
* and an instruction-prefix change silently changes vectors.
|
|
50
50
|
*/
|
|
51
|
+
declare const KnowledgeEmbeddingProfileObject: z.ZodObject<{
|
|
52
|
+
id: z.ZodString;
|
|
53
|
+
model: z.ZodObject<{
|
|
54
|
+
repo: z.ZodString;
|
|
55
|
+
revision: z.ZodString;
|
|
56
|
+
onnxFile: z.ZodOptional<z.ZodString>;
|
|
57
|
+
onnxDigest: z.ZodOptional<z.ZodString>;
|
|
58
|
+
}, z.core.$strip>;
|
|
59
|
+
tokenizer: z.ZodObject<{
|
|
60
|
+
kind: z.ZodString;
|
|
61
|
+
file: z.ZodOptional<z.ZodString>;
|
|
62
|
+
digest: z.ZodOptional<z.ZodString>;
|
|
63
|
+
}, z.core.$strip>;
|
|
64
|
+
pooling: z.ZodEnum<{
|
|
65
|
+
mean: "mean";
|
|
66
|
+
cls: "cls";
|
|
67
|
+
last: "last";
|
|
68
|
+
}>;
|
|
69
|
+
normalized: z.ZodBoolean;
|
|
70
|
+
dimensions: z.ZodNumber;
|
|
71
|
+
maxTokens: z.ZodNumber;
|
|
72
|
+
queryInstruction: z.ZodString;
|
|
73
|
+
passageInstruction: z.ZodString;
|
|
74
|
+
vectorEncoding: z.ZodEnum<{
|
|
75
|
+
"bit+int8": "bit+int8";
|
|
76
|
+
}>;
|
|
77
|
+
distance: z.ZodObject<{
|
|
78
|
+
stage1: z.ZodLiteral<"hamming">;
|
|
79
|
+
stage2: z.ZodLiteral<"cosine">;
|
|
80
|
+
}, z.core.$strip>;
|
|
81
|
+
quantization: z.ZodObject<{
|
|
82
|
+
int8: z.ZodObject<{
|
|
83
|
+
method: z.ZodLiteral<"symmetric-linear">;
|
|
84
|
+
scale: z.ZodLiteral<127>;
|
|
85
|
+
}, z.core.$strip>;
|
|
86
|
+
binary: z.ZodObject<{
|
|
87
|
+
method: z.ZodEnum<{
|
|
88
|
+
sign: "sign";
|
|
89
|
+
"centered-sign": "centered-sign";
|
|
90
|
+
}>;
|
|
91
|
+
threshold: z.ZodLiteral<0>;
|
|
92
|
+
packing: z.ZodLiteral<"lsb-first">;
|
|
93
|
+
center: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
94
|
+
}, z.core.$strip>;
|
|
95
|
+
}, z.core.$strip>;
|
|
96
|
+
}, z.core.$strip>;
|
|
97
|
+
type KnowledgeEmbeddingProfile = z.infer<typeof KnowledgeEmbeddingProfileObject>;
|
|
98
|
+
/**
|
|
99
|
+
* The cross-field rule the object schema cannot express on its own: a
|
|
100
|
+
* `centered-sign` profile must carry a center of exactly `dimensions`
|
|
101
|
+
* entries, and a `sign` profile must not carry one. Returns the problem, or
|
|
102
|
+
* null when the profile is consistent.
|
|
103
|
+
*/
|
|
104
|
+
declare function embeddingProfileCenterProblem(profile: KnowledgeEmbeddingProfile): string | null;
|
|
105
|
+
/** The profile's binary center as a Float32Array, or null for a plain `sign` profile. */
|
|
106
|
+
declare function embeddingProfileCenter(profile: KnowledgeEmbeddingProfile): Float32Array | null;
|
|
107
|
+
/** The profile schema proper: the object shape plus the center rule above. */
|
|
51
108
|
declare const KnowledgeEmbeddingProfileSchema: z.ZodObject<{
|
|
52
109
|
id: z.ZodString;
|
|
53
110
|
model: z.ZodObject<{
|
|
@@ -84,13 +141,16 @@ declare const KnowledgeEmbeddingProfileSchema: z.ZodObject<{
|
|
|
84
141
|
scale: z.ZodLiteral<127>;
|
|
85
142
|
}, z.core.$strip>;
|
|
86
143
|
binary: z.ZodObject<{
|
|
87
|
-
method: z.
|
|
144
|
+
method: z.ZodEnum<{
|
|
145
|
+
sign: "sign";
|
|
146
|
+
"centered-sign": "centered-sign";
|
|
147
|
+
}>;
|
|
88
148
|
threshold: z.ZodLiteral<0>;
|
|
89
149
|
packing: z.ZodLiteral<"lsb-first">;
|
|
150
|
+
center: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
90
151
|
}, z.core.$strip>;
|
|
91
152
|
}, z.core.$strip>;
|
|
92
153
|
}, z.core.$strip>;
|
|
93
|
-
type KnowledgeEmbeddingProfile = z.infer<typeof KnowledgeEmbeddingProfileSchema>;
|
|
94
154
|
/** The exact repo files a profile pins, with the format defaults applied. */
|
|
95
155
|
declare function embeddingProfileArtifacts(profile: KnowledgeEmbeddingProfile): {
|
|
96
156
|
onnxFile: string;
|
|
@@ -101,12 +161,19 @@ declare function embeddingProfileArtifacts(profile: KnowledgeEmbeddingProfile):
|
|
|
101
161
|
/**
|
|
102
162
|
* Whether two profiles describe one vector space, so vectors from either
|
|
103
163
|
* may be compared: same model files at the same revision, same tokenizer,
|
|
104
|
-
* pooling, normalization, dimensions, instruction prefixes and
|
|
105
|
-
* The profile id and `maxTokens` are not compared — the
|
|
106
|
-
* the second a compile-time bound. A digest counts only
|
|
107
|
-
* declare one: two pins at one revision that hash
|
|
108
|
-
* different artifacts, whatever the path says; an
|
|
109
|
-
* simply not a claim.
|
|
164
|
+
* pooling, normalization, dimensions, instruction prefixes, encoding and
|
|
165
|
+
* int8 quantization. The profile id and `maxTokens` are not compared — the
|
|
166
|
+
* first is a label, the second a compile-time bound. A digest counts only
|
|
167
|
+
* when both sides declare one: two pins at one revision that hash
|
|
168
|
+
* differently name different artifacts, whatever the path says; an
|
|
169
|
+
* undeclared digest is simply not a claim.
|
|
170
|
+
*
|
|
171
|
+
* The binary (stage-1) parameters are deliberately NOT compared. They
|
|
172
|
+
* describe how a catalog derived its own pre-filter bits, and a reader
|
|
173
|
+
* applies them from the catalog's profile echo; they never change what a
|
|
174
|
+
* query vector is. A daemon that shares a model with a `sign` profile
|
|
175
|
+
* therefore also shares it with the `centered-sign` revision of the same
|
|
176
|
+
* model — same floats in, same int8 rerank, only the bit scan differs.
|
|
110
177
|
*/
|
|
111
178
|
declare function sameVectorSpace(a: KnowledgeEmbeddingProfile, b: KnowledgeEmbeddingProfile): boolean;
|
|
112
179
|
/**
|
|
@@ -217,9 +284,13 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
|
217
284
|
scale: z.ZodLiteral<127>;
|
|
218
285
|
}, z.core.$strip>;
|
|
219
286
|
binary: z.ZodObject<{
|
|
220
|
-
method: z.
|
|
287
|
+
method: z.ZodEnum<{
|
|
288
|
+
sign: "sign";
|
|
289
|
+
"centered-sign": "centered-sign";
|
|
290
|
+
}>;
|
|
221
291
|
threshold: z.ZodLiteral<0>;
|
|
222
292
|
packing: z.ZodLiteral<"lsb-first">;
|
|
293
|
+
center: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
223
294
|
}, z.core.$strip>;
|
|
224
295
|
}, z.core.$strip>;
|
|
225
296
|
}, z.core.$strip>;
|
|
@@ -490,9 +561,11 @@ declare const ZIP_FIXED_MTIME: Date;
|
|
|
490
561
|
declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n ordinal INTEGER,\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n meta_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, ordinal, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
|
|
491
562
|
/**
|
|
492
563
|
* Per-shard tables. Vectors are plain BLOB columns keyed by chunk id:
|
|
493
|
-
* `chunk_vectors_bit.v` holds ceil(dim/8) bytes of sign bits (LSB-first
|
|
494
|
-
* `
|
|
495
|
-
*
|
|
564
|
+
* `chunk_vectors_bit.v` holds ceil(dim/8) bytes of sign bits (LSB-first; of
|
|
565
|
+
* the unit vector for a `sign` profile, of `vector − center` for a
|
|
566
|
+
* `centered-sign` profile — the profile echo in the router's meta says
|
|
567
|
+
* which) and `chunk_vectors_int8.v` holds dim signed bytes
|
|
568
|
+
* (symmetric-linear, scale 127, never centered). Two tables, not one, so a stage-1 scan pages in 48 bytes per row
|
|
496
569
|
* rather than the whole record. Rowid alignment invariant:
|
|
497
570
|
* chunks.id == chunk_vectors_bit.chunk_id == chunk_vectors_int8.chunk_id ==
|
|
498
571
|
* fts_chunks.rowid, dense from 1.
|
|
@@ -567,9 +640,25 @@ declare function topicSortKeyForOrder(order: number): string;
|
|
|
567
640
|
* functions, so extension-internal semantics can never drift the format:
|
|
568
641
|
* both the compiler (packing) and the reader (query packing + rerank) call
|
|
569
642
|
* exactly these functions.
|
|
643
|
+
*
|
|
644
|
+
* `quantizeBinary` is the raw sign packing. What a catalog actually stores
|
|
645
|
+
* depends on the profile's `quantization.binary.method`: `sign` packs the
|
|
646
|
+
* unit vector, `centered-sign` packs `vector − center` (see the profile
|
|
647
|
+
* schema for why). `quantizeBinaryForProfile` is the one entry point that
|
|
648
|
+
* applies that rule, so the compiler and every reader derive bits the same
|
|
649
|
+
* way.
|
|
570
650
|
*/
|
|
651
|
+
|
|
571
652
|
/** int8: symmetric linear, scale 127, −128 never produced. */
|
|
572
653
|
declare function quantizeInt8(unitVector: ArrayLike<number>): Int8Array;
|
|
654
|
+
/** `vector − center`, the input to a `centered-sign` profile's bits. */
|
|
655
|
+
declare function centerVector(vector: ArrayLike<number>, center: ArrayLike<number>): Float32Array;
|
|
656
|
+
/**
|
|
657
|
+
* The sign bits a profile stores for a unit vector — and, symmetrically,
|
|
658
|
+
* the bits a hamming reader packs for a query: centered first when the
|
|
659
|
+
* profile pins a center, raw otherwise.
|
|
660
|
+
*/
|
|
661
|
+
declare function quantizeBinaryForProfile(profile: KnowledgeEmbeddingProfile, unitVector: ArrayLike<number>): Uint8Array;
|
|
573
662
|
/** binary: sign threshold 0 (exact 0.0 → 0), packed LSB-first. */
|
|
574
663
|
declare function quantizeBinary(unitVector: ArrayLike<number>): Uint8Array;
|
|
575
664
|
/**
|
|
@@ -599,4 +688,4 @@ declare function canonicalizeJson(value: unknown): string;
|
|
|
599
688
|
/** URL-ish slug for a document title (display/dedup aid, not an identity). */
|
|
600
689
|
declare function documentSlug(title: string): string;
|
|
601
690
|
|
|
602
|
-
export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, type CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, type GezkFormatVersion, type GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, type KnowledgeAssetExtension, type KnowledgeAssetKind, KnowledgeAssetPathSchema, type KnowledgeCatalogManifest, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, type KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, type KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, type KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, type KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, type KnowledgeRegistryIndex, KnowledgeRegistryIndexSchema, type KnowledgeSignature, KnowledgeSignatureSchema, type KnowledgeUri, type KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, type SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder };
|
|
691
|
+
export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, type CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, type GezkFormatVersion, type GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, type KnowledgeAssetExtension, type KnowledgeAssetKind, KnowledgeAssetPathSchema, type KnowledgeCatalogManifest, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, type KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, type KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, type KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, type KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, type KnowledgeRegistryIndex, KnowledgeRegistryIndexSchema, type KnowledgeSignature, KnowledgeSignatureSchema, type KnowledgeUri, type KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, type SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, centerVector, documentSlug, embeddingProfileArtifacts, embeddingProfileCenter, embeddingProfileCenterProblem, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder };
|
package/dist/index.js
CHANGED
|
@@ -28,7 +28,7 @@ var RepoRelativePathSchema = z2.string().min(1).refine(
|
|
|
28
28
|
);
|
|
29
29
|
var DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
|
|
30
30
|
var DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
|
|
31
|
-
var
|
|
31
|
+
var KnowledgeEmbeddingProfileObject = z2.object({
|
|
32
32
|
/** e.g. `multilingual-e5-small@1`. A new model, prefix or encoding is a new id. */
|
|
33
33
|
id: z2.string().min(1),
|
|
34
34
|
model: z2.object({
|
|
@@ -69,13 +69,59 @@ var KnowledgeEmbeddingProfileSchema = z2.object({
|
|
|
69
69
|
method: z2.literal("symmetric-linear"),
|
|
70
70
|
scale: z2.literal(127)
|
|
71
71
|
}),
|
|
72
|
+
/**
|
|
73
|
+
* The stage-1 sign bits. `sign` takes them straight from the unit
|
|
74
|
+
* vector. `centered-sign` takes them from `vector − center`, where
|
|
75
|
+
* `center` is a fixed vector the profile pins (length = `dimensions`).
|
|
76
|
+
*
|
|
77
|
+
* Why a center exists: some embedding models put most of every vector
|
|
78
|
+
* on one shared direction (multilingual-e5-small: unrelated passages
|
|
79
|
+
* sit at cosine ≈ 0.8 and the corpus mean has norm ≈ 0.87). The sign
|
|
80
|
+
* of a dimension is then decided by that shared component rather than
|
|
81
|
+
* by the passage, hamming distance stops tracking cosine, and stage 1
|
|
82
|
+
* discards the true neighbours before the int8 rerank can see them.
|
|
83
|
+
* Subtracting the center first restores the correlation. The int8
|
|
84
|
+
* vectors are untouched either way, so query vectors stay comparable
|
|
85
|
+
* across both methods; only how the bits are derived differs.
|
|
86
|
+
*/
|
|
72
87
|
binary: z2.object({
|
|
73
|
-
method: z2.
|
|
88
|
+
method: z2.enum(["sign", "centered-sign"]),
|
|
74
89
|
threshold: z2.literal(0),
|
|
75
|
-
packing: z2.literal("lsb-first")
|
|
90
|
+
packing: z2.literal("lsb-first"),
|
|
91
|
+
/** Required by `centered-sign`, refused by `sign`; one entry per dimension. */
|
|
92
|
+
center: z2.array(z2.number()).optional()
|
|
76
93
|
})
|
|
77
94
|
})
|
|
78
95
|
});
|
|
96
|
+
function embeddingProfileCenterProblem(profile) {
|
|
97
|
+
const { method, center } = profile.quantization.binary;
|
|
98
|
+
if (method === "centered-sign") {
|
|
99
|
+
if (!center) return "centered-sign requires quantization.binary.center";
|
|
100
|
+
if (center.length !== profile.dimensions) {
|
|
101
|
+
return `quantization.binary.center has ${center.length} entries, dimensions is ${profile.dimensions}`;
|
|
102
|
+
}
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
return center ? "quantization.binary.center is only valid with method centered-sign" : null;
|
|
106
|
+
}
|
|
107
|
+
function embeddingProfileCenter(profile) {
|
|
108
|
+
const problem = embeddingProfileCenterProblem(profile);
|
|
109
|
+
if (problem) throw new Error(`invalid embedding profile ${profile.id}: ${problem}`);
|
|
110
|
+
const { center } = profile.quantization.binary;
|
|
111
|
+
return center ? Float32Array.from(center) : null;
|
|
112
|
+
}
|
|
113
|
+
var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefine(
|
|
114
|
+
(profile, ctx) => {
|
|
115
|
+
const problem = embeddingProfileCenterProblem(profile);
|
|
116
|
+
if (problem) {
|
|
117
|
+
ctx.addIssue({
|
|
118
|
+
code: "custom",
|
|
119
|
+
message: problem,
|
|
120
|
+
path: ["quantization", "binary", "center"]
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
);
|
|
79
125
|
function embeddingProfileArtifacts(profile) {
|
|
80
126
|
return {
|
|
81
127
|
onnxFile: profile.model.onnxFile ?? DEFAULT_EMBEDDING_ONNX_FILE,
|
|
@@ -88,7 +134,7 @@ function sameVectorSpace(a, b) {
|
|
|
88
134
|
const aa = embeddingProfileArtifacts(a);
|
|
89
135
|
const ab = embeddingProfileArtifacts(b);
|
|
90
136
|
const digestsAgree = (x, y) => !x || !y || x === y;
|
|
91
|
-
return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale
|
|
137
|
+
return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale;
|
|
92
138
|
}
|
|
93
139
|
var KnowledgeChunkingProfileSchema = z2.object({
|
|
94
140
|
/** e.g. `markdown-chunks@2`. */
|
|
@@ -639,6 +685,18 @@ function quantizeInt8(unitVector) {
|
|
|
639
685
|
}
|
|
640
686
|
return out;
|
|
641
687
|
}
|
|
688
|
+
function centerVector(vector, center) {
|
|
689
|
+
if (vector.length !== center.length) {
|
|
690
|
+
throw new Error(`vector has ${vector.length} dimensions, center has ${center.length}`);
|
|
691
|
+
}
|
|
692
|
+
const out = new Float32Array(vector.length);
|
|
693
|
+
for (let i = 0; i < vector.length; i++) out[i] = vector[i] - center[i];
|
|
694
|
+
return out;
|
|
695
|
+
}
|
|
696
|
+
function quantizeBinaryForProfile(profile, unitVector) {
|
|
697
|
+
const center = embeddingProfileCenter(profile);
|
|
698
|
+
return quantizeBinary(center ? centerVector(unitVector, center) : unitVector);
|
|
699
|
+
}
|
|
642
700
|
function quantizeBinary(unitVector) {
|
|
643
701
|
const bytes = Math.ceil(unitVector.length / 8);
|
|
644
702
|
const out = new Uint8Array(bytes);
|
|
@@ -777,8 +835,11 @@ export {
|
|
|
777
835
|
assetExtension,
|
|
778
836
|
assetKindForExtension,
|
|
779
837
|
canonicalizeJson,
|
|
838
|
+
centerVector,
|
|
780
839
|
documentSlug,
|
|
781
840
|
embeddingProfileArtifacts,
|
|
841
|
+
embeddingProfileCenter,
|
|
842
|
+
embeddingProfileCenterProblem,
|
|
782
843
|
formatKnowledgeUri,
|
|
783
844
|
isKnowledgeAssetPath,
|
|
784
845
|
isSupportedFormatVersion,
|
|
@@ -786,6 +847,7 @@ export {
|
|
|
786
847
|
l2Normalize,
|
|
787
848
|
parseKnowledgeUri,
|
|
788
849
|
quantizeBinary,
|
|
850
|
+
quantizeBinaryForProfile,
|
|
789
851
|
quantizeInt8,
|
|
790
852
|
rerankScore,
|
|
791
853
|
sameVectorSpace,
|
package/dist/node.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { KnowledgeCatalogManifest, KnowledgeRegistryIndex } from './index.js';
|
|
2
|
-
export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, KnowledgeAssetExtension, KnowledgeAssetKind, KnowledgeAssetPathSchema, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, KnowledgeRegistryIndexSchema, KnowledgeSignature, KnowledgeSignatureSchema, KnowledgeUri, KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder } from './index.js';
|
|
2
|
+
export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, KnowledgeAssetExtension, KnowledgeAssetKind, KnowledgeAssetPathSchema, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, KnowledgeRegistryIndexSchema, KnowledgeSignature, KnowledgeSignatureSchema, KnowledgeUri, KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, centerVector, documentSlug, embeddingProfileArtifacts, embeddingProfileCenter, embeddingProfileCenterProblem, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder } from './index.js';
|
|
3
3
|
import 'zod';
|
|
4
4
|
|
|
5
5
|
/**
|
package/dist/node.js
CHANGED
|
@@ -28,7 +28,7 @@ var RepoRelativePathSchema = z2.string().min(1).refine(
|
|
|
28
28
|
);
|
|
29
29
|
var DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
|
|
30
30
|
var DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
|
|
31
|
-
var
|
|
31
|
+
var KnowledgeEmbeddingProfileObject = z2.object({
|
|
32
32
|
/** e.g. `multilingual-e5-small@1`. A new model, prefix or encoding is a new id. */
|
|
33
33
|
id: z2.string().min(1),
|
|
34
34
|
model: z2.object({
|
|
@@ -69,13 +69,59 @@ var KnowledgeEmbeddingProfileSchema = z2.object({
|
|
|
69
69
|
method: z2.literal("symmetric-linear"),
|
|
70
70
|
scale: z2.literal(127)
|
|
71
71
|
}),
|
|
72
|
+
/**
|
|
73
|
+
* The stage-1 sign bits. `sign` takes them straight from the unit
|
|
74
|
+
* vector. `centered-sign` takes them from `vector − center`, where
|
|
75
|
+
* `center` is a fixed vector the profile pins (length = `dimensions`).
|
|
76
|
+
*
|
|
77
|
+
* Why a center exists: some embedding models put most of every vector
|
|
78
|
+
* on one shared direction (multilingual-e5-small: unrelated passages
|
|
79
|
+
* sit at cosine ≈ 0.8 and the corpus mean has norm ≈ 0.87). The sign
|
|
80
|
+
* of a dimension is then decided by that shared component rather than
|
|
81
|
+
* by the passage, hamming distance stops tracking cosine, and stage 1
|
|
82
|
+
* discards the true neighbours before the int8 rerank can see them.
|
|
83
|
+
* Subtracting the center first restores the correlation. The int8
|
|
84
|
+
* vectors are untouched either way, so query vectors stay comparable
|
|
85
|
+
* across both methods; only how the bits are derived differs.
|
|
86
|
+
*/
|
|
72
87
|
binary: z2.object({
|
|
73
|
-
method: z2.
|
|
88
|
+
method: z2.enum(["sign", "centered-sign"]),
|
|
74
89
|
threshold: z2.literal(0),
|
|
75
|
-
packing: z2.literal("lsb-first")
|
|
90
|
+
packing: z2.literal("lsb-first"),
|
|
91
|
+
/** Required by `centered-sign`, refused by `sign`; one entry per dimension. */
|
|
92
|
+
center: z2.array(z2.number()).optional()
|
|
76
93
|
})
|
|
77
94
|
})
|
|
78
95
|
});
|
|
96
|
+
function embeddingProfileCenterProblem(profile) {
|
|
97
|
+
const { method, center } = profile.quantization.binary;
|
|
98
|
+
if (method === "centered-sign") {
|
|
99
|
+
if (!center) return "centered-sign requires quantization.binary.center";
|
|
100
|
+
if (center.length !== profile.dimensions) {
|
|
101
|
+
return `quantization.binary.center has ${center.length} entries, dimensions is ${profile.dimensions}`;
|
|
102
|
+
}
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
return center ? "quantization.binary.center is only valid with method centered-sign" : null;
|
|
106
|
+
}
|
|
107
|
+
function embeddingProfileCenter(profile) {
|
|
108
|
+
const problem = embeddingProfileCenterProblem(profile);
|
|
109
|
+
if (problem) throw new Error(`invalid embedding profile ${profile.id}: ${problem}`);
|
|
110
|
+
const { center } = profile.quantization.binary;
|
|
111
|
+
return center ? Float32Array.from(center) : null;
|
|
112
|
+
}
|
|
113
|
+
var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefine(
|
|
114
|
+
(profile, ctx) => {
|
|
115
|
+
const problem = embeddingProfileCenterProblem(profile);
|
|
116
|
+
if (problem) {
|
|
117
|
+
ctx.addIssue({
|
|
118
|
+
code: "custom",
|
|
119
|
+
message: problem,
|
|
120
|
+
path: ["quantization", "binary", "center"]
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
);
|
|
79
125
|
function embeddingProfileArtifacts(profile) {
|
|
80
126
|
return {
|
|
81
127
|
onnxFile: profile.model.onnxFile ?? DEFAULT_EMBEDDING_ONNX_FILE,
|
|
@@ -88,7 +134,7 @@ function sameVectorSpace(a, b) {
|
|
|
88
134
|
const aa = embeddingProfileArtifacts(a);
|
|
89
135
|
const ab = embeddingProfileArtifacts(b);
|
|
90
136
|
const digestsAgree = (x, y) => !x || !y || x === y;
|
|
91
|
-
return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale
|
|
137
|
+
return a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale;
|
|
92
138
|
}
|
|
93
139
|
var KnowledgeChunkingProfileSchema = z2.object({
|
|
94
140
|
/** e.g. `markdown-chunks@2`. */
|
|
@@ -639,6 +685,18 @@ function quantizeInt8(unitVector) {
|
|
|
639
685
|
}
|
|
640
686
|
return out;
|
|
641
687
|
}
|
|
688
|
+
function centerVector(vector, center) {
|
|
689
|
+
if (vector.length !== center.length) {
|
|
690
|
+
throw new Error(`vector has ${vector.length} dimensions, center has ${center.length}`);
|
|
691
|
+
}
|
|
692
|
+
const out = new Float32Array(vector.length);
|
|
693
|
+
for (let i = 0; i < vector.length; i++) out[i] = vector[i] - center[i];
|
|
694
|
+
return out;
|
|
695
|
+
}
|
|
696
|
+
function quantizeBinaryForProfile(profile, unitVector) {
|
|
697
|
+
const center = embeddingProfileCenter(profile);
|
|
698
|
+
return quantizeBinary(center ? centerVector(unitVector, center) : unitVector);
|
|
699
|
+
}
|
|
642
700
|
function quantizeBinary(unitVector) {
|
|
643
701
|
const bytes = Math.ceil(unitVector.length / 8);
|
|
644
702
|
const out = new Uint8Array(bytes);
|
|
@@ -897,10 +955,13 @@ export {
|
|
|
897
955
|
assetExtension,
|
|
898
956
|
assetKindForExtension,
|
|
899
957
|
canonicalizeJson,
|
|
958
|
+
centerVector,
|
|
900
959
|
chunkContentHash,
|
|
901
960
|
chunkUid,
|
|
902
961
|
documentSlug,
|
|
903
962
|
embeddingProfileArtifacts,
|
|
963
|
+
embeddingProfileCenter,
|
|
964
|
+
embeddingProfileCenterProblem,
|
|
904
965
|
formatKnowledgeUri,
|
|
905
966
|
generateKnowledgeSigningKeyPair,
|
|
906
967
|
hashFileStreaming,
|
|
@@ -912,6 +973,7 @@ export {
|
|
|
912
973
|
manifestSigningPayload,
|
|
913
974
|
parseKnowledgeUri,
|
|
914
975
|
quantizeBinary,
|
|
976
|
+
quantizeBinaryForProfile,
|
|
915
977
|
quantizeInt8,
|
|
916
978
|
rerankScore,
|
|
917
979
|
sameVectorSpace,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/gezk",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.2",
|
|
4
4
|
"description": "The gezk knowledge-catalog format: manifest and registry schemas, id grammars, knowledge:// references, vector quantization, SQLite DDL, canonical JSON, and Ed25519 manifest signatures.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|