@bendyline/gezk 1.0.3 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +355 -42
- package/dist/index.js +455 -22
- package/dist/node.d.ts +1 -1
- package/dist/node.js +455 -22
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -28,6 +28,43 @@ var RepoRelativePathSchema = z2.string().min(1).refine(
|
|
|
28
28
|
);
|
|
29
29
|
var DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
|
|
30
30
|
var DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
|
|
31
|
+
var EmbeddingModelFileSchema = z2.object({
|
|
32
|
+
path: RepoRelativePathSchema,
|
|
33
|
+
digest: ArtifactDigestSchema.optional()
|
|
34
|
+
});
|
|
35
|
+
var EmbeddingMediaEncoderSchema = z2.object({
|
|
36
|
+
onnxFile: RepoRelativePathSchema,
|
|
37
|
+
onnxDigest: ArtifactDigestSchema.optional(),
|
|
38
|
+
files: z2.array(EmbeddingModelFileSchema).optional()
|
|
39
|
+
});
|
|
40
|
+
var EmbeddingMediaSchema = z2.object({
|
|
41
|
+
image: z2.object({
|
|
42
|
+
encoder: EmbeddingMediaEncoderSchema,
|
|
43
|
+
/** Vision tokens per image; changing it moves every image vector. */
|
|
44
|
+
tokenBudget: z2.number().int().positive(),
|
|
45
|
+
resample: z2.enum(["bilinear", "bicubic"]),
|
|
46
|
+
/** Transparent pixels are composited onto white before encoding. */
|
|
47
|
+
alpha: z2.literal("composite-white")
|
|
48
|
+
}).optional(),
|
|
49
|
+
/** Video frames go through the image encoder, so `video` requires `image`. */
|
|
50
|
+
video: z2.object({
|
|
51
|
+
framesPerSecond: z2.number().positive(),
|
|
52
|
+
maxFrames: z2.number().int().positive(),
|
|
53
|
+
tokenBudgetPerFrame: z2.number().int().positive()
|
|
54
|
+
}).optional(),
|
|
55
|
+
audio: z2.object({
|
|
56
|
+
encoder: EmbeddingMediaEncoderSchema,
|
|
57
|
+
sampleRate: z2.number().int().positive(),
|
|
58
|
+
channels: z2.literal(1),
|
|
59
|
+
/** Longest window one audio row may cover. */
|
|
60
|
+
maxWindowMs: z2.number().int().positive()
|
|
61
|
+
}).optional()
|
|
62
|
+
}).refine((media) => Boolean(media.image || media.video || media.audio), {
|
|
63
|
+
message: "media must describe at least one modality"
|
|
64
|
+
}).refine((media) => !media.video || Boolean(media.image), {
|
|
65
|
+
message: "media.video requires media.image (frames go through the image encoder)",
|
|
66
|
+
path: ["video"]
|
|
67
|
+
});
|
|
31
68
|
var KnowledgeEmbeddingProfileObject = z2.object({
|
|
32
69
|
/** e.g. `multilingual-e5-small@1`. A new model, prefix or encoding is a new id. */
|
|
33
70
|
id: z2.string().min(1),
|
|
@@ -44,7 +81,9 @@ var KnowledgeEmbeddingProfileObject = z2.object({
|
|
|
44
81
|
*/
|
|
45
82
|
onnxFile: RepoRelativePathSchema.optional(),
|
|
46
83
|
/** sha256 of that file's bytes at `revision`. */
|
|
47
|
-
onnxDigest: ArtifactDigestSchema.optional()
|
|
84
|
+
onnxDigest: ArtifactDigestSchema.optional(),
|
|
85
|
+
/** Every further file the runtime loads: weight sidecars, configs (0.8). */
|
|
86
|
+
files: z2.array(EmbeddingModelFileSchema).optional()
|
|
48
87
|
}),
|
|
49
88
|
tokenizer: z2.object({
|
|
50
89
|
kind: z2.string().min(1),
|
|
@@ -55,7 +94,17 @@ var KnowledgeEmbeddingProfileObject = z2.object({
|
|
|
55
94
|
}),
|
|
56
95
|
pooling: z2.enum(["mean", "cls", "last"]),
|
|
57
96
|
normalized: z2.boolean(),
|
|
97
|
+
/** The stored vector width — after truncation, when the profile truncates. */
|
|
58
98
|
dimensions: z2.number().int().positive(),
|
|
99
|
+
/**
|
|
100
|
+
* Matryoshka truncation (0.8): the model emits `sourceDimensions` values,
|
|
101
|
+
* the first `dimensions` are kept, and the result is L2-normalized again.
|
|
102
|
+
* One rule for passages, queries and media; `profileUnitVector` owns it.
|
|
103
|
+
*/
|
|
104
|
+
truncation: z2.object({
|
|
105
|
+
method: z2.literal("prefix"),
|
|
106
|
+
sourceDimensions: z2.number().int().positive()
|
|
107
|
+
}).optional(),
|
|
59
108
|
maxTokens: z2.number().int().positive(),
|
|
60
109
|
queryInstruction: z2.string(),
|
|
61
110
|
passageInstruction: z2.string(),
|
|
@@ -91,7 +140,8 @@ var KnowledgeEmbeddingProfileObject = z2.object({
|
|
|
91
140
|
/** Required by `centered-sign`, refused by `sign`; one entry per dimension. */
|
|
92
141
|
center: z2.array(z2.number()).optional()
|
|
93
142
|
})
|
|
94
|
-
})
|
|
143
|
+
}),
|
|
144
|
+
media: EmbeddingMediaSchema.optional()
|
|
95
145
|
});
|
|
96
146
|
function embeddingProfileCenterProblem(profile) {
|
|
97
147
|
const { method, center } = profile.quantization.binary;
|
|
@@ -110,6 +160,19 @@ function embeddingProfileCenter(profile) {
|
|
|
110
160
|
const { center } = profile.quantization.binary;
|
|
111
161
|
return center ? Float32Array.from(center) : null;
|
|
112
162
|
}
|
|
163
|
+
function embeddingProfileTruncationProblem(profile) {
|
|
164
|
+
if (!profile.truncation) return null;
|
|
165
|
+
if (profile.truncation.sourceDimensions <= profile.dimensions) {
|
|
166
|
+
return `truncation.sourceDimensions (${profile.truncation.sourceDimensions}) must exceed dimensions (${profile.dimensions})`;
|
|
167
|
+
}
|
|
168
|
+
return profile.normalized ? null : "truncation requires a normalized profile";
|
|
169
|
+
}
|
|
170
|
+
function embeddingProfileSourceDimensions(profile) {
|
|
171
|
+
return profile.truncation?.sourceDimensions ?? profile.dimensions;
|
|
172
|
+
}
|
|
173
|
+
function embeddingProfileMinimumFormat(profile) {
|
|
174
|
+
return profile.truncation || profile.model.files || profile.media ? "0.8" : "0.5";
|
|
175
|
+
}
|
|
113
176
|
var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefine(
|
|
114
177
|
(profile, ctx) => {
|
|
115
178
|
const problem = embeddingProfileCenterProblem(profile);
|
|
@@ -120,6 +183,10 @@ var KnowledgeEmbeddingProfileSchema = KnowledgeEmbeddingProfileObject.superRefin
|
|
|
120
183
|
path: ["quantization", "binary", "center"]
|
|
121
184
|
});
|
|
122
185
|
}
|
|
186
|
+
const truncation = embeddingProfileTruncationProblem(profile);
|
|
187
|
+
if (truncation) {
|
|
188
|
+
ctx.addIssue({ code: "custom", message: truncation, path: ["truncation"] });
|
|
189
|
+
}
|
|
123
190
|
}
|
|
124
191
|
);
|
|
125
192
|
function embeddingProfileArtifacts(profile) {
|
|
@@ -127,14 +194,19 @@ function embeddingProfileArtifacts(profile) {
|
|
|
127
194
|
onnxFile: profile.model.onnxFile ?? DEFAULT_EMBEDDING_ONNX_FILE,
|
|
128
195
|
onnxDigest: profile.model.onnxDigest ?? null,
|
|
129
196
|
tokenizerFile: profile.tokenizer.file ?? DEFAULT_EMBEDDING_TOKENIZER_FILE,
|
|
130
|
-
tokenizerDigest: profile.tokenizer.digest ?? null
|
|
197
|
+
tokenizerDigest: profile.tokenizer.digest ?? null,
|
|
198
|
+
files: (profile.model.files ?? []).map((f) => ({ path: f.path, digest: f.digest ?? null }))
|
|
131
199
|
};
|
|
132
200
|
}
|
|
133
201
|
function sameVectorSpace(a, b) {
|
|
134
202
|
const aa = embeddingProfileArtifacts(a);
|
|
135
203
|
const ab = embeddingProfileArtifacts(b);
|
|
136
204
|
const digestsAgree = (x, y) => !x || !y || x === y;
|
|
137
|
-
|
|
205
|
+
const bFiles = new Map(ab.files.map((f) => [f.path, f.digest]));
|
|
206
|
+
const filesAgree = aa.files.every(
|
|
207
|
+
(f) => !bFiles.has(f.path) || digestsAgree(f.digest, bFiles.get(f.path) ?? null)
|
|
208
|
+
);
|
|
209
|
+
return filesAgree && a.truncation?.method === b.truncation?.method && embeddingProfileSourceDimensions(a) === embeddingProfileSourceDimensions(b) && a.model.repo === b.model.repo && a.model.revision === b.model.revision && aa.onnxFile === ab.onnxFile && digestsAgree(aa.onnxDigest, ab.onnxDigest) && a.tokenizer.kind === b.tokenizer.kind && aa.tokenizerFile === ab.tokenizerFile && digestsAgree(aa.tokenizerDigest, ab.tokenizerDigest) && a.pooling === b.pooling && a.normalized === b.normalized && a.dimensions === b.dimensions && a.queryInstruction === b.queryInstruction && a.passageInstruction === b.passageInstruction && a.vectorEncoding === b.vectorEncoding && a.distance.stage1 === b.distance.stage1 && a.distance.stage2 === b.distance.stage2 && a.quantization.int8.method === b.quantization.int8.method && a.quantization.int8.scale === b.quantization.int8.scale;
|
|
138
210
|
}
|
|
139
211
|
var KnowledgeChunkingProfileSchema = z2.object({
|
|
140
212
|
/** e.g. `markdown-chunks@2`. */
|
|
@@ -150,14 +222,17 @@ var KnowledgeChunkingProfileSchema = z2.object({
|
|
|
150
222
|
import { z as z4 } from "zod";
|
|
151
223
|
|
|
152
224
|
// src/format/constants.ts
|
|
153
|
-
var GEZK_FORMAT_VERSION = "0.
|
|
154
|
-
var GEZK_INDEX_SCHEMA_VERSION =
|
|
155
|
-
var GEZK_FORMAT_GENERATIONS = { "0.5": 2, "0.6": 3, "0.7": 4 };
|
|
225
|
+
var GEZK_FORMAT_VERSION = "0.8";
|
|
226
|
+
var GEZK_INDEX_SCHEMA_VERSION = 5;
|
|
227
|
+
var GEZK_FORMAT_GENERATIONS = { "0.5": 2, "0.6": 3, "0.7": 4, "0.8": 5 };
|
|
156
228
|
var GEZK_SUPPORTED_FORMAT_VERSIONS = Object.keys(GEZK_FORMAT_GENERATIONS);
|
|
157
229
|
var GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS = Object.values(GEZK_FORMAT_GENERATIONS);
|
|
158
230
|
function isSupportedFormatVersion(value) {
|
|
159
231
|
return typeof value === "string" && Object.hasOwn(GEZK_FORMAT_GENERATIONS, value);
|
|
160
232
|
}
|
|
233
|
+
function formatAtLeast(version, minimum) {
|
|
234
|
+
return GEZK_FORMAT_GENERATIONS[version] >= GEZK_FORMAT_GENERATIONS[minimum];
|
|
235
|
+
}
|
|
161
236
|
function isSupportedIndexSchemaVersion(value) {
|
|
162
237
|
return typeof value === "number" && GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.includes(value);
|
|
163
238
|
}
|
|
@@ -321,7 +396,7 @@ var KnowledgeSignatureSchema = z4.object({
|
|
|
321
396
|
var KnowledgeCatalogManifestSchema = z4.object({
|
|
322
397
|
kind: z4.literal(GEZK_MANIFEST_KIND),
|
|
323
398
|
formatVersion: z4.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
324
|
-
indexSchemaVersion: z4.
|
|
399
|
+
indexSchemaVersion: z4.literal(GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS),
|
|
325
400
|
id: KnowledgeIdSchema,
|
|
326
401
|
version: KnowledgeVersionSchema,
|
|
327
402
|
name: z4.string().min(1),
|
|
@@ -378,7 +453,16 @@ var KnowledgeCatalogManifestSchema = z4.object({
|
|
|
378
453
|
chunks: z4.number().int().nonnegative(),
|
|
379
454
|
shards: z4.number().int().positive(),
|
|
380
455
|
/** Files under `assets/` (0.6+). */
|
|
381
|
-
assets: z4.number().int().nonnegative().optional()
|
|
456
|
+
assets: z4.number().int().nonnegative().optional(),
|
|
457
|
+
/**
|
|
458
|
+
* Media rows by modality (0.8, required there). They are counted in
|
|
459
|
+
* `chunks` too: every row of the shard `chunks` table is a chunk.
|
|
460
|
+
*/
|
|
461
|
+
media: z4.object({
|
|
462
|
+
image: z4.number().int().nonnegative(),
|
|
463
|
+
video: z4.number().int().nonnegative(),
|
|
464
|
+
audio: z4.number().int().nonnegative()
|
|
465
|
+
}).optional()
|
|
382
466
|
}),
|
|
383
467
|
files: z4.array(KnowledgeManifestFileSchema).min(1),
|
|
384
468
|
spatial: KnowledgeSpatialManifestSchema.optional(),
|
|
@@ -405,18 +489,61 @@ var KnowledgeCatalogManifestSchema = z4.object({
|
|
|
405
489
|
}).optional(),
|
|
406
490
|
signature: KnowledgeSignatureSchema.optional()
|
|
407
491
|
}).superRefine((manifest, ctx) => {
|
|
408
|
-
|
|
492
|
+
const spatialEra = formatAtLeast(manifest.formatVersion, "0.7");
|
|
493
|
+
if (spatialEra && !manifest.spatial) {
|
|
409
494
|
ctx.addIssue({
|
|
410
495
|
code: "custom",
|
|
411
496
|
path: ["spatial"],
|
|
412
|
-
message: "0.7 requires a spatial summary, including catalogs with zero locations"
|
|
497
|
+
message: "0.7+ requires a spatial summary, including catalogs with zero locations"
|
|
413
498
|
});
|
|
414
499
|
}
|
|
415
|
-
if (manifest.spatial && (
|
|
500
|
+
if (manifest.spatial && (!spatialEra || manifest.spatial.locatedDocuments > manifest.counts.documents)) {
|
|
416
501
|
ctx.addIssue({
|
|
417
502
|
code: "custom",
|
|
418
503
|
path: ["spatial"],
|
|
419
|
-
message: "spatial requires 0.7 and valid document counts"
|
|
504
|
+
message: "spatial requires 0.7+ and valid document counts"
|
|
505
|
+
});
|
|
506
|
+
}
|
|
507
|
+
const mediaEra = formatAtLeast(manifest.formatVersion, "0.8");
|
|
508
|
+
const media = manifest.counts.media;
|
|
509
|
+
if (mediaEra && !media) {
|
|
510
|
+
ctx.addIssue({
|
|
511
|
+
code: "custom",
|
|
512
|
+
path: ["counts", "media"],
|
|
513
|
+
message: "0.8 requires counts.media, including catalogs with no media rows"
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
if (media && !mediaEra) {
|
|
517
|
+
ctx.addIssue({
|
|
518
|
+
code: "custom",
|
|
519
|
+
path: ["counts", "media"],
|
|
520
|
+
message: "counts.media is a 0.8 field"
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
if (media && media.image + media.video + media.audio > manifest.counts.chunks) {
|
|
524
|
+
ctx.addIssue({
|
|
525
|
+
code: "custom",
|
|
526
|
+
path: ["counts", "media"],
|
|
527
|
+
message: "media rows are chunks, so counts.media cannot exceed counts.chunks"
|
|
528
|
+
});
|
|
529
|
+
}
|
|
530
|
+
if (media) {
|
|
531
|
+
for (const modality of ["image", "video", "audio"]) {
|
|
532
|
+
if (media[modality] > 0 && !manifest.embedding.media?.[modality]) {
|
|
533
|
+
ctx.addIssue({
|
|
534
|
+
code: "custom",
|
|
535
|
+
path: ["embedding", "media", modality],
|
|
536
|
+
message: `${media[modality]} ${modality} rows need embedding.media.${modality}`
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
const profileFormat = embeddingProfileMinimumFormat(manifest.embedding);
|
|
542
|
+
if (!formatAtLeast(manifest.formatVersion, profileFormat)) {
|
|
543
|
+
ctx.addIssue({
|
|
544
|
+
code: "custom",
|
|
545
|
+
path: ["embedding"],
|
|
546
|
+
message: `embedding profile ${manifest.embedding.id} uses ${profileFormat} vocabulary (truncation, model.files or media), which ${manifest.formatVersion} cannot carry`
|
|
420
547
|
});
|
|
421
548
|
}
|
|
422
549
|
if (GEZK_FORMAT_GENERATIONS[manifest.formatVersion] !== manifest.indexSchemaVersion) {
|
|
@@ -669,7 +796,8 @@ CREATE VIRTUAL TABLE fts_documents USING fts5(
|
|
|
669
796
|
tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
|
|
670
797
|
);
|
|
671
798
|
`;
|
|
672
|
-
|
|
799
|
+
function shardDdl(mediaColumns, mediaIndex) {
|
|
800
|
+
return `
|
|
673
801
|
CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
674
802
|
|
|
675
803
|
CREATE TABLE chunks(
|
|
@@ -683,10 +811,10 @@ CREATE TABLE chunks(
|
|
|
683
811
|
line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,
|
|
684
812
|
token_count INTEGER NOT NULL,
|
|
685
813
|
content_hash TEXT NOT NULL,
|
|
686
|
-
text TEXT NOT NULL
|
|
814
|
+
text TEXT NOT NULL${mediaColumns}
|
|
687
815
|
);
|
|
688
816
|
CREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);
|
|
689
|
-
CREATE INDEX chunks_document ON chunks(document_id, ordinal)
|
|
817
|
+
CREATE INDEX chunks_document ON chunks(document_id, ordinal);${mediaIndex}
|
|
690
818
|
|
|
691
819
|
CREATE VIRTUAL TABLE fts_chunks USING fts5(
|
|
692
820
|
title, heading_text, text,
|
|
@@ -704,6 +832,30 @@ CREATE TABLE chunk_vectors_int8(
|
|
|
704
832
|
v BLOB NOT NULL
|
|
705
833
|
);
|
|
706
834
|
`;
|
|
835
|
+
}
|
|
836
|
+
var MEDIA_COLUMNS = `,
|
|
837
|
+
modality TEXT NOT NULL DEFAULT 'text'
|
|
838
|
+
CHECK (modality IN ('text','image','video','audio')),
|
|
839
|
+
asset_path TEXT,
|
|
840
|
+
mime_type TEXT,
|
|
841
|
+
width INTEGER, height INTEGER,
|
|
842
|
+
start_ms INTEGER, end_ms INTEGER,
|
|
843
|
+
thumbnail_path TEXT,
|
|
844
|
+
attribution_json TEXT,
|
|
845
|
+
CHECK (modality <> 'text' OR (asset_path IS NULL AND mime_type IS NULL
|
|
846
|
+
AND width IS NULL AND height IS NULL AND start_ms IS NULL AND end_ms IS NULL
|
|
847
|
+
AND thumbnail_path IS NULL AND attribution_json IS NULL)),
|
|
848
|
+
CHECK (modality = 'text' OR (asset_path IS NOT NULL AND mime_type IS NOT NULL)),
|
|
849
|
+
CHECK (modality <> 'image' OR (start_ms IS NULL AND end_ms IS NULL)),
|
|
850
|
+
CHECK (modality NOT IN ('video','audio') OR (start_ms >= 0 AND end_ms > start_ms))`;
|
|
851
|
+
var MEDIA_INDEX = `
|
|
852
|
+
CREATE INDEX chunks_media ON chunks(modality, id) WHERE modality <> 'text';`;
|
|
853
|
+
var SHARD_DDL_V4 = shardDdl("", "");
|
|
854
|
+
var SHARD_DDL_V5 = shardDdl(MEDIA_COLUMNS, MEDIA_INDEX);
|
|
855
|
+
var SHARD_DDL = SHARD_DDL_V5;
|
|
856
|
+
function shardDdlFor(indexSchemaVersion) {
|
|
857
|
+
return indexSchemaVersion === 5 ? SHARD_DDL_V5 : SHARD_DDL_V4;
|
|
858
|
+
}
|
|
707
859
|
|
|
708
860
|
// src/format/assets.ts
|
|
709
861
|
import { z as z7 } from "zod";
|
|
@@ -716,11 +868,23 @@ var KNOWLEDGE_ASSET_TYPES = {
|
|
|
716
868
|
webp: "image/webp",
|
|
717
869
|
svg: "image/svg+xml"
|
|
718
870
|
};
|
|
871
|
+
var KNOWLEDGE_MEDIA_ASSET_TYPES = {
|
|
872
|
+
mp4: "video/mp4",
|
|
873
|
+
webm: "video/webm",
|
|
874
|
+
mp3: "audio/mpeg",
|
|
875
|
+
m4a: "audio/mp4",
|
|
876
|
+
ogg: "audio/ogg",
|
|
877
|
+
opus: "audio/ogg",
|
|
878
|
+
wav: "audio/wav",
|
|
879
|
+
flac: "audio/flac"
|
|
880
|
+
};
|
|
719
881
|
var MAX_KNOWLEDGE_ASSET_BYTES = 8 * 1024 * 1024;
|
|
720
882
|
var MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES = 256 * 1024 * 1024;
|
|
883
|
+
var MAX_KNOWLEDGE_MEDIA_ASSET_BYTES = 512 * 1024 * 1024;
|
|
884
|
+
var MAX_KNOWLEDGE_MEDIA_ASSETS_TOTAL_BYTES = 8 * 1024 * 1024 * 1024;
|
|
721
885
|
var MAX_KNOWLEDGE_ASSET_COUNT = 8192;
|
|
722
886
|
var MAX_KNOWLEDGE_ASSET_PATH_LENGTH = 512;
|
|
723
|
-
var KNOWLEDGE_ASSET_PATH_PATTERN = /^assets\/(?:[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\/){0,15}[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\.(?:png|jpe?g|gif|webp|svg)$/i;
|
|
887
|
+
var KNOWLEDGE_ASSET_PATH_PATTERN = /^assets\/(?:[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\/){0,15}[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\.(?:png|jpe?g|gif|webp|svg|mp4|webm|mp3|m4a|ogg|opus|wav|flac)$/i;
|
|
724
888
|
var KnowledgeAssetPathSchema = z7.string().max(MAX_KNOWLEDGE_ASSET_PATH_LENGTH).regex(KNOWLEDGE_ASSET_PATH_PATTERN, "not a valid asset path");
|
|
725
889
|
function isKnowledgeAssetPath(path) {
|
|
726
890
|
return path.length <= MAX_KNOWLEDGE_ASSET_PATH_LENGTH && KNOWLEDGE_ASSET_PATH_PATTERN.test(path);
|
|
@@ -729,14 +893,61 @@ function assetExtension(path) {
|
|
|
729
893
|
const dot = path.lastIndexOf(".");
|
|
730
894
|
if (dot < 0) return null;
|
|
731
895
|
const ext = path.slice(dot + 1).toLowerCase();
|
|
732
|
-
|
|
896
|
+
if (Object.hasOwn(KNOWLEDGE_ASSET_TYPES, ext)) return ext;
|
|
897
|
+
if (Object.hasOwn(KNOWLEDGE_MEDIA_ASSET_TYPES, ext)) return ext;
|
|
898
|
+
return null;
|
|
733
899
|
}
|
|
734
900
|
function assetContentType(path) {
|
|
735
901
|
const ext = assetExtension(path);
|
|
736
|
-
|
|
902
|
+
if (!ext) return null;
|
|
903
|
+
return ext in KNOWLEDGE_ASSET_TYPES ? KNOWLEDGE_ASSET_TYPES[ext] : KNOWLEDGE_MEDIA_ASSET_TYPES[ext];
|
|
904
|
+
}
|
|
905
|
+
function isKnowledgeMediaAssetPath(path) {
|
|
906
|
+
const ext = assetExtension(path);
|
|
907
|
+
return ext !== null && Object.hasOwn(KNOWLEDGE_MEDIA_ASSET_TYPES, ext);
|
|
908
|
+
}
|
|
909
|
+
function assetModality(path) {
|
|
910
|
+
const ext = assetExtension(path);
|
|
911
|
+
if (!ext) return null;
|
|
912
|
+
if (ext === "mp4" || ext === "webm") return "video";
|
|
913
|
+
return Object.hasOwn(KNOWLEDGE_MEDIA_ASSET_TYPES, ext) ? "audio" : "image";
|
|
914
|
+
}
|
|
915
|
+
function maxKnowledgeAssetBytes(path) {
|
|
916
|
+
return isKnowledgeMediaAssetPath(path) ? MAX_KNOWLEDGE_MEDIA_ASSET_BYTES : MAX_KNOWLEDGE_ASSET_BYTES;
|
|
917
|
+
}
|
|
918
|
+
function knowledgeAssetLimitsProblem(files) {
|
|
919
|
+
if (files.length > MAX_KNOWLEDGE_ASSET_COUNT) {
|
|
920
|
+
return `${files.length} assets exceed the ${MAX_KNOWLEDGE_ASSET_COUNT}-file limit`;
|
|
921
|
+
}
|
|
922
|
+
let imageBytes = 0;
|
|
923
|
+
let mediaBytes = 0;
|
|
924
|
+
for (const file of files) {
|
|
925
|
+
if (file.sizeBytes > maxKnowledgeAssetBytes(file.path)) {
|
|
926
|
+
return `${file.path} is ${file.sizeBytes} bytes, over its ${maxKnowledgeAssetBytes(file.path)}-byte limit`;
|
|
927
|
+
}
|
|
928
|
+
if (isKnowledgeMediaAssetPath(file.path)) mediaBytes += file.sizeBytes;
|
|
929
|
+
else imageBytes += file.sizeBytes;
|
|
930
|
+
}
|
|
931
|
+
if (imageBytes > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
|
|
932
|
+
return `image assets total ${imageBytes} bytes, over the ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES}-byte limit`;
|
|
933
|
+
}
|
|
934
|
+
if (mediaBytes > MAX_KNOWLEDGE_MEDIA_ASSETS_TOTAL_BYTES) {
|
|
935
|
+
return `audio and video assets total ${mediaBytes} bytes, over the ${MAX_KNOWLEDGE_MEDIA_ASSETS_TOTAL_BYTES}-byte limit`;
|
|
936
|
+
}
|
|
937
|
+
return null;
|
|
737
938
|
}
|
|
738
939
|
function assetKindForExtension(ext) {
|
|
739
|
-
|
|
940
|
+
switch (ext) {
|
|
941
|
+
case "jpg":
|
|
942
|
+
return "jpeg";
|
|
943
|
+
case "mp4":
|
|
944
|
+
case "m4a":
|
|
945
|
+
return "isobmff";
|
|
946
|
+
case "opus":
|
|
947
|
+
return "ogg";
|
|
948
|
+
default:
|
|
949
|
+
return ext;
|
|
950
|
+
}
|
|
740
951
|
}
|
|
741
952
|
var PNG_MAGIC = [137, 80, 78, 71, 13, 10, 26, 10];
|
|
742
953
|
var JPEG_MAGIC = [255, 216, 255];
|
|
@@ -744,6 +955,12 @@ var GIF87 = [71, 73, 70, 56, 55, 97];
|
|
|
744
955
|
var GIF89 = [71, 73, 70, 56, 57, 97];
|
|
745
956
|
var RIFF = [82, 73, 70, 70];
|
|
746
957
|
var WEBP = [87, 69, 66, 80];
|
|
958
|
+
var WAVE = [87, 65, 86, 69];
|
|
959
|
+
var FTYP = [102, 116, 121, 112];
|
|
960
|
+
var EBML = [26, 69, 223, 163];
|
|
961
|
+
var OGGS = [79, 103, 103, 83];
|
|
962
|
+
var FLAC = [102, 76, 97, 67];
|
|
963
|
+
var ID3 = [73, 68, 51];
|
|
747
964
|
function startsWith(bytes, magic, offset = 0) {
|
|
748
965
|
if (bytes.length < offset + magic.length) return false;
|
|
749
966
|
for (let i = 0; i < magic.length; i++) {
|
|
@@ -756,8 +973,20 @@ function sniffAssetType(bytes) {
|
|
|
756
973
|
if (startsWith(bytes, JPEG_MAGIC)) return "jpeg";
|
|
757
974
|
if (startsWith(bytes, GIF87) || startsWith(bytes, GIF89)) return "gif";
|
|
758
975
|
if (startsWith(bytes, RIFF) && startsWith(bytes, WEBP, 8)) return "webp";
|
|
976
|
+
if (startsWith(bytes, RIFF) && startsWith(bytes, WAVE, 8)) return "wav";
|
|
977
|
+
if (startsWith(bytes, FTYP, 4)) return "isobmff";
|
|
978
|
+
if (startsWith(bytes, EBML)) return "webm";
|
|
979
|
+
if (startsWith(bytes, OGGS)) return "ogg";
|
|
980
|
+
if (startsWith(bytes, FLAC)) return "flac";
|
|
981
|
+
if (startsWith(bytes, ID3) || isMpegAudioFrame(bytes)) return "mp3";
|
|
759
982
|
return looksLikeSvg(decodeText(bytes)) ? "svg" : null;
|
|
760
983
|
}
|
|
984
|
+
function isMpegAudioFrame(bytes) {
|
|
985
|
+
if (bytes.length < 2) return false;
|
|
986
|
+
const b0 = bytes[0];
|
|
987
|
+
const b1 = bytes[1];
|
|
988
|
+
return b0 === 255 && (b1 & 224) === 224 && (b1 & 6) !== 0;
|
|
989
|
+
}
|
|
761
990
|
function decodeText(bytes) {
|
|
762
991
|
try {
|
|
763
992
|
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes);
|
|
@@ -852,6 +1081,16 @@ function topicSortKeyForOrder(order) {
|
|
|
852
1081
|
}
|
|
853
1082
|
|
|
854
1083
|
// src/format/quantize.ts
|
|
1084
|
+
function profileUnitVector(profile, raw) {
|
|
1085
|
+
const source = embeddingProfileSourceDimensions(profile);
|
|
1086
|
+
if (raw.length !== source) {
|
|
1087
|
+
throw new Error(
|
|
1088
|
+
`embedding has ${raw.length} dimensions, profile ${profile.id} expects ${source}`
|
|
1089
|
+
);
|
|
1090
|
+
}
|
|
1091
|
+
if (source === profile.dimensions) return l2Normalize(raw);
|
|
1092
|
+
return l2Normalize(Array.prototype.slice.call(raw, 0, profile.dimensions));
|
|
1093
|
+
}
|
|
855
1094
|
function quantizeInt8(unitVector) {
|
|
856
1095
|
const out = new Int8Array(unitVector.length);
|
|
857
1096
|
for (let i = 0; i < unitVector.length; i++) {
|
|
@@ -883,9 +1122,13 @@ function quantizeBinary(unitVector) {
|
|
|
883
1122
|
return out;
|
|
884
1123
|
}
|
|
885
1124
|
function rerankScore(query, passageInt8) {
|
|
1125
|
+
if (query.length !== passageInt8.length) {
|
|
1126
|
+
throw new Error(`query has ${query.length} dimensions, passage has ${passageInt8.length}`);
|
|
1127
|
+
}
|
|
886
1128
|
let dot = 0;
|
|
887
|
-
|
|
888
|
-
|
|
1129
|
+
for (let i = 0; i < query.length; i++) {
|
|
1130
|
+
dot += query[i] * (passageInt8[i] / 127);
|
|
1131
|
+
}
|
|
889
1132
|
return dot;
|
|
890
1133
|
}
|
|
891
1134
|
function l2Normalize(vector) {
|
|
@@ -951,6 +1194,178 @@ function documentSlug(title) {
|
|
|
951
1194
|
const slug = title.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
952
1195
|
return slug || "untitled";
|
|
953
1196
|
}
|
|
1197
|
+
|
|
1198
|
+
// src/query-stopwords.ts
|
|
1199
|
+
var QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
|
|
1200
|
+
// articles + determiners
|
|
1201
|
+
"a",
|
|
1202
|
+
"an",
|
|
1203
|
+
"the",
|
|
1204
|
+
"this",
|
|
1205
|
+
"that",
|
|
1206
|
+
"these",
|
|
1207
|
+
"those",
|
|
1208
|
+
"some",
|
|
1209
|
+
"any",
|
|
1210
|
+
"all",
|
|
1211
|
+
"each",
|
|
1212
|
+
"both",
|
|
1213
|
+
"every",
|
|
1214
|
+
"other",
|
|
1215
|
+
"another",
|
|
1216
|
+
"same",
|
|
1217
|
+
"such",
|
|
1218
|
+
// pronouns
|
|
1219
|
+
"i",
|
|
1220
|
+
"me",
|
|
1221
|
+
"my",
|
|
1222
|
+
"we",
|
|
1223
|
+
"us",
|
|
1224
|
+
"our",
|
|
1225
|
+
"you",
|
|
1226
|
+
"your",
|
|
1227
|
+
"it",
|
|
1228
|
+
"its",
|
|
1229
|
+
"he",
|
|
1230
|
+
"him",
|
|
1231
|
+
"his",
|
|
1232
|
+
"she",
|
|
1233
|
+
"her",
|
|
1234
|
+
"they",
|
|
1235
|
+
"them",
|
|
1236
|
+
"their",
|
|
1237
|
+
// be / have / do
|
|
1238
|
+
"am",
|
|
1239
|
+
"is",
|
|
1240
|
+
"are",
|
|
1241
|
+
"was",
|
|
1242
|
+
"were",
|
|
1243
|
+
"be",
|
|
1244
|
+
"been",
|
|
1245
|
+
"being",
|
|
1246
|
+
"have",
|
|
1247
|
+
"has",
|
|
1248
|
+
"had",
|
|
1249
|
+
"do",
|
|
1250
|
+
"does",
|
|
1251
|
+
"did",
|
|
1252
|
+
"done",
|
|
1253
|
+
// modals
|
|
1254
|
+
"can",
|
|
1255
|
+
"could",
|
|
1256
|
+
"will",
|
|
1257
|
+
"would",
|
|
1258
|
+
"shall",
|
|
1259
|
+
"should",
|
|
1260
|
+
"may",
|
|
1261
|
+
"might",
|
|
1262
|
+
"must",
|
|
1263
|
+
// question words
|
|
1264
|
+
"how",
|
|
1265
|
+
"what",
|
|
1266
|
+
"when",
|
|
1267
|
+
"where",
|
|
1268
|
+
"which",
|
|
1269
|
+
"who",
|
|
1270
|
+
"whom",
|
|
1271
|
+
"why",
|
|
1272
|
+
// prepositions + conjunctions
|
|
1273
|
+
"about",
|
|
1274
|
+
"and",
|
|
1275
|
+
"as",
|
|
1276
|
+
"at",
|
|
1277
|
+
"but",
|
|
1278
|
+
"by",
|
|
1279
|
+
"for",
|
|
1280
|
+
"from",
|
|
1281
|
+
"if",
|
|
1282
|
+
"in",
|
|
1283
|
+
"into",
|
|
1284
|
+
"of",
|
|
1285
|
+
"on",
|
|
1286
|
+
"or",
|
|
1287
|
+
"so",
|
|
1288
|
+
"than",
|
|
1289
|
+
"then",
|
|
1290
|
+
"there",
|
|
1291
|
+
"to",
|
|
1292
|
+
"with",
|
|
1293
|
+
// request scaffolding + degree words
|
|
1294
|
+
"please",
|
|
1295
|
+
"help",
|
|
1296
|
+
"let",
|
|
1297
|
+
"tell",
|
|
1298
|
+
"want",
|
|
1299
|
+
"need",
|
|
1300
|
+
"create",
|
|
1301
|
+
"make",
|
|
1302
|
+
"give",
|
|
1303
|
+
"get",
|
|
1304
|
+
"got",
|
|
1305
|
+
"use",
|
|
1306
|
+
"using",
|
|
1307
|
+
"just",
|
|
1308
|
+
"like",
|
|
1309
|
+
"also",
|
|
1310
|
+
"very",
|
|
1311
|
+
"only",
|
|
1312
|
+
"new",
|
|
1313
|
+
"more",
|
|
1314
|
+
"most",
|
|
1315
|
+
"much",
|
|
1316
|
+
"many",
|
|
1317
|
+
"few",
|
|
1318
|
+
"own",
|
|
1319
|
+
// conversational scaffolding
|
|
1320
|
+
"hey",
|
|
1321
|
+
"hi",
|
|
1322
|
+
"hiya",
|
|
1323
|
+
"hello",
|
|
1324
|
+
"thanks",
|
|
1325
|
+
"thank",
|
|
1326
|
+
"okay",
|
|
1327
|
+
"ok",
|
|
1328
|
+
"yep",
|
|
1329
|
+
"yeah",
|
|
1330
|
+
"yes",
|
|
1331
|
+
"nope",
|
|
1332
|
+
"good",
|
|
1333
|
+
"great",
|
|
1334
|
+
"fine",
|
|
1335
|
+
"nice",
|
|
1336
|
+
"cool",
|
|
1337
|
+
"ready",
|
|
1338
|
+
"going",
|
|
1339
|
+
"doing",
|
|
1340
|
+
"sounds",
|
|
1341
|
+
"morning",
|
|
1342
|
+
"afternoon",
|
|
1343
|
+
"evening",
|
|
1344
|
+
"night",
|
|
1345
|
+
// Apostrophes are token boundaries, so contractions otherwise leave these
|
|
1346
|
+
// fragments looking like rare, highly distinctive query terms.
|
|
1347
|
+
"s",
|
|
1348
|
+
"t",
|
|
1349
|
+
"re",
|
|
1350
|
+
"ve",
|
|
1351
|
+
"ll",
|
|
1352
|
+
"d",
|
|
1353
|
+
"m",
|
|
1354
|
+
"isn",
|
|
1355
|
+
"aren",
|
|
1356
|
+
"wasn",
|
|
1357
|
+
"weren",
|
|
1358
|
+
"don",
|
|
1359
|
+
"doesn",
|
|
1360
|
+
"didn",
|
|
1361
|
+
"hasn",
|
|
1362
|
+
"haven",
|
|
1363
|
+
"hadn",
|
|
1364
|
+
"won",
|
|
1365
|
+
"wouldn",
|
|
1366
|
+
"shouldn",
|
|
1367
|
+
"couldn"
|
|
1368
|
+
]);
|
|
954
1369
|
export {
|
|
955
1370
|
ASSETS_PREFIX,
|
|
956
1371
|
ArtifactDigestSchema,
|
|
@@ -958,6 +1373,8 @@ export {
|
|
|
958
1373
|
CatalogDocumentSchema,
|
|
959
1374
|
DEFAULT_EMBEDDING_ONNX_FILE,
|
|
960
1375
|
DEFAULT_EMBEDDING_TOKENIZER_FILE,
|
|
1376
|
+
EmbeddingMediaSchema,
|
|
1377
|
+
EmbeddingModelFileSchema,
|
|
961
1378
|
GEZK_APPLICATION_ID,
|
|
962
1379
|
GEZK_EARTH_RADIUS_METERS,
|
|
963
1380
|
GEZK_FORMAT_GENERATIONS,
|
|
@@ -972,6 +1389,7 @@ export {
|
|
|
972
1389
|
KNOWLEDGE_ASSET_TYPES,
|
|
973
1390
|
KNOWLEDGE_ID_PATTERN,
|
|
974
1391
|
KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS,
|
|
1392
|
+
KNOWLEDGE_MEDIA_ASSET_TYPES,
|
|
975
1393
|
KNOWLEDGE_VERSION_PATTERN,
|
|
976
1394
|
KnowledgeAssetPathSchema,
|
|
977
1395
|
KnowledgeBoundsSchema,
|
|
@@ -1003,13 +1421,18 @@ export {
|
|
|
1003
1421
|
MAX_KNOWLEDGE_ASSET_PATH_LENGTH,
|
|
1004
1422
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
1005
1423
|
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
1424
|
+
MAX_KNOWLEDGE_MEDIA_ASSETS_TOTAL_BYTES,
|
|
1425
|
+
MAX_KNOWLEDGE_MEDIA_ASSET_BYTES,
|
|
1006
1426
|
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
1007
1427
|
MIMETYPE_PATH,
|
|
1428
|
+
QUERY_STOP_WORDS,
|
|
1008
1429
|
README_PATH,
|
|
1009
1430
|
ROUTER_DB_PATH,
|
|
1010
1431
|
ROUTER_DDL,
|
|
1011
1432
|
RepoRelativePathSchema,
|
|
1012
1433
|
SHARD_DDL,
|
|
1434
|
+
SHARD_DDL_V4,
|
|
1435
|
+
SHARD_DDL_V5,
|
|
1013
1436
|
SOURCE_NOTICES_PATH,
|
|
1014
1437
|
Sha256HexSchema,
|
|
1015
1438
|
SourceNoticesSchema,
|
|
@@ -1017,27 +1440,37 @@ export {
|
|
|
1017
1440
|
assetContentType,
|
|
1018
1441
|
assetExtension,
|
|
1019
1442
|
assetKindForExtension,
|
|
1443
|
+
assetModality,
|
|
1020
1444
|
canonicalizeJson,
|
|
1021
1445
|
centerVector,
|
|
1022
1446
|
documentSlug,
|
|
1023
1447
|
embeddingProfileArtifacts,
|
|
1024
1448
|
embeddingProfileCenter,
|
|
1025
1449
|
embeddingProfileCenterProblem,
|
|
1450
|
+
embeddingProfileMinimumFormat,
|
|
1451
|
+
embeddingProfileSourceDimensions,
|
|
1452
|
+
embeddingProfileTruncationProblem,
|
|
1453
|
+
formatAtLeast,
|
|
1026
1454
|
formatKnowledgeUri,
|
|
1027
1455
|
isKnowledgeAssetPath,
|
|
1456
|
+
isKnowledgeMediaAssetPath,
|
|
1028
1457
|
isSupportedFormatVersion,
|
|
1029
1458
|
isSupportedIndexSchemaVersion,
|
|
1459
|
+
knowledgeAssetLimitsProblem,
|
|
1030
1460
|
l2Normalize,
|
|
1031
1461
|
locationDistanceMeters,
|
|
1032
1462
|
locationInBounds,
|
|
1463
|
+
maxKnowledgeAssetBytes,
|
|
1033
1464
|
normalizeLongitude,
|
|
1034
1465
|
parseKnowledgeUri,
|
|
1466
|
+
profileUnitVector,
|
|
1035
1467
|
quantizeBinary,
|
|
1036
1468
|
quantizeBinaryForProfile,
|
|
1037
1469
|
quantizeInt8,
|
|
1038
1470
|
radiusBounds,
|
|
1039
1471
|
rerankScore,
|
|
1040
1472
|
sameVectorSpace,
|
|
1473
|
+
shardDdlFor,
|
|
1041
1474
|
sniffAssetType,
|
|
1042
1475
|
spatialManifest,
|
|
1043
1476
|
svgInertnessProblem,
|