@bendyline/gezk 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +223 -24
- package/dist/index.js +325 -16
- package/dist/node.d.ts +1 -1
- package/dist/node.js +324 -15
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -48,6 +48,63 @@ declare const DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
|
|
|
48
48
|
* name or a dimension is unsafe — two 384-dim models do not share a space,
|
|
49
49
|
* and an instruction-prefix change silently changes vectors.
|
|
50
50
|
*/
|
|
51
|
+
declare const KnowledgeEmbeddingProfileObject: z.ZodObject<{
|
|
52
|
+
id: z.ZodString;
|
|
53
|
+
model: z.ZodObject<{
|
|
54
|
+
repo: z.ZodString;
|
|
55
|
+
revision: z.ZodString;
|
|
56
|
+
onnxFile: z.ZodOptional<z.ZodString>;
|
|
57
|
+
onnxDigest: z.ZodOptional<z.ZodString>;
|
|
58
|
+
}, z.core.$strip>;
|
|
59
|
+
tokenizer: z.ZodObject<{
|
|
60
|
+
kind: z.ZodString;
|
|
61
|
+
file: z.ZodOptional<z.ZodString>;
|
|
62
|
+
digest: z.ZodOptional<z.ZodString>;
|
|
63
|
+
}, z.core.$strip>;
|
|
64
|
+
pooling: z.ZodEnum<{
|
|
65
|
+
mean: "mean";
|
|
66
|
+
cls: "cls";
|
|
67
|
+
last: "last";
|
|
68
|
+
}>;
|
|
69
|
+
normalized: z.ZodBoolean;
|
|
70
|
+
dimensions: z.ZodNumber;
|
|
71
|
+
maxTokens: z.ZodNumber;
|
|
72
|
+
queryInstruction: z.ZodString;
|
|
73
|
+
passageInstruction: z.ZodString;
|
|
74
|
+
vectorEncoding: z.ZodEnum<{
|
|
75
|
+
"bit+int8": "bit+int8";
|
|
76
|
+
}>;
|
|
77
|
+
distance: z.ZodObject<{
|
|
78
|
+
stage1: z.ZodLiteral<"hamming">;
|
|
79
|
+
stage2: z.ZodLiteral<"cosine">;
|
|
80
|
+
}, z.core.$strip>;
|
|
81
|
+
quantization: z.ZodObject<{
|
|
82
|
+
int8: z.ZodObject<{
|
|
83
|
+
method: z.ZodLiteral<"symmetric-linear">;
|
|
84
|
+
scale: z.ZodLiteral<127>;
|
|
85
|
+
}, z.core.$strip>;
|
|
86
|
+
binary: z.ZodObject<{
|
|
87
|
+
method: z.ZodEnum<{
|
|
88
|
+
sign: "sign";
|
|
89
|
+
"centered-sign": "centered-sign";
|
|
90
|
+
}>;
|
|
91
|
+
threshold: z.ZodLiteral<0>;
|
|
92
|
+
packing: z.ZodLiteral<"lsb-first">;
|
|
93
|
+
center: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
94
|
+
}, z.core.$strip>;
|
|
95
|
+
}, z.core.$strip>;
|
|
96
|
+
}, z.core.$strip>;
|
|
97
|
+
type KnowledgeEmbeddingProfile = z.infer<typeof KnowledgeEmbeddingProfileObject>;
|
|
98
|
+
/**
|
|
99
|
+
* The cross-field rule the object schema cannot express on its own: a
|
|
100
|
+
* `centered-sign` profile must carry a center of exactly `dimensions`
|
|
101
|
+
* entries, and a `sign` profile must not carry one. Returns the problem, or
|
|
102
|
+
* null when the profile is consistent.
|
|
103
|
+
*/
|
|
104
|
+
declare function embeddingProfileCenterProblem(profile: KnowledgeEmbeddingProfile): string | null;
|
|
105
|
+
/** The profile's binary center as a Float32Array, or null for a plain `sign` profile. */
|
|
106
|
+
declare function embeddingProfileCenter(profile: KnowledgeEmbeddingProfile): Float32Array | null;
|
|
107
|
+
/** The profile schema proper: the object shape plus the center rule above. */
|
|
51
108
|
declare const KnowledgeEmbeddingProfileSchema: z.ZodObject<{
|
|
52
109
|
id: z.ZodString;
|
|
53
110
|
model: z.ZodObject<{
|
|
@@ -84,13 +141,16 @@ declare const KnowledgeEmbeddingProfileSchema: z.ZodObject<{
|
|
|
84
141
|
scale: z.ZodLiteral<127>;
|
|
85
142
|
}, z.core.$strip>;
|
|
86
143
|
binary: z.ZodObject<{
|
|
87
|
-
method: z.
|
|
144
|
+
method: z.ZodEnum<{
|
|
145
|
+
sign: "sign";
|
|
146
|
+
"centered-sign": "centered-sign";
|
|
147
|
+
}>;
|
|
88
148
|
threshold: z.ZodLiteral<0>;
|
|
89
149
|
packing: z.ZodLiteral<"lsb-first">;
|
|
150
|
+
center: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
90
151
|
}, z.core.$strip>;
|
|
91
152
|
}, z.core.$strip>;
|
|
92
153
|
}, z.core.$strip>;
|
|
93
|
-
type KnowledgeEmbeddingProfile = z.infer<typeof KnowledgeEmbeddingProfileSchema>;
|
|
94
154
|
/** The exact repo files a profile pins, with the format defaults applied. */
|
|
95
155
|
declare function embeddingProfileArtifacts(profile: KnowledgeEmbeddingProfile): {
|
|
96
156
|
onnxFile: string;
|
|
@@ -101,12 +161,19 @@ declare function embeddingProfileArtifacts(profile: KnowledgeEmbeddingProfile):
|
|
|
101
161
|
/**
|
|
102
162
|
* Whether two profiles describe one vector space, so vectors from either
|
|
103
163
|
* may be compared: same model files at the same revision, same tokenizer,
|
|
104
|
-
* pooling, normalization, dimensions, instruction prefixes and
|
|
105
|
-
* The profile id and `maxTokens` are not compared — the
|
|
106
|
-
* the second a compile-time bound. A digest counts only
|
|
107
|
-
* declare one: two pins at one revision that hash
|
|
108
|
-
* different artifacts, whatever the path says; an
|
|
109
|
-
* simply not a claim.
|
|
164
|
+
* pooling, normalization, dimensions, instruction prefixes, encoding and
|
|
165
|
+
* int8 quantization. The profile id and `maxTokens` are not compared — the
|
|
166
|
+
* first is a label, the second a compile-time bound. A digest counts only
|
|
167
|
+
* when both sides declare one: two pins at one revision that hash
|
|
168
|
+
* differently name different artifacts, whatever the path says; an
|
|
169
|
+
* undeclared digest is simply not a claim.
|
|
170
|
+
*
|
|
171
|
+
* The binary (stage-1) parameters are deliberately NOT compared. They
|
|
172
|
+
* describe how a catalog derived its own pre-filter bits, and a reader
|
|
173
|
+
* applies them from the catalog's profile echo; they never change what a
|
|
174
|
+
* query vector is. A daemon that shares a model with a `sign` profile
|
|
175
|
+
* therefore also shares it with the `centered-sign` revision of the same
|
|
176
|
+
* model — same floats in, same int8 rerank, only the bit scan differs.
|
|
110
177
|
*/
|
|
111
178
|
declare function sameVectorSpace(a: KnowledgeEmbeddingProfile, b: KnowledgeEmbeddingProfile): boolean;
|
|
112
179
|
/**
|
|
@@ -154,8 +221,11 @@ declare const KnowledgeSignatureSchema: z.ZodObject<{
|
|
|
154
221
|
type KnowledgeSignature = z.infer<typeof KnowledgeSignatureSchema>;
|
|
155
222
|
declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
156
223
|
kind: z.ZodLiteral<"gezk-catalog">;
|
|
157
|
-
formatVersion: z.
|
|
158
|
-
|
|
224
|
+
formatVersion: z.ZodEnum<{
|
|
225
|
+
0.6: "0.6";
|
|
226
|
+
0.5: "0.5";
|
|
227
|
+
}>;
|
|
228
|
+
indexSchemaVersion: z.ZodUnion<readonly [z.ZodLiteral<2>, z.ZodLiteral<3>]>;
|
|
159
229
|
id: z.ZodString;
|
|
160
230
|
version: z.ZodString;
|
|
161
231
|
name: z.ZodString;
|
|
@@ -214,9 +284,13 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
|
214
284
|
scale: z.ZodLiteral<127>;
|
|
215
285
|
}, z.core.$strip>;
|
|
216
286
|
binary: z.ZodObject<{
|
|
217
|
-
method: z.
|
|
287
|
+
method: z.ZodEnum<{
|
|
288
|
+
sign: "sign";
|
|
289
|
+
"centered-sign": "centered-sign";
|
|
290
|
+
}>;
|
|
218
291
|
threshold: z.ZodLiteral<0>;
|
|
219
292
|
packing: z.ZodLiteral<"lsb-first">;
|
|
293
|
+
center: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
220
294
|
}, z.core.$strip>;
|
|
221
295
|
}, z.core.$strip>;
|
|
222
296
|
}, z.core.$strip>;
|
|
@@ -239,6 +313,9 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
|
239
313
|
topics: z.ZodArray<z.ZodObject<{
|
|
240
314
|
id: z.ZodString;
|
|
241
315
|
name: z.ZodString;
|
|
316
|
+
parentId: z.ZodOptional<z.ZodString>;
|
|
317
|
+
sortKey: z.ZodOptional<z.ZodString>;
|
|
318
|
+
description: z.ZodOptional<z.ZodString>;
|
|
242
319
|
}, z.core.$strip>>;
|
|
243
320
|
router: z.ZodObject<{
|
|
244
321
|
shardTargetChunks: z.ZodNumber;
|
|
@@ -256,6 +333,7 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
|
256
333
|
documents: z.ZodNumber;
|
|
257
334
|
chunks: z.ZodNumber;
|
|
258
335
|
shards: z.ZodNumber;
|
|
336
|
+
assets: z.ZodOptional<z.ZodNumber>;
|
|
259
337
|
}, z.core.$strip>;
|
|
260
338
|
files: z.ZodArray<z.ZodObject<{
|
|
261
339
|
path: z.ZodString;
|
|
@@ -263,7 +341,10 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
|
263
341
|
sha256: z.ZodString;
|
|
264
342
|
}, z.core.$strip>>;
|
|
265
343
|
requires: z.ZodObject<{
|
|
266
|
-
formatVersion: z.
|
|
344
|
+
formatVersion: z.ZodEnum<{
|
|
345
|
+
0.6: "0.6";
|
|
346
|
+
0.5: "0.5";
|
|
347
|
+
}>;
|
|
267
348
|
features: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
268
349
|
}, z.core.$strip>;
|
|
269
350
|
smokeQueries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
@@ -287,6 +368,8 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
|
287
368
|
}, z.core.$strip>>;
|
|
288
369
|
}, z.core.$strip>;
|
|
289
370
|
type KnowledgeCatalogManifest = z.infer<typeof KnowledgeCatalogManifestSchema>;
|
|
371
|
+
/** The (formatVersion, indexSchemaVersion) pairs the manifest schema admits. */
|
|
372
|
+
declare const KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS: [GezkIndexSchemaVersion, ...GezkIndexSchemaVersion[]];
|
|
290
373
|
/** `LICENSES/source-notices.json` — per-source attribution for combined catalogs. */
|
|
291
374
|
declare const SourceNoticesSchema: z.ZodObject<{
|
|
292
375
|
sources: z.ZodArray<z.ZodObject<{
|
|
@@ -300,6 +383,16 @@ declare const SourceNoticesSchema: z.ZodObject<{
|
|
|
300
383
|
}, z.core.$strip>;
|
|
301
384
|
type SourceNotices = z.infer<typeof SourceNoticesSchema>;
|
|
302
385
|
|
|
386
|
+
/** A signed 32-bit listing position; unordered documents sort after ordered ones. */
|
|
387
|
+
declare const KnowledgeOrdinalSchema: z.ZodNumber;
|
|
388
|
+
/**
|
|
389
|
+
* Producer-defined document metadata, opaque to readers. Stored canonically
|
|
390
|
+
* (RFC 8785) in `documents.meta_json`, bounded by
|
|
391
|
+
* `MAX_KNOWLEDGE_DOCUMENT_META_BYTES`. Keys beginning with `gezk.` are
|
|
392
|
+
* reserved for future versions of the format.
|
|
393
|
+
*/
|
|
394
|
+
declare const KnowledgeDocumentMetaSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
395
|
+
type KnowledgeDocumentMeta = z.infer<typeof KnowledgeDocumentMetaSchema>;
|
|
303
396
|
/** One normalized document streamed into the compiler. */
|
|
304
397
|
declare const CatalogDocumentSchema: z.ZodObject<{
|
|
305
398
|
id: z.ZodString;
|
|
@@ -314,6 +407,8 @@ declare const CatalogDocumentSchema: z.ZodObject<{
|
|
|
314
407
|
sourceUpdatedAt: z.ZodOptional<z.ZodString>;
|
|
315
408
|
attribution: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
316
409
|
aliases: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
410
|
+
ordinal: z.ZodOptional<z.ZodNumber>;
|
|
411
|
+
meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
317
412
|
}, z.core.$strip>;
|
|
318
413
|
type CatalogDocument = z.infer<typeof CatalogDocumentSchema>;
|
|
319
414
|
|
|
@@ -349,7 +444,10 @@ type KnowledgeRegistryEntry = z.infer<typeof KnowledgeRegistryEntrySchema>;
|
|
|
349
444
|
*/
|
|
350
445
|
declare const KnowledgeRegistryIndexSchema: z.ZodObject<{
|
|
351
446
|
kind: z.ZodLiteral<"gezk-registry">;
|
|
352
|
-
formatVersion: z.
|
|
447
|
+
formatVersion: z.ZodEnum<{
|
|
448
|
+
0.6: "0.6";
|
|
449
|
+
0.5: "0.5";
|
|
450
|
+
}>;
|
|
353
451
|
publisher: z.ZodObject<{
|
|
354
452
|
id: z.ZodString;
|
|
355
453
|
name: z.ZodString;
|
|
@@ -407,12 +505,29 @@ declare function parseKnowledgeUri(raw: string): KnowledgeUri | null;
|
|
|
407
505
|
|
|
408
506
|
/** Container-level constants of the `.gezk` format. */
|
|
409
507
|
/**
|
|
410
|
-
* Public format version. `0.x` is preliminary:
|
|
411
|
-
* any minor until 1.0, and a reader supports
|
|
508
|
+
* Public format version this implementation writes. `0.x` is preliminary:
|
|
509
|
+
* breaking changes may land in any minor until 1.0, and a reader supports
|
|
510
|
+
* exactly the versions it names (see {@link GEZK_FORMAT_GENERATIONS}).
|
|
412
511
|
*/
|
|
413
|
-
declare const GEZK_FORMAT_VERSION = "0.
|
|
414
|
-
/** SQLite schema generation (`PRAGMA user_version`
|
|
415
|
-
declare const GEZK_INDEX_SCHEMA_VERSION =
|
|
512
|
+
declare const GEZK_FORMAT_VERSION = "0.6";
|
|
513
|
+
/** SQLite schema generation this implementation writes (`PRAGMA user_version`). */
|
|
514
|
+
declare const GEZK_INDEX_SCHEMA_VERSION = 3;
|
|
515
|
+
/**
|
|
516
|
+
* Every (formatVersion, indexSchemaVersion) pairing this implementation
|
|
517
|
+
* reads. The writer always emits the current pair; older generations stay
|
|
518
|
+
* readable so a catalog published under an earlier 0.x keeps opening after
|
|
519
|
+
* a reader upgrade. A manifest that pairs the two differently is corrupt.
|
|
520
|
+
*/
|
|
521
|
+
declare const GEZK_FORMAT_GENERATIONS: {
|
|
522
|
+
readonly '0.5': 2;
|
|
523
|
+
readonly '0.6': 3;
|
|
524
|
+
};
|
|
525
|
+
type GezkFormatVersion = keyof typeof GEZK_FORMAT_GENERATIONS;
|
|
526
|
+
type GezkIndexSchemaVersion = (typeof GEZK_FORMAT_GENERATIONS)[GezkFormatVersion];
|
|
527
|
+
declare const GEZK_SUPPORTED_FORMAT_VERSIONS: [GezkFormatVersion, ...GezkFormatVersion[]];
|
|
528
|
+
declare const GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS: [GezkIndexSchemaVersion, ...GezkIndexSchemaVersion[]];
|
|
529
|
+
declare function isSupportedFormatVersion(value: unknown): value is GezkFormatVersion;
|
|
530
|
+
declare function isSupportedIndexSchemaVersion(value: unknown): value is GezkIndexSchemaVersion;
|
|
416
531
|
/** `PRAGMA application_id` for every database in a catalog: 'GEZK'. */
|
|
417
532
|
declare const GEZK_APPLICATION_ID = 1195727435;
|
|
418
533
|
/**
|
|
@@ -431,35 +546,119 @@ declare const ROUTER_DB_PATH = "index/router.db";
|
|
|
431
546
|
declare const BODY_CODEC_MIN_BYTES = 512;
|
|
432
547
|
/** Maximum decompressed Markdown body returned from one catalog document. */
|
|
433
548
|
declare const MAX_KNOWLEDGE_DOCUMENT_BYTES: number;
|
|
549
|
+
/** Maximum canonical UTF-8 size of one document's `meta_json` object. */
|
|
550
|
+
declare const MAX_KNOWLEDGE_DOCUMENT_META_BYTES: number;
|
|
551
|
+
/** Maximum depth of the topic forest (root = 1). */
|
|
552
|
+
declare const MAX_KNOWLEDGE_TOPIC_DEPTH = 16;
|
|
434
553
|
/** Fixed ZIP entry timestamp for deterministic archives. */
|
|
435
554
|
declare const ZIP_FIXED_MTIME: Date;
|
|
436
555
|
|
|
437
556
|
/**
|
|
438
|
-
* Index schema
|
|
557
|
+
* Index schema 3 DDL. Readers never migrate these. Every table is plain
|
|
439
558
|
* SQLite (FTS5 is the only virtual-table module used), so any SQLite client
|
|
440
559
|
* can read a catalog without extensions.
|
|
441
560
|
*/
|
|
442
|
-
declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
|
|
561
|
+
declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n ordinal INTEGER,\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n meta_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, ordinal, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
|
|
443
562
|
/**
|
|
444
563
|
* Per-shard tables. Vectors are plain BLOB columns keyed by chunk id:
|
|
445
|
-
* `chunk_vectors_bit.v` holds ceil(dim/8) bytes of sign bits (LSB-first
|
|
446
|
-
* `
|
|
447
|
-
*
|
|
564
|
+
* `chunk_vectors_bit.v` holds ceil(dim/8) bytes of sign bits (LSB-first; of
|
|
565
|
+
* the unit vector for a `sign` profile, of `vector − center` for a
|
|
566
|
+
* `centered-sign` profile — the profile echo in the router's meta says
|
|
567
|
+
* which) and `chunk_vectors_int8.v` holds dim signed bytes
|
|
568
|
+
* (symmetric-linear, scale 127, never centered). Two tables, not one, so a stage-1 scan pages in 48 bytes per row
|
|
448
569
|
* rather than the whole record. Rowid alignment invariant:
|
|
449
570
|
* chunks.id == chunk_vectors_bit.chunk_id == chunk_vectors_int8.chunk_id ==
|
|
450
571
|
* fts_chunks.rowid, dense from 1.
|
|
451
572
|
*/
|
|
452
573
|
declare const SHARD_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE chunks(\n id INTEGER PRIMARY KEY,\n chunk_uid TEXT NOT NULL,\n document_id TEXT NOT NULL,\n ordinal INTEGER NOT NULL,\n title TEXT NOT NULL,\n heading_path TEXT NOT NULL,\n heading_text TEXT NOT NULL,\n line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,\n token_count INTEGER NOT NULL,\n content_hash TEXT NOT NULL,\n text TEXT NOT NULL\n);\nCREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);\nCREATE INDEX chunks_document ON chunks(document_id, ordinal);\n\nCREATE VIRTUAL TABLE fts_chunks USING fts5(\n title, heading_text, text,\n content = 'chunks', content_rowid = 'id',\n tokenize = 'unicode61 remove_diacritics 2'\n);\n\nCREATE TABLE chunk_vectors_bit(\n chunk_id INTEGER PRIMARY KEY,\n v BLOB NOT NULL\n);\n\nCREATE TABLE chunk_vectors_int8(\n chunk_id INTEGER PRIMARY KEY,\n v BLOB NOT NULL\n);\n";
|
|
453
574
|
|
|
575
|
+
/**
|
|
576
|
+
* Assets: images a catalog ships under `assets/` for its document bodies.
|
|
577
|
+
* A document references one by its archive path (``),
|
|
578
|
+
* and a reader resolves that against the catalog, never the network. The
|
|
579
|
+
* rules here are the format's: which files may be assets, how large, and
|
|
580
|
+
* the inertness an SVG must prove before a reader serves it — an SVG is a
|
|
581
|
+
* document, not a bitmap, so the format refuses anything that could run.
|
|
582
|
+
*/
|
|
583
|
+
|
|
584
|
+
declare const ASSETS_PREFIX = "assets/";
|
|
585
|
+
declare const KNOWLEDGE_ASSET_TYPES: {
|
|
586
|
+
readonly png: "image/png";
|
|
587
|
+
readonly jpg: "image/jpeg";
|
|
588
|
+
readonly jpeg: "image/jpeg";
|
|
589
|
+
readonly gif: "image/gif";
|
|
590
|
+
readonly webp: "image/webp";
|
|
591
|
+
readonly svg: "image/svg+xml";
|
|
592
|
+
};
|
|
593
|
+
type KnowledgeAssetExtension = keyof typeof KNOWLEDGE_ASSET_TYPES;
|
|
594
|
+
type KnowledgeAssetKind = 'png' | 'jpeg' | 'gif' | 'webp' | 'svg';
|
|
595
|
+
declare const MAX_KNOWLEDGE_ASSET_BYTES: number;
|
|
596
|
+
declare const MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES: number;
|
|
597
|
+
declare const MAX_KNOWLEDGE_ASSET_COUNT = 8192;
|
|
598
|
+
declare const MAX_KNOWLEDGE_ASSET_PATH_LENGTH = 512;
|
|
599
|
+
/**
|
|
600
|
+
* `assets/(<dir>/)*<name>.<ext>` — up to 15 directory segments, each
|
|
601
|
+
* segment `[A-Za-z0-9._-]` and never starting with a dot (so `.`, `..` and
|
|
602
|
+
* hidden files are impossible by construction), extension from the table.
|
|
603
|
+
*/
|
|
604
|
+
declare const KNOWLEDGE_ASSET_PATH_PATTERN: RegExp;
|
|
605
|
+
declare const KnowledgeAssetPathSchema: z.ZodString;
|
|
606
|
+
declare function isKnowledgeAssetPath(path: string): boolean;
|
|
607
|
+
declare function assetExtension(path: string): KnowledgeAssetExtension | null;
|
|
608
|
+
/** The media type an asset path implies, from its extension. */
|
|
609
|
+
declare function assetContentType(path: string): string | null;
|
|
610
|
+
/** The kind an extension declares, normalized (`jpg` and `jpeg` are one kind). */
|
|
611
|
+
declare function assetKindForExtension(ext: KnowledgeAssetExtension): KnowledgeAssetKind;
|
|
612
|
+
/**
|
|
613
|
+
* What the leading bytes say the file is. Rasters are recognized by their
|
|
614
|
+
* magic numbers; an SVG is UTF-8 text whose first element, after any BOM,
|
|
615
|
+
* whitespace, XML declaration, comments and DOCTYPE, is `<svg`.
|
|
616
|
+
*/
|
|
617
|
+
declare function sniffAssetType(bytes: Uint8Array): KnowledgeAssetKind | null;
|
|
618
|
+
/**
|
|
619
|
+
* Why an SVG is not inert, or `null` when it is. Scripts, event handlers,
|
|
620
|
+
* foreign content, entity declarations and any reference that leaves the
|
|
621
|
+
* file (other than embedded raster data) all disqualify it: a reader may
|
|
622
|
+
* hand the bytes to an image element, but it may also let a person open
|
|
623
|
+
* them directly, and an SVG that can run is a page, not a picture.
|
|
624
|
+
*/
|
|
625
|
+
declare function svgInertnessProblem(bytes: Uint8Array): string | null;
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* Topic ordering keys. `topics.sort_key` is a TEXT column compared bytewise
|
|
629
|
+
* by every reader (`ORDER BY sort_key, id`), so a producer that orders topics
|
|
630
|
+
* numerically must encode the number so that byte order equals numeric
|
|
631
|
+
* order. Offsetting by 2^31 makes every int32 non-negative and ten digits
|
|
632
|
+
* wide; zero-padding then makes the string comparison agree with the
|
|
633
|
+
* integer comparison, negatives included.
|
|
634
|
+
*/
|
|
635
|
+
declare function topicSortKeyForOrder(order: number): string;
|
|
636
|
+
|
|
454
637
|
/**
|
|
455
638
|
* Frozen quantization formulas (the gezk spec §6.2). Deliberately computed
|
|
456
639
|
* here in TypeScript rather than via sqlite-vec's `vec_quantize_*` scalar
|
|
457
640
|
* functions, so extension-internal semantics can never drift the format:
|
|
458
641
|
* both the compiler (packing) and the reader (query packing + rerank) call
|
|
459
642
|
* exactly these functions.
|
|
643
|
+
*
|
|
644
|
+
* `quantizeBinary` is the raw sign packing. What a catalog actually stores
|
|
645
|
+
* depends on the profile's `quantization.binary.method`: `sign` packs the
|
|
646
|
+
* unit vector, `centered-sign` packs `vector − center` (see the profile
|
|
647
|
+
* schema for why). `quantizeBinaryForProfile` is the one entry point that
|
|
648
|
+
* applies that rule, so the compiler and every reader derive bits the same
|
|
649
|
+
* way.
|
|
460
650
|
*/
|
|
651
|
+
|
|
461
652
|
/** int8: symmetric linear, scale 127, −128 never produced. */
|
|
462
653
|
declare function quantizeInt8(unitVector: ArrayLike<number>): Int8Array;
|
|
654
|
+
/** `vector − center`, the input to a `centered-sign` profile's bits. */
|
|
655
|
+
declare function centerVector(vector: ArrayLike<number>, center: ArrayLike<number>): Float32Array;
|
|
656
|
+
/**
|
|
657
|
+
* The sign bits a profile stores for a unit vector — and, symmetrically,
|
|
658
|
+
* the bits a hamming reader packs for a query: centered first when the
|
|
659
|
+
* profile pins a center, raw otherwise.
|
|
660
|
+
*/
|
|
661
|
+
declare function quantizeBinaryForProfile(profile: KnowledgeEmbeddingProfile, unitVector: ArrayLike<number>): Uint8Array;
|
|
463
662
|
/** binary: sign threshold 0 (exact 0.0 → 0), packed LSB-first. */
|
|
464
663
|
declare function quantizeBinary(unitVector: ArrayLike<number>): Uint8Array;
|
|
465
664
|
/**
|
|
@@ -489,4 +688,4 @@ declare function canonicalizeJson(value: unknown): string;
|
|
|
489
688
|
/** URL-ish slug for a document title (display/dedup aid, not an identity). */
|
|
490
689
|
declare function documentSlug(title: string): string;
|
|
491
690
|
|
|
492
|
-
export { ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, type CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_VERSION_PATTERN, type KnowledgeCatalogManifest, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, type KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, type KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, type KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, type KnowledgeRegistryIndex, KnowledgeRegistryIndexSchema, type KnowledgeSignature, KnowledgeSignatureSchema, type KnowledgeUri, type KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, type SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace };
|
|
691
|
+
export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, type CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, type GezkFormatVersion, type GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, type KnowledgeAssetExtension, type KnowledgeAssetKind, KnowledgeAssetPathSchema, type KnowledgeCatalogManifest, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, type KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, type KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, type KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, type KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, type KnowledgeRegistryIndex, KnowledgeRegistryIndexSchema, type KnowledgeSignature, KnowledgeSignatureSchema, type KnowledgeUri, type KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, type SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, centerVector, documentSlug, embeddingProfileArtifacts, embeddingProfileCenter, embeddingProfileCenterProblem, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder };
|