@bendyline/gezk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +43 -0
- package/dist/index.d.ts +492 -0
- package/dist/index.js +548 -0
- package/dist/node.d.ts +72 -0
- package/dist/node.js +678 -0
- package/package.json +71 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Bendyline
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# @bendyline/gezk
|
|
2
|
+
|
|
3
|
+
The `.gezk` knowledge-catalog **format**, as code: the manifest and registry
|
|
4
|
+
schemas (Zod), the id grammars, `knowledge://` references, the vector
|
|
5
|
+
quantization formulas, the SQLite DDL, RFC 8785 canonical JSON, and Ed25519
|
|
6
|
+
manifest signatures. It has no dependency on gezel — the product and the
|
|
7
|
+
`@bendyline/gezel-knowledge` toolchain (compiler, archive reader, retrieval)
|
|
8
|
+
both build on it.
|
|
9
|
+
|
|
10
|
+
```sh
|
|
11
|
+
npm install @bendyline/gezk
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
ESM only, Node 24+. `zod` is the sole runtime dependency.
|
|
15
|
+
|
|
16
|
+
- `@bendyline/gezk` — browser-safe: schemas, ids, URIs, quantization, DDL, JCS.
|
|
17
|
+
- `@bendyline/gezk/node` — everything above plus chunk ids, file digests and
|
|
18
|
+
signing, which need `node:crypto` / `node:fs`.
|
|
19
|
+
|
|
20
|
+
```ts
|
|
21
|
+
import { KnowledgeCatalogManifestSchema, parseKnowledgeUri, quantizeInt8 } from '@bendyline/gezk';
|
|
22
|
+
import { verifyManifestSignature } from '@bendyline/gezk/node';
|
|
23
|
+
|
|
24
|
+
// Citations are a parsed grammar, not string surgery.
|
|
25
|
+
parseKnowledgeUri('knowledge://bendyline/handboek/guides/intro#line=3-9');
|
|
26
|
+
// → { publisherId: 'bendyline', catalogId: 'handboek', documentId: 'guides/intro',
|
|
27
|
+
// fragment: { lineStart: 3, lineEnd: 9 } }
|
|
28
|
+
|
|
29
|
+
// A manifest is only ever trusted after it parses AND verifies.
|
|
30
|
+
const manifest = KnowledgeCatalogManifestSchema.parse(JSON.parse(rawManifestJson));
|
|
31
|
+
const verdict = verifyManifestSignature(manifest, trustAnchors);
|
|
32
|
+
if (!verdict.ok) throw new Error(`untrusted catalog: ${verdict.reason}`);
|
|
33
|
+
|
|
34
|
+
// Passage vectors are packed with these exact formulas, never a SQLite extension's.
|
|
35
|
+
const packed = quantizeInt8(unitVector);
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The format specification, JSON Schemas, conformance fixtures and the Python
|
|
39
|
+
reference reader live in [bendyline/gezk](https://github.com/bendyline/gezk).
|
|
40
|
+
|
|
41
|
+
## License
|
|
42
|
+
|
|
43
|
+
MIT © Bendyline
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,492 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
/** DNS-label style: publishers, catalogs, topics. */
|
|
4
|
+
declare const KNOWLEDGE_ID_PATTERN: RegExp;
|
|
5
|
+
declare const KnowledgeIdSchema: z.ZodString;
|
|
6
|
+
/**
|
|
7
|
+
* Portable, single-directory catalog version. Versions are identities, not
|
|
8
|
+
* paths: separators, drive/ADS syntax, controls, dot segments, trailing dots
|
|
9
|
+
* or spaces, and Windows device names are all rejected on every platform.
|
|
10
|
+
*/
|
|
11
|
+
declare const KNOWLEDGE_VERSION_PATTERN: RegExp;
|
|
12
|
+
declare const KnowledgeVersionSchema: z.ZodString;
|
|
13
|
+
/** 1–256 Unicode scalars, NFC, no controls, trimmed. Wikipedia = decimal curid. */
|
|
14
|
+
declare const KnowledgeDocumentIdSchema: z.ZodString;
|
|
15
|
+
/** Exactly 32 lowercase hex chars (first 16 bytes of the chunk-uid hash). */
|
|
16
|
+
declare const KnowledgeChunkUidSchema: z.ZodString;
|
|
17
|
+
/** Lowercase hex SHA-256. */
|
|
18
|
+
declare const Sha256HexSchema: z.ZodString;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Vector encodings a catalog may ship. `bit+int8` is the two-stage encoding:
|
|
22
|
+
* sign bits for a hamming pre-filter plus int8 for the cosine rerank. A new
|
|
23
|
+
* encoding is a new enum value; readers reject values they do not implement.
|
|
24
|
+
*/
|
|
25
|
+
declare const KnowledgeVectorEncodingSchema: z.ZodEnum<{
|
|
26
|
+
"bit+int8": "bit+int8";
|
|
27
|
+
}>;
|
|
28
|
+
type KnowledgeVectorEncoding = z.infer<typeof KnowledgeVectorEncodingSchema>;
|
|
29
|
+
/**
|
|
30
|
+
* A model-artifact digest: `sha256:` plus 64 lowercase hex digits. The
|
|
31
|
+
* algorithm travels with the value because, unlike the manifest's `sha256`
|
|
32
|
+
* keys, the field names (`onnxDigest`, `digest`) do not name it. Hugging
|
|
33
|
+
* Face's LFS object id for a file is exactly this sha256, so a pin can be
|
|
34
|
+
* checked against the Hub's file metadata without downloading the file.
|
|
35
|
+
*/
|
|
36
|
+
declare const ArtifactDigestSchema: z.ZodString;
|
|
37
|
+
/** A path inside a model repository: forward slashes, no `..`, no leading `/`. */
|
|
38
|
+
declare const RepoRelativePathSchema: z.ZodString;
|
|
39
|
+
/** The ONNX graph a profile pins when `model.onnxFile` is absent (full precision). */
|
|
40
|
+
declare const DEFAULT_EMBEDDING_ONNX_FILE = "onnx/model.onnx";
|
|
41
|
+
/** The tokenizer definition a profile pins when `tokenizer.file` is absent. */
|
|
42
|
+
declare const DEFAULT_EMBEDDING_TOKENIZER_FILE = "tokenizer.json";
|
|
43
|
+
/**
|
|
44
|
+
* The FULL vector-space identity of a catalog's embeddings, self-describing
|
|
45
|
+
* so any reader can reproduce query vectors: the model by Hugging Face
|
|
46
|
+
* coordinates, the tokenizer, pooling, normalization, dimensions, the
|
|
47
|
+
* instruction prefixes, and the exact quantization. Matching only a model
|
|
48
|
+
* name or a dimension is unsafe — two 384-dim models do not share a space,
|
|
49
|
+
* and an instruction-prefix change silently changes vectors.
|
|
50
|
+
*/
|
|
51
|
+
declare const KnowledgeEmbeddingProfileSchema: z.ZodObject<{
|
|
52
|
+
id: z.ZodString;
|
|
53
|
+
model: z.ZodObject<{
|
|
54
|
+
repo: z.ZodString;
|
|
55
|
+
revision: z.ZodString;
|
|
56
|
+
onnxFile: z.ZodOptional<z.ZodString>;
|
|
57
|
+
onnxDigest: z.ZodOptional<z.ZodString>;
|
|
58
|
+
}, z.core.$strip>;
|
|
59
|
+
tokenizer: z.ZodObject<{
|
|
60
|
+
kind: z.ZodString;
|
|
61
|
+
file: z.ZodOptional<z.ZodString>;
|
|
62
|
+
digest: z.ZodOptional<z.ZodString>;
|
|
63
|
+
}, z.core.$strip>;
|
|
64
|
+
pooling: z.ZodEnum<{
|
|
65
|
+
mean: "mean";
|
|
66
|
+
cls: "cls";
|
|
67
|
+
last: "last";
|
|
68
|
+
}>;
|
|
69
|
+
normalized: z.ZodBoolean;
|
|
70
|
+
dimensions: z.ZodNumber;
|
|
71
|
+
maxTokens: z.ZodNumber;
|
|
72
|
+
queryInstruction: z.ZodString;
|
|
73
|
+
passageInstruction: z.ZodString;
|
|
74
|
+
vectorEncoding: z.ZodEnum<{
|
|
75
|
+
"bit+int8": "bit+int8";
|
|
76
|
+
}>;
|
|
77
|
+
distance: z.ZodObject<{
|
|
78
|
+
stage1: z.ZodLiteral<"hamming">;
|
|
79
|
+
stage2: z.ZodLiteral<"cosine">;
|
|
80
|
+
}, z.core.$strip>;
|
|
81
|
+
quantization: z.ZodObject<{
|
|
82
|
+
int8: z.ZodObject<{
|
|
83
|
+
method: z.ZodLiteral<"symmetric-linear">;
|
|
84
|
+
scale: z.ZodLiteral<127>;
|
|
85
|
+
}, z.core.$strip>;
|
|
86
|
+
binary: z.ZodObject<{
|
|
87
|
+
method: z.ZodLiteral<"sign">;
|
|
88
|
+
threshold: z.ZodLiteral<0>;
|
|
89
|
+
packing: z.ZodLiteral<"lsb-first">;
|
|
90
|
+
}, z.core.$strip>;
|
|
91
|
+
}, z.core.$strip>;
|
|
92
|
+
}, z.core.$strip>;
|
|
93
|
+
type KnowledgeEmbeddingProfile = z.infer<typeof KnowledgeEmbeddingProfileSchema>;
|
|
94
|
+
/** The exact repo files a profile pins, with the format defaults applied. */
|
|
95
|
+
declare function embeddingProfileArtifacts(profile: KnowledgeEmbeddingProfile): {
|
|
96
|
+
onnxFile: string;
|
|
97
|
+
onnxDigest: string | null;
|
|
98
|
+
tokenizerFile: string;
|
|
99
|
+
tokenizerDigest: string | null;
|
|
100
|
+
};
|
|
101
|
+
/**
|
|
102
|
+
* Whether two profiles describe one vector space, so vectors from either
|
|
103
|
+
* may be compared: same model files at the same revision, same tokenizer,
|
|
104
|
+
* pooling, normalization, dimensions, instruction prefixes and encoding.
|
|
105
|
+
* The profile id and `maxTokens` are not compared — the first is a label,
|
|
106
|
+
* the second a compile-time bound. A digest counts only when both sides
|
|
107
|
+
* declare one: two pins at one revision that hash differently name
|
|
108
|
+
* different artifacts, whatever the path says; an undeclared digest is
|
|
109
|
+
* simply not a claim.
|
|
110
|
+
*/
|
|
111
|
+
declare function sameVectorSpace(a: KnowledgeEmbeddingProfile, b: KnowledgeEmbeddingProfile): boolean;
|
|
112
|
+
/**
|
|
113
|
+
* How documents were split. `unit` names what `target`, `overlap` and the
|
|
114
|
+
* context-header budget count: tokens of the embedding profile's tokenizer
|
|
115
|
+
* (`tokenizer: 'profile'`) or characters (`tokenizer: 'none'`).
|
|
116
|
+
*/
|
|
117
|
+
declare const KnowledgeChunkingProfileSchema: z.ZodObject<{
|
|
118
|
+
id: z.ZodString;
|
|
119
|
+
unit: z.ZodEnum<{
|
|
120
|
+
tokens: "tokens";
|
|
121
|
+
chars: "chars";
|
|
122
|
+
}>;
|
|
123
|
+
tokenizer: z.ZodEnum<{
|
|
124
|
+
profile: "profile";
|
|
125
|
+
none: "none";
|
|
126
|
+
}>;
|
|
127
|
+
target: z.ZodNumber;
|
|
128
|
+
overlap: z.ZodNumber;
|
|
129
|
+
contextHeader: z.ZodObject<{
|
|
130
|
+
max: z.ZodNumber;
|
|
131
|
+
}, z.core.$strip>;
|
|
132
|
+
}, z.core.$strip>;
|
|
133
|
+
type KnowledgeChunkingProfile = z.infer<typeof KnowledgeChunkingProfileSchema>;
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* `manifest.json` — the container's self-description. `formatVersion`
|
|
137
|
+
* governs the container layout, `indexSchemaVersion` the SQLite DDL. Readers
|
|
138
|
+
* never migrate a catalog: incompatibility is a typed disabled-with-reason
|
|
139
|
+
* state, and a publisher's signed artifact is never rewritten.
|
|
140
|
+
*/
|
|
141
|
+
declare const GEZK_MANIFEST_KIND = "gezk-catalog";
|
|
142
|
+
declare const GEZK_REGISTRY_KIND = "gezk-registry";
|
|
143
|
+
declare const KnowledgeManifestFileSchema: z.ZodObject<{
|
|
144
|
+
path: z.ZodString;
|
|
145
|
+
sizeBytes: z.ZodNumber;
|
|
146
|
+
sha256: z.ZodString;
|
|
147
|
+
}, z.core.$strip>;
|
|
148
|
+
declare const KnowledgeSignatureSchema: z.ZodObject<{
|
|
149
|
+
algorithm: z.ZodLiteral<"ed25519">;
|
|
150
|
+
keyId: z.ZodString;
|
|
151
|
+
canonicalization: z.ZodLiteral<"rfc8785">;
|
|
152
|
+
value: z.ZodString;
|
|
153
|
+
}, z.core.$strip>;
|
|
154
|
+
type KnowledgeSignature = z.infer<typeof KnowledgeSignatureSchema>;
|
|
155
|
+
declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
156
|
+
kind: z.ZodLiteral<"gezk-catalog">;
|
|
157
|
+
formatVersion: z.ZodLiteral<"0.5">;
|
|
158
|
+
indexSchemaVersion: z.ZodLiteral<2>;
|
|
159
|
+
id: z.ZodString;
|
|
160
|
+
version: z.ZodString;
|
|
161
|
+
name: z.ZodString;
|
|
162
|
+
description: z.ZodOptional<z.ZodString>;
|
|
163
|
+
language: z.ZodString;
|
|
164
|
+
publisher: z.ZodObject<{
|
|
165
|
+
id: z.ZodString;
|
|
166
|
+
name: z.ZodString;
|
|
167
|
+
url: z.ZodOptional<z.ZodString>;
|
|
168
|
+
}, z.core.$strip>;
|
|
169
|
+
createdAt: z.ZodString;
|
|
170
|
+
sourceSnapshot: z.ZodOptional<z.ZodObject<{
|
|
171
|
+
name: z.ZodString;
|
|
172
|
+
date: z.ZodString;
|
|
173
|
+
taxonomyVersion: z.ZodOptional<z.ZodString>;
|
|
174
|
+
}, z.core.$strip>>;
|
|
175
|
+
license: z.ZodObject<{
|
|
176
|
+
name: z.ZodString;
|
|
177
|
+
spdx: z.ZodOptional<z.ZodString>;
|
|
178
|
+
noticePath: z.ZodString;
|
|
179
|
+
attributionRequired: z.ZodBoolean;
|
|
180
|
+
}, z.core.$strip>;
|
|
181
|
+
embedding: z.ZodObject<{
|
|
182
|
+
id: z.ZodString;
|
|
183
|
+
model: z.ZodObject<{
|
|
184
|
+
repo: z.ZodString;
|
|
185
|
+
revision: z.ZodString;
|
|
186
|
+
onnxFile: z.ZodOptional<z.ZodString>;
|
|
187
|
+
onnxDigest: z.ZodOptional<z.ZodString>;
|
|
188
|
+
}, z.core.$strip>;
|
|
189
|
+
tokenizer: z.ZodObject<{
|
|
190
|
+
kind: z.ZodString;
|
|
191
|
+
file: z.ZodOptional<z.ZodString>;
|
|
192
|
+
digest: z.ZodOptional<z.ZodString>;
|
|
193
|
+
}, z.core.$strip>;
|
|
194
|
+
pooling: z.ZodEnum<{
|
|
195
|
+
mean: "mean";
|
|
196
|
+
cls: "cls";
|
|
197
|
+
last: "last";
|
|
198
|
+
}>;
|
|
199
|
+
normalized: z.ZodBoolean;
|
|
200
|
+
dimensions: z.ZodNumber;
|
|
201
|
+
maxTokens: z.ZodNumber;
|
|
202
|
+
queryInstruction: z.ZodString;
|
|
203
|
+
passageInstruction: z.ZodString;
|
|
204
|
+
vectorEncoding: z.ZodEnum<{
|
|
205
|
+
"bit+int8": "bit+int8";
|
|
206
|
+
}>;
|
|
207
|
+
distance: z.ZodObject<{
|
|
208
|
+
stage1: z.ZodLiteral<"hamming">;
|
|
209
|
+
stage2: z.ZodLiteral<"cosine">;
|
|
210
|
+
}, z.core.$strip>;
|
|
211
|
+
quantization: z.ZodObject<{
|
|
212
|
+
int8: z.ZodObject<{
|
|
213
|
+
method: z.ZodLiteral<"symmetric-linear">;
|
|
214
|
+
scale: z.ZodLiteral<127>;
|
|
215
|
+
}, z.core.$strip>;
|
|
216
|
+
binary: z.ZodObject<{
|
|
217
|
+
method: z.ZodLiteral<"sign">;
|
|
218
|
+
threshold: z.ZodLiteral<0>;
|
|
219
|
+
packing: z.ZodLiteral<"lsb-first">;
|
|
220
|
+
}, z.core.$strip>;
|
|
221
|
+
}, z.core.$strip>;
|
|
222
|
+
}, z.core.$strip>;
|
|
223
|
+
chunking: z.ZodObject<{
|
|
224
|
+
id: z.ZodString;
|
|
225
|
+
unit: z.ZodEnum<{
|
|
226
|
+
tokens: "tokens";
|
|
227
|
+
chars: "chars";
|
|
228
|
+
}>;
|
|
229
|
+
tokenizer: z.ZodEnum<{
|
|
230
|
+
profile: "profile";
|
|
231
|
+
none: "none";
|
|
232
|
+
}>;
|
|
233
|
+
target: z.ZodNumber;
|
|
234
|
+
overlap: z.ZodNumber;
|
|
235
|
+
contextHeader: z.ZodObject<{
|
|
236
|
+
max: z.ZodNumber;
|
|
237
|
+
}, z.core.$strip>;
|
|
238
|
+
}, z.core.$strip>;
|
|
239
|
+
topics: z.ZodArray<z.ZodObject<{
|
|
240
|
+
id: z.ZodString;
|
|
241
|
+
name: z.ZodString;
|
|
242
|
+
}, z.core.$strip>>;
|
|
243
|
+
router: z.ZodObject<{
|
|
244
|
+
shardTargetChunks: z.ZodNumber;
|
|
245
|
+
shards: z.ZodArray<z.ZodObject<{
|
|
246
|
+
id: z.ZodNumber;
|
|
247
|
+
path: z.ZodString;
|
|
248
|
+
chunks: z.ZodNumber;
|
|
249
|
+
documents: z.ZodNumber;
|
|
250
|
+
centroids: z.ZodNumber;
|
|
251
|
+
sha256: z.ZodString;
|
|
252
|
+
}, z.core.$strip>>;
|
|
253
|
+
totalCentroids: z.ZodNumber;
|
|
254
|
+
}, z.core.$strip>;
|
|
255
|
+
counts: z.ZodObject<{
|
|
256
|
+
documents: z.ZodNumber;
|
|
257
|
+
chunks: z.ZodNumber;
|
|
258
|
+
shards: z.ZodNumber;
|
|
259
|
+
}, z.core.$strip>;
|
|
260
|
+
files: z.ZodArray<z.ZodObject<{
|
|
261
|
+
path: z.ZodString;
|
|
262
|
+
sizeBytes: z.ZodNumber;
|
|
263
|
+
sha256: z.ZodString;
|
|
264
|
+
}, z.core.$strip>>;
|
|
265
|
+
requires: z.ZodObject<{
|
|
266
|
+
formatVersion: z.ZodLiteral<"0.5">;
|
|
267
|
+
features: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
268
|
+
}, z.core.$strip>;
|
|
269
|
+
smokeQueries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
270
|
+
query: z.ZodString;
|
|
271
|
+
expectedDocumentIds: z.ZodArray<z.ZodString>;
|
|
272
|
+
}, z.core.$strip>>>;
|
|
273
|
+
toolchain: z.ZodOptional<z.ZodObject<{
|
|
274
|
+
name: z.ZodString;
|
|
275
|
+
version: z.ZodString;
|
|
276
|
+
node: z.ZodOptional<z.ZodString>;
|
|
277
|
+
sqlite: z.ZodOptional<z.ZodString>;
|
|
278
|
+
platform: z.ZodOptional<z.ZodString>;
|
|
279
|
+
modelDigest: z.ZodOptional<z.ZodString>;
|
|
280
|
+
tokenizerDigest: z.ZodOptional<z.ZodString>;
|
|
281
|
+
}, z.core.$strip>>;
|
|
282
|
+
signature: z.ZodOptional<z.ZodObject<{
|
|
283
|
+
algorithm: z.ZodLiteral<"ed25519">;
|
|
284
|
+
keyId: z.ZodString;
|
|
285
|
+
canonicalization: z.ZodLiteral<"rfc8785">;
|
|
286
|
+
value: z.ZodString;
|
|
287
|
+
}, z.core.$strip>>;
|
|
288
|
+
}, z.core.$strip>;
|
|
289
|
+
type KnowledgeCatalogManifest = z.infer<typeof KnowledgeCatalogManifestSchema>;
|
|
290
|
+
/** `LICENSES/source-notices.json` — per-source attribution for combined catalogs. */
|
|
291
|
+
declare const SourceNoticesSchema: z.ZodObject<{
|
|
292
|
+
sources: z.ZodArray<z.ZodObject<{
|
|
293
|
+
name: z.ZodString;
|
|
294
|
+
url: z.ZodOptional<z.ZodString>;
|
|
295
|
+
license: z.ZodString;
|
|
296
|
+
licenseUrl: z.ZodOptional<z.ZodString>;
|
|
297
|
+
snapshot: z.ZodOptional<z.ZodString>;
|
|
298
|
+
notice: z.ZodOptional<z.ZodString>;
|
|
299
|
+
}, z.core.$strip>>;
|
|
300
|
+
}, z.core.$strip>;
|
|
301
|
+
type SourceNotices = z.infer<typeof SourceNoticesSchema>;
|
|
302
|
+
|
|
303
|
+
/** One normalized document streamed into the compiler. */
|
|
304
|
+
declare const CatalogDocumentSchema: z.ZodObject<{
|
|
305
|
+
id: z.ZodString;
|
|
306
|
+
title: z.ZodString;
|
|
307
|
+
slug: z.ZodString;
|
|
308
|
+
summary: z.ZodOptional<z.ZodString>;
|
|
309
|
+
language: z.ZodString;
|
|
310
|
+
topicPath: z.ZodArray<z.ZodString>;
|
|
311
|
+
markdown: z.ZodString;
|
|
312
|
+
sourceUrl: z.ZodOptional<z.ZodString>;
|
|
313
|
+
sourceRevision: z.ZodOptional<z.ZodString>;
|
|
314
|
+
sourceUpdatedAt: z.ZodOptional<z.ZodString>;
|
|
315
|
+
attribution: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
316
|
+
aliases: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
317
|
+
}, z.core.$strip>;
|
|
318
|
+
type CatalogDocument = z.infer<typeof CatalogDocumentSchema>;
|
|
319
|
+
|
|
320
|
+
/** One downloadable catalog release a publisher's registry advertises. */
|
|
321
|
+
declare const KnowledgeRegistryEntrySchema: z.ZodObject<{
|
|
322
|
+
catalogId: z.ZodString;
|
|
323
|
+
version: z.ZodString;
|
|
324
|
+
name: z.ZodString;
|
|
325
|
+
description: z.ZodOptional<z.ZodString>;
|
|
326
|
+
language: z.ZodString;
|
|
327
|
+
documents: z.ZodNumber;
|
|
328
|
+
archiveBytes: z.ZodNumber;
|
|
329
|
+
contentDigest: z.ZodString;
|
|
330
|
+
url: z.ZodString;
|
|
331
|
+
license: z.ZodObject<{
|
|
332
|
+
name: z.ZodString;
|
|
333
|
+
attributionRequired: z.ZodBoolean;
|
|
334
|
+
}, z.core.$strip>;
|
|
335
|
+
sourceSnapshot: z.ZodOptional<z.ZodObject<{
|
|
336
|
+
name: z.ZodString;
|
|
337
|
+
date: z.ZodString;
|
|
338
|
+
taxonomyVersion: z.ZodOptional<z.ZodString>;
|
|
339
|
+
}, z.core.$strip>>;
|
|
340
|
+
}, z.core.$strip>;
|
|
341
|
+
type KnowledgeRegistryEntry = z.infer<typeof KnowledgeRegistryEntrySchema>;
|
|
342
|
+
/**
|
|
343
|
+
* The publisher registry document. Signed exactly like a catalog manifest:
|
|
344
|
+
* Ed25519 over the RFC 8785 canonical form minus `signature`, verified
|
|
345
|
+
* against keyId-indexed trust anchors. Published LAST — every archive it
|
|
346
|
+
* names must already be live, so a half-published release is invisible
|
|
347
|
+
* rather than broken, and withdrawing a release is deleting its row and
|
|
348
|
+
* re-signing (installed catalogs are never affected).
|
|
349
|
+
*/
|
|
350
|
+
declare const KnowledgeRegistryIndexSchema: z.ZodObject<{
|
|
351
|
+
kind: z.ZodLiteral<"gezk-registry">;
|
|
352
|
+
formatVersion: z.ZodLiteral<"0.5">;
|
|
353
|
+
publisher: z.ZodObject<{
|
|
354
|
+
id: z.ZodString;
|
|
355
|
+
name: z.ZodString;
|
|
356
|
+
url: z.ZodOptional<z.ZodString>;
|
|
357
|
+
}, z.core.$strip>;
|
|
358
|
+
generatedAt: z.ZodString;
|
|
359
|
+
catalogs: z.ZodArray<z.ZodObject<{
|
|
360
|
+
catalogId: z.ZodString;
|
|
361
|
+
version: z.ZodString;
|
|
362
|
+
name: z.ZodString;
|
|
363
|
+
description: z.ZodOptional<z.ZodString>;
|
|
364
|
+
language: z.ZodString;
|
|
365
|
+
documents: z.ZodNumber;
|
|
366
|
+
archiveBytes: z.ZodNumber;
|
|
367
|
+
contentDigest: z.ZodString;
|
|
368
|
+
url: z.ZodString;
|
|
369
|
+
license: z.ZodObject<{
|
|
370
|
+
name: z.ZodString;
|
|
371
|
+
attributionRequired: z.ZodBoolean;
|
|
372
|
+
}, z.core.$strip>;
|
|
373
|
+
sourceSnapshot: z.ZodOptional<z.ZodObject<{
|
|
374
|
+
name: z.ZodString;
|
|
375
|
+
date: z.ZodString;
|
|
376
|
+
taxonomyVersion: z.ZodOptional<z.ZodString>;
|
|
377
|
+
}, z.core.$strip>>;
|
|
378
|
+
}, z.core.$strip>>;
|
|
379
|
+
signature: z.ZodOptional<z.ZodObject<{
|
|
380
|
+
algorithm: z.ZodLiteral<"ed25519">;
|
|
381
|
+
keyId: z.ZodString;
|
|
382
|
+
canonicalization: z.ZodLiteral<"rfc8785">;
|
|
383
|
+
value: z.ZodString;
|
|
384
|
+
}, z.core.$strip>>;
|
|
385
|
+
}, z.core.$strip>;
|
|
386
|
+
type KnowledgeRegistryIndex = z.infer<typeof KnowledgeRegistryIndexSchema>;
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* `knowledge://<publisherId>/<catalogId>/<documentId>[#chunk=…|#line=n[-m]]`
|
|
390
|
+
* — the citation grammar. The publisher is part of the authority because
|
|
391
|
+
* catalog ids are only unique per publisher.
|
|
392
|
+
*/
|
|
393
|
+
interface KnowledgeUri {
|
|
394
|
+
publisherId: string;
|
|
395
|
+
catalogId: string;
|
|
396
|
+
documentId: string;
|
|
397
|
+
fragment?: {
|
|
398
|
+
chunk: string;
|
|
399
|
+
} | {
|
|
400
|
+
lineStart: number;
|
|
401
|
+
lineEnd?: number;
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
declare function formatKnowledgeUri(uri: KnowledgeUri): string;
|
|
405
|
+
/** Strict parse; null for anything that is not a well-formed knowledge URI. */
|
|
406
|
+
declare function parseKnowledgeUri(raw: string): KnowledgeUri | null;
|
|
407
|
+
|
|
408
|
+
/** Container-level constants of the `.gezk` format. */
|
|
409
|
+
/**
|
|
410
|
+
* Public format version. `0.x` is preliminary: breaking changes may land in
|
|
411
|
+
* any minor until 1.0, and a reader supports exactly the versions it names.
|
|
412
|
+
*/
|
|
413
|
+
declare const GEZK_FORMAT_VERSION = "0.5";
|
|
414
|
+
/** SQLite schema generation (`PRAGMA user_version` of every database). */
|
|
415
|
+
declare const GEZK_INDEX_SCHEMA_VERSION = 2;
|
|
416
|
+
/** `PRAGMA application_id` for every database in a catalog: 'GEZK'. */
|
|
417
|
+
declare const GEZK_APPLICATION_ID = 1195727435;
|
|
418
|
+
/**
|
|
419
|
+
* The container's media type. Stored uncompressed as the FIRST ZIP entry,
|
|
420
|
+
* named `mimetype`, so the file is identifiable at a fixed byte offset (the
|
|
421
|
+
* EPUB/OpenDocument convention).
|
|
422
|
+
*/
|
|
423
|
+
declare const GEZK_MIME_TYPE = "application/vnd.gezk+zip";
|
|
424
|
+
declare const MIMETYPE_PATH = "mimetype";
|
|
425
|
+
declare const MANIFEST_PATH = "manifest.json";
|
|
426
|
+
declare const README_PATH = "README.md";
|
|
427
|
+
declare const LICENSE_NOTICE_PATH = "LICENSES/catalog.txt";
|
|
428
|
+
declare const SOURCE_NOTICES_PATH = "LICENSES/source-notices.json";
|
|
429
|
+
declare const ROUTER_DB_PATH = "index/router.db";
|
|
430
|
+
/** Bodies below this ship uncompressed (`body_codec = 'none'`). */
|
|
431
|
+
declare const BODY_CODEC_MIN_BYTES = 512;
|
|
432
|
+
/** Maximum decompressed Markdown body returned from one catalog document. */
|
|
433
|
+
declare const MAX_KNOWLEDGE_DOCUMENT_BYTES: number;
|
|
434
|
+
/** Fixed ZIP entry timestamp for deterministic archives. */
|
|
435
|
+
declare const ZIP_FIXED_MTIME: Date;
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* Index schema 2 DDL. Readers never migrate these. Every table is plain
|
|
439
|
+
* SQLite (FTS5 is the only virtual-table module used), so any SQLite client
|
|
440
|
+
* can read a catalog without extensions.
|
|
441
|
+
*/
|
|
442
|
+
declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
|
|
443
|
+
/**
|
|
444
|
+
* Per-shard tables. Vectors are plain BLOB columns keyed by chunk id:
|
|
445
|
+
* `chunk_vectors_bit.v` holds ceil(dim/8) bytes of sign bits (LSB-first) and
|
|
446
|
+
* `chunk_vectors_int8.v` holds dim signed bytes (symmetric-linear, scale
|
|
447
|
+
* 127). Two tables, not one, so a stage-1 scan pages in 48 bytes per row
|
|
448
|
+
* rather than the whole record. Rowid alignment invariant:
|
|
449
|
+
* chunks.id == chunk_vectors_bit.chunk_id == chunk_vectors_int8.chunk_id ==
|
|
450
|
+
* fts_chunks.rowid, dense from 1.
|
|
451
|
+
*/
|
|
452
|
+
declare const SHARD_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE chunks(\n id INTEGER PRIMARY KEY,\n chunk_uid TEXT NOT NULL,\n document_id TEXT NOT NULL,\n ordinal INTEGER NOT NULL,\n title TEXT NOT NULL,\n heading_path TEXT NOT NULL,\n heading_text TEXT NOT NULL,\n line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,\n token_count INTEGER NOT NULL,\n content_hash TEXT NOT NULL,\n text TEXT NOT NULL\n);\nCREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);\nCREATE INDEX chunks_document ON chunks(document_id, ordinal);\n\nCREATE VIRTUAL TABLE fts_chunks USING fts5(\n title, heading_text, text,\n content = 'chunks', content_rowid = 'id',\n tokenize = 'unicode61 remove_diacritics 2'\n);\n\nCREATE TABLE chunk_vectors_bit(\n chunk_id INTEGER PRIMARY KEY,\n v BLOB NOT NULL\n);\n\nCREATE TABLE chunk_vectors_int8(\n chunk_id INTEGER PRIMARY KEY,\n v BLOB NOT NULL\n);\n";
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Frozen quantization formulas (the gezk spec §6.2). Deliberately computed
|
|
456
|
+
* here in TypeScript rather than via sqlite-vec's `vec_quantize_*` scalar
|
|
457
|
+
* functions, so extension-internal semantics can never drift the format:
|
|
458
|
+
* both the compiler (packing) and the reader (query packing + rerank) call
|
|
459
|
+
* exactly these functions.
|
|
460
|
+
*/
|
|
461
|
+
/** int8: symmetric linear, scale 127, −128 never produced. */
|
|
462
|
+
declare function quantizeInt8(unitVector: ArrayLike<number>): Int8Array;
|
|
463
|
+
/** binary: sign threshold 0 (exact 0.0 → 0), packed LSB-first. */
|
|
464
|
+
declare function quantizeBinary(unitVector: ArrayLike<number>): Uint8Array;
|
|
465
|
+
/**
|
|
466
|
+
* Rerank score: dot(float32 unit query, dequantized int8 passage). Passages
|
|
467
|
+
* were unit vectors, so this approximates cosine; the query is not quantized.
|
|
468
|
+
*/
|
|
469
|
+
declare function rerankScore(query: ArrayLike<number>, passageInt8: Int8Array): number;
|
|
470
|
+
/** L2-normalize in place-free fashion; throws on a degenerate vector. */
|
|
471
|
+
declare function l2Normalize(vector: ArrayLike<number>): Float32Array;
|
|
472
|
+
|
|
473
|
+
/**
|
|
474
|
+
* RFC 8785 (JSON Canonicalization Scheme) serializer — the byte-exact input
|
|
475
|
+
* to the manifest's Ed25519 signature (the gezk spec §12). ECMAScript's
|
|
476
|
+
* own JSON.stringify already implements the RFC's number and string
|
|
477
|
+
* serialization; canonicalization adds only (a) object keys sorted by UTF-16
|
|
478
|
+
* code units and (b) a hard rejection of anything without a JSON identity
|
|
479
|
+
* (undefined, functions, non-finite numbers, BigInt).
|
|
480
|
+
*
|
|
481
|
+
* `toJSON()` is honoured exactly as JSON.stringify honours it, so a value
|
|
482
|
+
* carrying a JSON identity of its own — a Date, most notably — canonicalizes
|
|
483
|
+
* to that identity. Without it a Date serializes as `{}`, which is how a
|
|
484
|
+
* signer holding a Date and a verifier holding its ISO string would agree on
|
|
485
|
+
* a signature over two different documents.
|
|
486
|
+
*/
|
|
487
|
+
declare function canonicalizeJson(value: unknown): string;
|
|
488
|
+
|
|
489
|
+
/** URL-ish slug for a document title (display/dedup aid, not an identity). */
|
|
490
|
+
declare function documentSlug(title: string): string;
|
|
491
|
+
|
|
492
|
+
export { ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, type CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_VERSION_PATTERN, type KnowledgeCatalogManifest, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, type KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, type KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, type KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, type KnowledgeRegistryIndex, KnowledgeRegistryIndexSchema, type KnowledgeSignature, KnowledgeSignatureSchema, type KnowledgeUri, type KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, type SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace };
|