@bendyline/gezel-knowledge 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Bendyline
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,82 @@
1
+ # @bendyline/gezel-knowledge
2
+
3
+ The `.gezk` knowledge-catalog toolchain for
4
+ [gezel](https://github.com/bendyline/gezel): a deterministic compiler, a
5
+ verified archive reader, and a read-only catalog handle with two-stage
6
+ `bit384+int8` retrieval. One format and one runtime serve a 200-document
7
+ personal reference and a 2,000,000-document Wikipedia-class corpus.
8
+
9
+ ```bash
10
+ npm install @bendyline/gezel-knowledge
11
+ ```
12
+
13
+ ## What a `.gezk` is
14
+
15
+ A standard ZIP holding `manifest.json`, a `router.db` (topics, document
16
+ directory, brotli bodies, routing centroids), and shard SQLite databases with
17
+ FTS and quantized vectors. Every catalog ships a browsable table of contents.
18
+ The format is frozen — see `docs/gezk-format-v1.md` in the gezel repo.
19
+
20
+ ## Building a catalog
21
+
22
+ ```ts
23
+ import {
24
+ GEZEL_BGE_SMALL_EN_V15_1,
25
+ GEZEL_MARKDOWN_CHUNKS_2,
26
+ compileKnowledgeCatalog,
27
+ createProfileEmbedder,
28
+ loadMarkdownCatalog,
29
+ } from '@bendyline/gezel-knowledge';
30
+
31
+ const source = await loadMarkdownCatalog('./my-notes', { language: 'en' });
32
+ const embedder = await createProfileEmbedder(GEZEL_BGE_SMALL_EN_V15_1);
33
+ await compileKnowledgeCatalog({
34
+ catalog: {
35
+ id: 'my-notes',
36
+ version: '1.0.0',
37
+ name: 'My Notes',
38
+ language: 'en',
39
+ publisher: { id: 'me', name: 'Me' },
40
+ createdAt: new Date().toISOString(),
41
+ license: { name: 'MIT', attributionRequired: false },
42
+ },
43
+ topics: source.topics,
44
+ documents: (async function* () {
45
+ for (const doc of source.documents) yield doc;
46
+ })(),
47
+ outputPath: './my-notes-1.0.0.gezk',
48
+ embeddingProfile: GEZEL_BGE_SMALL_EN_V15_1,
49
+ chunkingProfile: GEZEL_MARKDOWN_CHUNKS_2,
50
+ embed: (texts) => embedder.embed(texts),
51
+ countTokens: (text) => embedder.countTokens(text),
52
+ workDir: './.gezk-work',
53
+ });
54
+ ```
55
+
56
+ Embedding requires `@huggingface/transformers` at runtime; the module is
57
+ imported dynamically so everything else works without it. The `gezel
58
+ knowledge` CLI wraps this same API (`init`, `build`, `validate`, `inspect`,
59
+ `search`), entirely offline — no daemon.
60
+
61
+ ## Reading a catalog
62
+
63
+ ```ts
64
+ import { CatalogHandle, extractGezkVerified } from '@bendyline/gezel-knowledge';
65
+
66
+ await extractGezkVerified('./my-notes-1.0.0.gezk', './extracted');
67
+ const handle = CatalogHandle.open('./extracted');
68
+ const topics = handle.topics();
69
+ const hits = handle.searchDocumentsFts('woodworking');
70
+ handle.close();
71
+ ```
72
+
73
+ Extraction verifies every file against the manifest (both directions —
74
+ undeclared and missing files are equally fatal) and rejects unsafe archive
75
+ entries. Catalog databases are opened strictly read-only and immutable.
76
+
77
+ ## Stability
78
+
79
+ Published for the gezel toolchain and the Qualla content pipeline. The
80
+ archive format is versioned (`formatVersion` / `indexSchemaVersion`) and
81
+ readers never migrate a catalog — incompatibility is a typed, reported
82
+ state.
@@ -0,0 +1,582 @@
1
+ import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest } from '@bendyline/gezel';
2
+ import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
3
+
4
+ /** `.gezk` v1 constants — frozen in docs/gezk-format-v1.md. */
5
+ declare const GEZK_FORMAT_VERSION = 1;
6
+ declare const GEZK_INDEX_SCHEMA_VERSION = 1;
7
+ /** `PRAGMA application_id` for every database in a catalog: 'GEZK'. */
8
+ declare const GEZK_APPLICATION_ID = 1195727435;
9
+ declare const ROUTER_DB_PATH = "index/router.db";
10
+ declare const MANIFEST_PATH = "manifest.json";
11
+ /** Compiler shard sizing (§3.1). */
12
+ declare const SHARD_TARGET_CHUNKS = 200000;
13
+ declare const SHARD_MAX_CHUNKS = 250000;
14
+ /** Bodies below this ship uncompressed (`body_codec = 'none'`). */
15
+ declare const BODY_CODEC_MIN_BYTES = 512;
16
+ /** Maximum decompressed Markdown body returned from one catalog document. */
17
+ declare const MAX_KNOWLEDGE_DOCUMENT_BYTES: number;
18
+ /** Routing (§3.3–3.4). */
19
+ declare const CENTROID_CHUNKS_PER = 12500;
20
+ declare const CENTROID_MAX_PER_SHARD = 32;
21
+ declare const CENTROID_KMEANS_MAX_ITER = 25;
22
+ declare const CENTROID_SAMPLE_MAX = 65536;
23
+ declare const ROUTE_SHARDS_PROACTIVE = 3;
24
+ declare const ROUTE_SHARDS_EXPLICIT = 6;
25
+ /** Two-stage retrieval (§2): hamming top-K → int8 rerank. */
26
+ declare const RERANK_FINAL_K = 24;
27
+ declare function rerankK(finalK?: number, chunkCount?: number): number;
28
+ /** Fixed ZIP entry timestamp for deterministic archives (§11). */
29
+ declare const ZIP_FIXED_MTIME: Date;
30
+
31
+ /** Frozen v1 DDL (gezk-format-v1.md §6). Readers never migrate these. */
32
+ declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
33
+ /** Per-shard tables. The vec0 table is created separately (needs sqlite-vec). */
34
+ declare const SHARD_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE chunks(\n id INTEGER PRIMARY KEY,\n chunk_uid TEXT NOT NULL,\n document_id TEXT NOT NULL,\n ordinal INTEGER NOT NULL,\n title TEXT NOT NULL,\n heading_path TEXT NOT NULL,\n heading_text TEXT NOT NULL,\n line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,\n token_count INTEGER NOT NULL,\n content_hash TEXT NOT NULL,\n text TEXT NOT NULL\n);\nCREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);\nCREATE INDEX chunks_document ON chunks(document_id, ordinal);\n\nCREATE VIRTUAL TABLE fts_chunks USING fts5(\n title, heading_text, text,\n content = 'chunks', content_rowid = 'id',\n tokenize = 'unicode61 remove_diacritics 2'\n);\n\nCREATE TABLE chunk_vectors_int8(\n chunk_id INTEGER PRIMARY KEY,\n v BLOB NOT NULL\n);\n";
35
+ /** vec0 table for the hamming stage; `dim` comes from the embedding profile. */
36
+ declare function vecChunksDdl(dim: number): string;
37
+
38
+ /**
39
+ * Stable chunk id (gezk-format-v1.md §5): lowercase hex of the FIRST 16
40
+ * bytes of SHA-256(docId || 0x00 || decimal-ordinal || 0x00 ||
41
+ * SHA-256(chunkText) raw 32 bytes). Content-derived so identical inputs on
42
+ * any toolchain produce identical ids — the anchor of both reproducibility
43
+ * and the `#chunk=` citation fragment.
44
+ */
45
+ declare function chunkUid(documentId: string, ordinal: number, chunkText: string): string;
46
+ /** Full SHA-256 hex of a chunk's text (`chunks.content_hash`). */
47
+ declare function chunkContentHash(chunkText: string): string;
48
+ /** URL-ish slug for a document title (display/dedup aid, not an identity). */
49
+ declare function documentSlug(title: string): string;
50
+
51
+ /**
52
+ * Frozen quantization formulas (gezk-format-v1.md §2). Deliberately computed
53
+ * here in TypeScript rather than via sqlite-vec's `vec_quantize_*` scalar
54
+ * functions, so extension-internal semantics can never drift the format:
55
+ * both the compiler (packing) and the reader (query packing + rerank) call
56
+ * exactly these functions.
57
+ */
58
+ /** int8: symmetric linear, scale 127, −128 never produced. */
59
+ declare function quantizeInt8(unitVector: ArrayLike<number>): Int8Array;
60
+ /** binary: sign threshold 0 (exact 0.0 → 0), packed LSB-first. */
61
+ declare function quantizeBinary(unitVector: ArrayLike<number>): Uint8Array;
62
+ /**
63
+ * Rerank score: dot(float32 unit query, dequantized int8 passage). Passages
64
+ * were unit vectors, so this approximates cosine; the query is not quantized.
65
+ */
66
+ declare function rerankScore(query: ArrayLike<number>, passageInt8: Int8Array): number;
67
+ /** L2-normalize in place-free fashion; throws on a degenerate vector. */
68
+ declare function l2Normalize(vector: ArrayLike<number>): Float32Array;
69
+
70
+ /**
71
+ * The canonical markdown chunker, parameterized over the two frozen
72
+ * profiles (docs/gezk-format-v1.md §5):
73
+ *
74
+ * - `gezel-markdown-chunks@1` — unit 'chars', 4000/400: byte-for-byte the
75
+ * algorithm lifted from the project indexer
76
+ * (packages/service/src/index-store/docs.ts `chunkMarkdown` /
77
+ * `chunkLineRange`), so the service can later re-point here with zero
78
+ * behavior change.
79
+ * - `gezel-markdown-chunks@2` — unit 'tokens', 420/64: the knowledge
80
+ * profile. Token counting is injected (`countTokens` — the embedding
81
+ * profile's own tokenizer), packing is per-line with sentence-level
82
+ * splitting for oversized lines, and every chunk carries a structural
83
+ * `headingPath` (root→nearest, the section's own heading last).
84
+ *
85
+ * Pure functions, no I/O, no dependencies — importable by the compiler, the
86
+ * service indexer, and tests alike.
87
+ */
88
+ interface MarkdownChunk {
89
+ kind: 'doc' | 'preamble' | 'section';
90
+ /** 1-based inclusive source line span. */
91
+ lineStart: number;
92
+ lineEnd: number;
93
+ text: string;
94
+ /** Raw heading texts, root → nearest (empty for preamble/doc chunks). */
95
+ headingPath: string[];
96
+ }
97
+ interface CharChunkerOptions {
98
+ unit: 'chars';
99
+ /** Max chunk size in characters (profile @1: 4000). */
100
+ target: number;
101
+ /** Overlap replayed between adjacent chunks, in characters (@1: 400). */
102
+ overlap: number;
103
+ }
104
+ interface TokenChunkerOptions {
105
+ unit: 'tokens';
106
+ /** Target chunk size in tokens (profile @2: 420). */
107
+ target: number;
108
+ /** Overlap in tokens (@2: 64). */
109
+ overlap: number;
110
+ /** The embedding profile's tokenizer — deterministic given its pinned digest. */
111
+ countTokens: (text: string) => number;
112
+ }
113
+ type ChunkerOptions = CharChunkerOptions | TokenChunkerOptions;
114
+ /** Split markdown into heading-aware, bounded, overlapping chunks. */
115
+ declare function chunkMarkdownProfile(md: string, opts: ChunkerOptions): MarkdownChunk[];
116
+
117
+ /**
118
+ * `.gezk` compiler (gezk-format-v1.md). One owner of normalization, chunk
119
+ * ids, embedding-profile conformance, SQLite schema, shard assignment,
120
+ * centroid routing, and deterministic archive layout — the CLI's Markdown
121
+ * adapter and Qualla's Wikipedia adapter both feed this API.
122
+ *
123
+ * Determinism contract (§11): identical inputs + identical toolchain
124
+ * fingerprint ⇒ byte-identical archive. Everything time- or random-shaped is
125
+ * an input (`createdAt`) or seeded (k-means); the embedder is injected so
126
+ * callers control its determinism (tests inject a hash-based embedder; the
127
+ * real pipeline's content-hash vector cache is the reproducibility
128
+ * authority across machines).
129
+ *
130
+ * Memory note: documents are collected and sorted in-process. That is fine
131
+ * for author-built catalogs (the CLI path); the Wikipedia-scale pipeline
132
+ * spools per-catalog NDJSON partitions and feeds each partition separately
133
+ * (its documents arrive pre-sorted), so the in-memory set stays bounded.
134
+ */
135
+
136
+ interface CompileTopic {
137
+ id: string;
138
+ name: string;
139
+ parentId?: string;
140
+ description?: string;
141
+ sortKey?: string;
142
+ }
143
+ interface CompileKnowledgeCatalogOptions {
144
+ catalog: {
145
+ id: string;
146
+ version: string;
147
+ name: string;
148
+ description?: string;
149
+ language: string;
150
+ publisher: {
151
+ id: string;
152
+ name: string;
153
+ url?: string;
154
+ };
155
+ /** Supplied, never wall-clock — determinism. */
156
+ createdAt: string;
157
+ license: {
158
+ name: string;
159
+ noticePath?: string;
160
+ attributionRequired: boolean;
161
+ };
162
+ sourceSnapshot?: {
163
+ name: string;
164
+ date: string;
165
+ taxonomyVersion?: string;
166
+ };
167
+ };
168
+ /** The shipped table of contents — the compiler REJECTS an empty list. */
169
+ topics: CompileTopic[];
170
+ documents: AsyncIterable<CatalogDocument>;
171
+ outputPath: string;
172
+ embeddingProfile: KnowledgeEmbeddingProfile;
173
+ chunkingProfile: KnowledgeChunkingProfile;
174
+ /**
175
+ * Profile-conformant passage embedder: takes ALREADY-PREFIXED texts,
176
+ * returns unit vectors at the profile's dimension. Injected so the CLI
177
+ * wires the daemon's pipeline, Qualla wires its worker pool, and tests
178
+ * wire a deterministic fake.
179
+ */
180
+ embed: (texts: string[]) => Promise<number[][]>;
181
+ /** The profile tokenizer's counter (chunking + header truncation). */
182
+ countTokens: (text: string) => number;
183
+ /** Scratch directory for the staged databases (removed on success). */
184
+ workDir: string;
185
+ smokeQueries?: Array<{
186
+ query: string;
187
+ expectedDocumentIds: string[];
188
+ }>;
189
+ toolchain?: KnowledgeCatalogManifest['toolchain'];
190
+ /** Extra archive files (README.md, LICENSES/…): path → utf8 content. */
191
+ extraFiles?: Record<string, string>;
192
+ /**
193
+ * Last touch before the manifest is archived — the signing seam
194
+ * (signatures/signing.ts `signManifest`). Must only add/replace the
195
+ * `signature` field; the result is re-parsed, so structural edits fail
196
+ * loudly rather than shipping.
197
+ */
198
+ finalizeManifest?: (manifest: KnowledgeCatalogManifest) => KnowledgeCatalogManifest;
199
+ onProgress?: (progress: {
200
+ phase: string;
201
+ done: number;
202
+ total: number;
203
+ }) => void;
204
+ embedBatchSize?: number;
205
+ }
206
+ interface CompileReport {
207
+ manifest: KnowledgeCatalogManifest;
208
+ documents: number;
209
+ chunks: number;
210
+ shards: number;
211
+ archiveBytes: number;
212
+ }
213
+ declare function compileKnowledgeCatalog(opts: CompileKnowledgeCatalogOptions): Promise<CompileReport>;
214
+
215
+ /**
216
+ * Deterministic `.gezk` ZIP writer (gezk-format-v1.md §11): entries sorted
217
+ * by path bytes, fixed timestamp (2000-01-01T00:00:00Z), mode 0644, UTF-8
218
+ * names, ZIP64 as needed (yazl handles the transition). Compression is
219
+ * zlib's default deflate — deterministic per toolchain, which is exactly the
220
+ * determinism promise (the manifest records the toolchain fingerprint).
221
+ */
222
+ interface ArchiveEntry {
223
+ /** Forward-slashed archive path. */
224
+ path: string;
225
+ /** Exactly one of absPath / content. */
226
+ absPath?: string;
227
+ content?: Buffer;
228
+ }
229
+ declare function writeGezkArchive(outputPath: string, entries: ArchiveEntry[]): Promise<void>;
230
+
231
+ /**
232
+ * Streaming `.gezk` inspection + verified extraction — the gezmodel archive
233
+ * discipline (packages/service/src/models/gezmodel.ts) applied to catalogs:
234
+ * yauzl lazyEntries, per-entry safety rejection (traversal, absolute paths,
235
+ * symlinks, encrypted entries, case-insensitive duplicates, directory
236
+ * entries), manifest-declared-only extraction with BOTH-direction
237
+ * reconciliation, and per-file SHA-256 verification against the manifest.
238
+ * Never inflates the archive into memory.
239
+ */
240
+
241
+ /** Catalog-specific limits — multi-GB archives are the point (draft §install). */
242
+ declare const GEZK_ARCHIVE_LIMITS: {
243
+ readonly maxEntries: 16384;
244
+ readonly maxManifestBytes: number;
245
+ readonly maxEntryBytes: number;
246
+ readonly maxTotalBytes: number;
247
+ readonly maxCompressionRatio: 400;
248
+ };
249
+ declare class GezkArchiveError extends Error {
250
+ readonly reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'io';
251
+ constructor(message: string, reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'io');
252
+ }
253
+ interface GezkArchiveInspection {
254
+ manifest: KnowledgeCatalogManifest;
255
+ entryCount: number;
256
+ totalUncompressedBytes: number;
257
+ }
258
+ /** Safety-check and reconcile the complete central directory without extracting. */
259
+ declare function inspectGezkArchive(archivePath: string): Promise<GezkArchiveInspection>;
260
+ /** Read + parse manifest.json without extracting anything else. */
261
+ declare function readGezkManifest(archivePath: string): Promise<KnowledgeCatalogManifest>;
262
+ /**
263
+ * Extract a verified catalog into `destDir`: entries and manifest.files must
264
+ * reconcile in BOTH directions (an undeclared archive file and a declared-
265
+ * but-missing file are equally fatal), and every extracted file's SHA-256
266
+ * must match its declaration.
267
+ */
268
+ declare function extractGezkVerified(archivePath: string, destDir: string): Promise<KnowledgeCatalogManifest>;
269
+
270
+ /**
271
+ * node:sqlite loader shim. The repo's tsup/esbuild pipeline rewrites static
272
+ * `node:`-prefixed imports to their bare legacy names, which is harmless for
273
+ * `fs`/`crypto` but fatal for `sqlite` — the builtin exists ONLY under the
274
+ * `node:` prefix, so the bundled import resolves to a nonexistent npm
275
+ * package. The service hit the same wall and loads it by string at runtime
276
+ * (sqlite-driver.ts `nodeRequire('node:sqlite')`); this shim is that pattern
277
+ * with the types re-exported so call sites stay typed.
278
+ */
279
+
280
+ declare const DatabaseSync: typeof DatabaseSync$1;
281
+ type DatabaseSync = DatabaseSync$1;
282
+
283
+ /**
284
+ * Read-only, immutable catalog database opener. Deliberately NOT the
285
+ * service's `openIndexDatabase`, whose behaviors are all wrong for a signed
286
+ * publisher artifact: it mkdirs the parent, switches on WAL (sidecar files),
287
+ * and quarantine-RENAMES a corrupt database. A catalog reader must fail
288
+ * with a typed reason and never touch the bytes.
289
+ */
290
+
291
+ declare class CatalogOpenError extends Error {
292
+ readonly reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'vec-unavailable' | 'corrupt';
293
+ constructor(message: string, reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'vec-unavailable' | 'corrupt');
294
+ }
295
+ interface CatalogDb {
296
+ db: DatabaseSync;
297
+ close(): void;
298
+ }
299
+ /**
300
+ * Open one catalog SQLite file read-only + immutable, load sqlite-vec, and
301
+ * verify the format stamps. `immutable=1` promises SQLite the file cannot
302
+ * change while open — no locking, no journal probing — which is true for a
303
+ * published catalog version and is what makes concurrent readers free.
304
+ */
305
+ declare function openCatalogDatabase(absPath: string): CatalogDb;
306
+
307
+ /**
308
+ * One mounted catalog: the router database plus lazily opened shards, and
309
+ * the two-stage query flow (gezk-format-v1.md §3, §9):
310
+ *
311
+ * embed query (caller) → centroid routing → per shard: bit-hamming KNN
312
+ * top-K → int8 rerank → fuse.
313
+ *
314
+ * The handle is synchronous (node:sqlite) and single-threaded by design —
315
+ * the daemon confines every handle to its knowledge worker thread; the CLI
316
+ * and tests call it directly.
317
+ */
318
+ interface CatalogTopic {
319
+ id: string;
320
+ parentId: string | null;
321
+ name: string;
322
+ description: string | null;
323
+ sortKey: string;
324
+ documentCount: number;
325
+ }
326
+ interface CatalogDocumentMeta {
327
+ id: string;
328
+ title: string;
329
+ slug: string;
330
+ summary: string | null;
331
+ language: string;
332
+ topicId: string;
333
+ sourceUrl: string | null;
334
+ sourceRevision: string | null;
335
+ sourceUpdatedAt: string | null;
336
+ attribution: Record<string, string> | null;
337
+ }
338
+ interface CatalogChunkHit {
339
+ chunkUid: string;
340
+ documentId: string;
341
+ title: string;
342
+ headingPath: string[];
343
+ lineStart: number;
344
+ lineEnd: number;
345
+ text: string;
346
+ /** Rerank cosine (semantic hits) — absent for FTS-only hits. */
347
+ cosine?: number;
348
+ source: 'vector' | 'fts';
349
+ shardId: number;
350
+ }
351
+ interface ShardInfo {
352
+ id: number;
353
+ path: string;
354
+ chunkCount: number;
355
+ }
356
+ declare class CatalogHandle {
357
+ readonly rootDir: string;
358
+ private readonly router;
359
+ private readonly connections;
360
+ readonly meta: Record<string, string>;
361
+ readonly shards: ShardInfo[];
362
+ private constructor();
363
+ /** Open a catalog's extracted version directory (the dir holding manifest.json). */
364
+ static open(rootDir: string): CatalogHandle;
365
+ close(): void;
366
+ /** Integrity check for the router connection, used by deep validation. */
367
+ routerQuickCheck(): string;
368
+ /** Metadata-only body profile; SQLite's length() does not materialize blobs. */
369
+ documentBodyProfile(): {
370
+ maxRawBytes: number;
371
+ maxCompressedBytes: number;
372
+ unknownCodecs: number;
373
+ };
374
+ private shardDb;
375
+ /** Resolve a manifest/router path without allowing it to escape the catalog. */
376
+ resolveCatalogPath(path: string): string;
377
+ topics(): CatalogTopic[];
378
+ documentsPage(opts?: {
379
+ topicId?: string;
380
+ offset?: number;
381
+ limit?: number;
382
+ }): {
383
+ documents: CatalogDocumentMeta[];
384
+ total: number;
385
+ };
386
+ getDocument(id: string): (CatalogDocumentMeta & {
387
+ markdown: string;
388
+ }) | null;
389
+ /** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
390
+ searchDocumentsFts(query: string, limit?: number): Array<{
391
+ documentId: string;
392
+ rank: number;
393
+ }>;
394
+ /**
395
+ * Score every shard against the (unit float32) query: shard score = max
396
+ * cosine over its centroids (§3.4). This is the cross-catalog routing
397
+ * primitive — the daemon merges scores from every active catalog and
398
+ * takes the top-S GLOBALLY, so multiple catalogs share one scan budget.
399
+ */
400
+ scoreShards(queryVector: Float32Array): Array<{
401
+ shardId: number;
402
+ score: number;
403
+ }>;
404
+ /** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
405
+ routeShards(queryVector: Float32Array, shardBudget: number): number[];
406
+ /**
407
+ * Two-stage semantic search over the routed shards: bit-hamming KNN
408
+ * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
409
+ */
410
+ searchSemantic(queryVector: Float32Array, opts?: {
411
+ shardBudget?: number;
412
+ finalK?: number;
413
+ }): CatalogChunkHit[];
414
+ /**
415
+ * The scan stage against an EXPLICIT shard list — what the daemon calls
416
+ * after global cross-catalog routing has already spent the S budget.
417
+ */
418
+ searchShards(queryVector: Float32Array, shardIds: number[], finalK: number): CatalogChunkHit[];
419
+ /** Chunk-body FTS over the routed shards (explicit search only). */
420
+ searchChunksFts(query: string, shardIds: number[], limitPerShard?: number): CatalogChunkHit[];
421
+ /**
422
+ * Embedder-free integrity smoke (§6.3): chunk 1's own bit vector must be
423
+ * its own nearest neighbor — proves the extension parses this file's vec
424
+ * blobs. Returns false instead of throwing (callers report a reason).
425
+ */
426
+ selfKnnSmoke(shardId: number): boolean;
427
+ }
428
+
429
+ /**
430
+ * Catalog validation (gezk-format-v1.md §6.3) over an EXTRACTED catalog
431
+ * directory: manifest parse, per-file hash reconciliation, read-only opens,
432
+ * count reconciliation, and — in deep mode — SQLite quick_check, per-shard
433
+ * vector-table alignment, the embedder-free self-KNN smoke, and the
434
+ * manifest's own smoke queries against fts_documents. Never mutates the
435
+ * catalog; every failure is a named check, not an exception.
436
+ */
437
+
438
+ interface CatalogCheck {
439
+ name: string;
440
+ ok: boolean;
441
+ detail?: string;
442
+ }
443
+ interface CatalogValidationReport {
444
+ ok: boolean;
445
+ manifest: KnowledgeCatalogManifest | null;
446
+ checks: CatalogCheck[];
447
+ }
448
+ interface ValidateCatalogOptions {
449
+ /** quick_check + vector alignment + self-KNN + smoke queries. */
450
+ deep?: boolean;
451
+ }
452
+ declare function validateExtractedCatalog(rootDir: string, opts?: ValidateCatalogOptions): Promise<CatalogValidationReport>;
453
+
454
+ /**
455
+ * The registered embedding profiles (gezk-format-v1.md §4) — the single
456
+ * source both the compiler and the reader import. A profile is the FULL
457
+ * vector-space identity: readers refuse to mix vectors across profile ids,
458
+ * and any change to model, instructions, or encoding is a NEW id.
459
+ *
460
+ * Revisions are the HF commit hashes captured at Phase-0 freeze
461
+ * (2026-08-20). The onnx/tokenizer digests are recorded when the release
462
+ * pipeline first pulls the artifacts (Phase 5/6 hardening); their absence
463
+ * does not weaken catalog verification, which pins vectors by content.
464
+ */
465
+
466
+ /** The public Qualla profile (user decision: multilingual for world catalogs). */
467
+ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
468
+ /** The local-build default — matches gezel's shipped project embedder. */
469
+ declare const GEZEL_BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
470
+ declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
471
+ declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
472
+ /** The knowledge chunking profile (gezk-format-v1.md §5). */
473
+ declare const GEZEL_MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
474
+
475
+ /**
476
+ * Profile-conformant embedder + tokenizer, driven entirely by a declarative
477
+ * KnowledgeEmbeddingProfile — model repo, pinned revision, pooling, and
478
+ * instruction prefixes all come from the profile object, never from env-var
479
+ * regexes. This is what the offline CLI wires into the compiler and the
480
+ * explicit-search path; the daemon wires its own pipeline pool later.
481
+ *
482
+ * `@huggingface/transformers` stays a dynamic import (mirroring the
483
+ * service's embed-core): the CLI must start instantly without it, and a
484
+ * missing module surfaces as an actionable install message, not an
485
+ * ERR_MODULE_NOT_FOUND. Honors GEZEL_HF_CACHE_DIR so CLI builds share the
486
+ * daemon's model cache.
487
+ */
488
+
489
+ interface ProfileEmbedder {
490
+ readonly profile: KnowledgeEmbeddingProfile;
491
+ /** Raw passage embed — takes ALREADY-PREFIXED texts (the compiler's contract). */
492
+ embed(texts: string[]): Promise<number[][]>;
493
+ /** Query embed — applies the profile's queryInstruction itself. */
494
+ embedQuery(text: string): Promise<Float32Array>;
495
+ /** Profile-tokenizer token count (sync once loaded — chunking's contract). */
496
+ countTokens(text: string): number;
497
+ dispose(): Promise<void>;
498
+ }
499
+ declare class EmbedderUnavailableError extends Error {
500
+ readonly isActionable = true;
501
+ constructor(message: string);
502
+ }
503
+ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
504
+ cacheDir?: string;
505
+ }): Promise<ProfileEmbedder>;
506
+
507
+ /**
508
+ * Markdown-folder catalog source — the `gezel knowledge build` input adapter.
509
+ * Folders become the shipped table of contents (a format requirement:
510
+ * gezk-format-v1.md §6.1), files become documents. A flat corpus — files at
511
+ * the root with no subfolders — gets the single root topic the compiler
512
+ * demands. Deterministic: sorted walk, stable topic-id collision suffixes.
513
+ */
514
+
515
+ interface MarkdownCatalogSource {
516
+ topics: CompileTopic[];
517
+ documents: CatalogDocument[];
518
+ }
519
+ interface LoadMarkdownCatalogOptions {
520
+ language: string;
521
+ /** Topic id used for root-level files (default `general`). */
522
+ rootTopicId?: string;
523
+ rootTopicName?: string;
524
+ }
525
+ /**
526
+ * Load a folder of Markdown into compiler inputs. Directory chain → topic
527
+ * chain; the relative path (without extension, `/`-separated) is the stable
528
+ * document id.
529
+ */
530
+ declare function loadMarkdownCatalog(rootDir: string, opts: LoadMarkdownCatalogOptions): Promise<MarkdownCatalogSource>;
531
+
532
+ /**
533
+ * RFC 8785 (JSON Canonicalization Scheme) serializer — the byte-exact input
534
+ * to the manifest's Ed25519 signature (gezk-format-v1.md §7). ECMAScript's
535
+ * own JSON.stringify already implements the RFC's number and string
536
+ * serialization; canonicalization adds only (a) object keys sorted by UTF-16
537
+ * code units and (b) a hard rejection of anything without a JSON identity
538
+ * (undefined, functions, non-finite numbers, BigInt).
539
+ */
540
+ declare function canonicalizeJson(value: unknown): string;
541
+
542
+ /**
543
+ * Manifest signing (gezk-format-v1.md §7): Ed25519 over the RFC 8785
544
+ * canonical form of the manifest WITHOUT its `signature` field. Key
545
+ * encodings follow the repo's identity conventions (remotes/identity.ts):
546
+ * SPKI/PKCS#8 PEM, base64 signatures, node:crypto one-shot sign/verify.
547
+ * The private key never appears in CI — signing is a manual workstation
548
+ * step (the plan's key-custody stance); verification is what ships.
549
+ */
550
+
551
+ interface KnowledgeTrustAnchor {
552
+ keyId: string;
553
+ publicKeyPem: string;
554
+ /** Display-only: whose key this is (e.g. `qualla`). */
555
+ publisherId?: string;
556
+ }
557
+ type ManifestSignatureVerdict = {
558
+ ok: true;
559
+ keyId: string;
560
+ } | {
561
+ ok: false;
562
+ reason: 'unsigned' | 'unknown-key' | 'bad-signature' | 'error';
563
+ detail?: string;
564
+ };
565
+ /** Stable key identity: first 16 hex chars of SHA-256 over the SPKI DER. */
566
+ declare function knowledgeKeyId(publicKeyPem: string): string;
567
+ declare function generateKnowledgeSigningKeyPair(): {
568
+ publicKeyPem: string;
569
+ privateKeyPem: string;
570
+ keyId: string;
571
+ };
572
+ /** The exact bytes the signature covers: JCS(manifest minus `signature`). */
573
+ declare function manifestSigningPayload(manifest: KnowledgeCatalogManifest): Buffer;
574
+ declare function signManifest(manifest: KnowledgeCatalogManifest, privateKeyPem: string): KnowledgeCatalogManifest;
575
+ /**
576
+ * Verify against keyId-indexed trust anchors. Key rotation = overlapping
577
+ * anchors for one publisher; an unknown keyId is a hard failure, never a
578
+ * fallback scan across anchors.
579
+ */
580
+ declare function verifyManifestSignature(manifest: KnowledgeCatalogManifest, anchors: readonly KnowledgeTrustAnchor[]): ManifestSignatureVerdict;
581
+
582
+ export { type ArchiveEntry, BODY_CODEC_MIN_BYTES, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, EmbedderUnavailableError, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_APPLICATION_ID, GEZK_ARCHIVE_LIMITS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GezkArchiveError, type GezkArchiveInspection, KNOWLEDGE_EMBEDDING_PROFILES, type KnowledgeTrustAnchor, type LoadMarkdownCatalogOptions, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, type ManifestSignatureVerdict, type MarkdownCatalogSource, type MarkdownChunk, type ProfileEmbedder, RERANK_FINAL_K, ROUTER_DB_PATH, ROUTER_DDL, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_DDL, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type TokenChunkerOptions, type ValidateCatalogOptions, ZIP_FIXED_MTIME, canonicalizeJson, chunkContentHash, chunkMarkdownProfile, chunkUid, compileKnowledgeCatalog, createProfileEmbedder, documentSlug, extractGezkVerified, generateKnowledgeSigningKeyPair, inspectGezkArchive, knowledgeEmbeddingProfile, knowledgeKeyId, l2Normalize, loadMarkdownCatalog, manifestSigningPayload, openCatalogDatabase, quantizeBinary, quantizeInt8, readGezkManifest, rerankK, rerankScore, signManifest, validateExtractedCatalog, vecChunksDdl, verifyManifestSignature, writeGezkArchive };