@bendyline/gezel-knowledge 1.0.6 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +138 -12
- package/dist/index.js +223 -27
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest } from '@bendyline/gezel';
|
|
1
|
+
import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezel';
|
|
2
|
+
export { CatalogDocument, KnowledgeCatalogManifest, KnowledgeCatalogRef, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex } from '@bendyline/gezel';
|
|
2
3
|
import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
|
|
3
4
|
|
|
4
5
|
/** `.gezk` v1 constants — frozen in docs/gezk-format-v1.md. */
|
|
@@ -186,6 +187,21 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
186
187
|
query: string;
|
|
187
188
|
expectedDocumentIds: string[];
|
|
188
189
|
}>;
|
|
190
|
+
/**
|
|
191
|
+
* How to treat `smokeQueries` at seal time. Every recorded query is
|
|
192
|
+
* verified against the BUILT index with the validator's exact semantics
|
|
193
|
+
* (sanitize → fts_documents → top-N) before the manifest is written — a
|
|
194
|
+
* catalog must never ship sanity queries it cannot answer (the arts-pilot
|
|
195
|
+
* "smoke:Abbey (missing 1315)" install failure).
|
|
196
|
+
*
|
|
197
|
+
* - 'require' (default): every query must pass; a miss fails the build
|
|
198
|
+
* with the query, the missing ids, and guidance.
|
|
199
|
+
* - 'select': the list is CANDIDATES — verify in order, record the first
|
|
200
|
+
* three that pass, and fail only when none do. For producers that
|
|
201
|
+
* derive queries mechanically (Qualla uses article titles) and cannot
|
|
202
|
+
* know in advance which titles win the BM25 rank race.
|
|
203
|
+
*/
|
|
204
|
+
smokeQueryPolicy?: 'require' | 'select';
|
|
189
205
|
toolchain?: KnowledgeCatalogManifest['toolchain'];
|
|
190
206
|
/** Extra archive files (README.md, LICENSES/…): path → utf8 content. */
|
|
191
207
|
extraFiles?: Record<string, string>;
|
|
@@ -202,6 +218,13 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
202
218
|
total: number;
|
|
203
219
|
}) => void;
|
|
204
220
|
embedBatchSize?: number;
|
|
221
|
+
/**
|
|
222
|
+
* TEST/EVAL override of the 200k shard target (gezk-format-v1.md §3.1) —
|
|
223
|
+
* lets a small corpus build multi-shard so routing recall can be measured
|
|
224
|
+
* against a scan-all control without a 2M-document corpus. Production
|
|
225
|
+
* builds must not set it: the constant IS the format's scale design.
|
|
226
|
+
*/
|
|
227
|
+
shardTargetChunks?: number;
|
|
205
228
|
}
|
|
206
229
|
interface CompileReport {
|
|
207
230
|
manifest: KnowledgeCatalogManifest;
|
|
@@ -391,6 +414,16 @@ declare class CatalogHandle {
|
|
|
391
414
|
documentId: string;
|
|
392
415
|
rank: number;
|
|
393
416
|
}>;
|
|
417
|
+
/**
|
|
418
|
+
* Router-level shape stats: topic-tree size and total route centroids.
|
|
419
|
+
* Read-only counts for inspectors and recall/routing diagnostics — a
|
|
420
|
+
* single-topic catalog fills shards in documentId order, which is the
|
|
421
|
+
* first thing to check when centroid routing underperforms.
|
|
422
|
+
*/
|
|
423
|
+
routerStats(): {
|
|
424
|
+
topics: number;
|
|
425
|
+
routeCentroids: number;
|
|
426
|
+
};
|
|
394
427
|
/**
|
|
395
428
|
* Score every shard against the (unit float32) query: shard score = max
|
|
396
429
|
* cosine over its centroids (§3.4). This is the cross-catalog routing
|
|
@@ -502,6 +535,14 @@ declare class EmbedderUnavailableError extends Error {
|
|
|
502
535
|
}
|
|
503
536
|
declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
|
|
504
537
|
cacheDir?: string;
|
|
538
|
+
/**
|
|
539
|
+
* onnxruntime session options (e.g. `{ intraOpNumThreads }`), passed
|
|
540
|
+
* through to the pipeline. Worker pools use this to divide cores between
|
|
541
|
+
* concurrent sessions instead of letting every session grab them all.
|
|
542
|
+
* Thread counts change scheduling, never numerics — batch composition is
|
|
543
|
+
* what affects values, and that stays with the caller.
|
|
544
|
+
*/
|
|
545
|
+
sessionOptions?: Record<string, unknown>;
|
|
505
546
|
}): Promise<ProfileEmbedder>;
|
|
506
547
|
|
|
507
548
|
/**
|
|
@@ -529,16 +570,6 @@ interface LoadMarkdownCatalogOptions {
|
|
|
529
570
|
*/
|
|
530
571
|
declare function loadMarkdownCatalog(rootDir: string, opts: LoadMarkdownCatalogOptions): Promise<MarkdownCatalogSource>;
|
|
531
572
|
|
|
532
|
-
/**
|
|
533
|
-
* RFC 8785 (JSON Canonicalization Scheme) serializer — the byte-exact input
|
|
534
|
-
* to the manifest's Ed25519 signature (gezk-format-v1.md §7). ECMAScript's
|
|
535
|
-
* own JSON.stringify already implements the RFC's number and string
|
|
536
|
-
* serialization; canonicalization adds only (a) object keys sorted by UTF-16
|
|
537
|
-
* code units and (b) a hard rejection of anything without a JSON identity
|
|
538
|
-
* (undefined, functions, non-finite numbers, BigInt).
|
|
539
|
-
*/
|
|
540
|
-
declare function canonicalizeJson(value: unknown): string;
|
|
541
|
-
|
|
542
573
|
/**
|
|
543
574
|
* Manifest signing (gezk-format-v1.md §7): Ed25519 over the RFC 8785
|
|
544
575
|
* canonical form of the manifest WITHOUT its `signature` field. Key
|
|
@@ -578,5 +609,100 @@ declare function signManifest(manifest: KnowledgeCatalogManifest, privateKeyPem:
|
|
|
578
609
|
* fallback scan across anchors.
|
|
579
610
|
*/
|
|
580
611
|
declare function verifyManifestSignature(manifest: KnowledgeCatalogManifest, anchors: readonly KnowledgeTrustAnchor[]): ManifestSignatureVerdict;
|
|
612
|
+
/**
|
|
613
|
+
* The publisher registry (`_knowledge/registry/index.json`) signs with the
|
|
614
|
+
* same discipline as a catalog manifest: Ed25519 over JCS(document minus
|
|
615
|
+
* `signature`). The signing side lives in Qualla's release command; readers
|
|
616
|
+
* (the machine broker's resolver, Settings → available catalogs) verify.
|
|
617
|
+
*/
|
|
618
|
+
declare function signRegistryIndex(index: KnowledgeRegistryIndex, privateKeyPem: string): KnowledgeRegistryIndex;
|
|
619
|
+
declare function verifyRegistryIndex(index: KnowledgeRegistryIndex, anchors: readonly KnowledgeTrustAnchor[]): ManifestSignatureVerdict;
|
|
620
|
+
|
|
621
|
+
/**
|
|
622
|
+
* Signed-registry consumption (knowledge-catalogs.md Phase 6): fetch a
|
|
623
|
+
* publisher's `_knowledge/registry/index.json`, verify its Ed25519 signature
|
|
624
|
+
* against keyId-indexed trust anchors, and locate archive coordinates.
|
|
625
|
+
*
|
|
626
|
+
* Trust posture: the registry only LOCATES bytes. Content trust stays with
|
|
627
|
+
* the coordinate's `contentDigest` — every consumer re-hashes the downloaded
|
|
628
|
+
* archive against the digest it already trusted, so a compromised registry
|
|
629
|
+
* (or CDN) can at worst fail to locate, never substitute content. Signature
|
|
630
|
+
* verification here protects the locate step itself: an unverified registry
|
|
631
|
+
* cannot steer downloads at attacker-sized garbage or hide/reorder releases.
|
|
632
|
+
*/
|
|
633
|
+
|
|
634
|
+
type KnowledgeRegistryFetchCode = 'http' | 'too-large' | 'invalid' | 'unsigned-or-untrusted';
|
|
635
|
+
declare class KnowledgeRegistryFetchError extends Error {
|
|
636
|
+
readonly code: KnowledgeRegistryFetchCode;
|
|
637
|
+
constructor(code: KnowledgeRegistryFetchCode, message: string);
|
|
638
|
+
}
|
|
639
|
+
interface FetchKnowledgeRegistryOptions {
|
|
640
|
+
anchors: readonly KnowledgeTrustAnchor[];
|
|
641
|
+
maxBytes?: number;
|
|
642
|
+
/** Awake-time budget (suspend-aware) for the whole fetch. */
|
|
643
|
+
timeoutMs?: number;
|
|
644
|
+
fetchImpl?: typeof fetch;
|
|
645
|
+
}
|
|
646
|
+
/**
|
|
647
|
+
* Fetch + schema-parse + signature-verify a registry. Throws
|
|
648
|
+
* `KnowledgeRegistryFetchError`; never returns an unverified document.
|
|
649
|
+
*/
|
|
650
|
+
declare function fetchKnowledgeRegistry(url: string, opts: FetchKnowledgeRegistryOptions): Promise<{
|
|
651
|
+
registry: KnowledgeRegistryIndex;
|
|
652
|
+
keyId: string;
|
|
653
|
+
}>;
|
|
654
|
+
/**
|
|
655
|
+
* Locate the row for an exact trusted coordinate. `contentDigest`, when
|
|
656
|
+
* given, must match the row — a registry that renames a digest under an
|
|
657
|
+
* existing (catalogId, version) simply fails to locate.
|
|
658
|
+
*/
|
|
659
|
+
declare function findRegistryEntry(registry: KnowledgeRegistryIndex, ref: {
|
|
660
|
+
publisherId: string;
|
|
661
|
+
catalogId: string;
|
|
662
|
+
version: string;
|
|
663
|
+
contentDigest?: string;
|
|
664
|
+
}): KnowledgeRegistryEntry | null;
|
|
665
|
+
/**
|
|
666
|
+
* Compare two catalog versions (CalVer-style dotted numerics, e.g.
|
|
667
|
+
* `2026.08.1`). Non-numeric segments fall back to lexicographic comparison
|
|
668
|
+
* so unconventional versions still order deterministically.
|
|
669
|
+
*/
|
|
670
|
+
declare function compareCatalogVersions(a: string, b: string): number;
|
|
671
|
+
/**
|
|
672
|
+
* Rows offering a strictly newer version of an installed catalog, newest
|
|
673
|
+
* first. Rows whose version does not exceed the installed one are ignored —
|
|
674
|
+
* a registry cannot "update" an install to an older or renamed release.
|
|
675
|
+
*/
|
|
676
|
+
declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installed: {
|
|
677
|
+
catalogId: string;
|
|
678
|
+
version: string;
|
|
679
|
+
}): KnowledgeRegistryEntry[];
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Built-in trust anchors for signed knowledge registries and catalogs.
|
|
683
|
+
*
|
|
684
|
+
* Empty until the Qualla production keypair exists: the private key is
|
|
685
|
+
* generated OFFLINE on a workstation (`sign-knowledge-release --generate-key`)
|
|
686
|
+
* and its `.pub` lands here — pasted into a normal reviewed release, never
|
|
687
|
+
* fetched at runtime. Rotation = overlapping keyIds across an app release:
|
|
688
|
+
* add the new anchor, keep the old one until every registry consumers care
|
|
689
|
+
* about is re-signed, then remove it.
|
|
690
|
+
*
|
|
691
|
+
* Operators and tests extend the set without a release via
|
|
692
|
+
* `GEZEL_KNOWLEDGE_TRUST_ANCHORS` (a JSON file of KnowledgeTrustAnchor rows),
|
|
693
|
+
* which the service layers ON TOP of these — it can add trust, not veto it.
|
|
694
|
+
*/
|
|
695
|
+
|
|
696
|
+
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
697
|
+
|
|
698
|
+
/**
|
|
699
|
+
* RFC 8785 (JSON Canonicalization Scheme) serializer — the byte-exact input
|
|
700
|
+
* to the manifest's Ed25519 signature (gezk-format-v1.md §7). ECMAScript's
|
|
701
|
+
* own JSON.stringify already implements the RFC's number and string
|
|
702
|
+
* serialization; canonicalization adds only (a) object keys sorted by UTF-16
|
|
703
|
+
* code units and (b) a hard rejection of anything without a JSON identity
|
|
704
|
+
* (undefined, functions, non-finite numbers, BigInt).
|
|
705
|
+
*/
|
|
706
|
+
declare function canonicalizeJson(value: unknown): string;
|
|
581
707
|
|
|
582
|
-
export { type ArchiveEntry, BODY_CODEC_MIN_BYTES, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, EmbedderUnavailableError, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_APPLICATION_ID, GEZK_ARCHIVE_LIMITS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GezkArchiveError, type GezkArchiveInspection, KNOWLEDGE_EMBEDDING_PROFILES, type KnowledgeTrustAnchor, type LoadMarkdownCatalogOptions, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, type ManifestSignatureVerdict, type MarkdownCatalogSource, type MarkdownChunk, type ProfileEmbedder, RERANK_FINAL_K, ROUTER_DB_PATH, ROUTER_DDL, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_DDL, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type TokenChunkerOptions, type ValidateCatalogOptions, ZIP_FIXED_MTIME, canonicalizeJson, chunkContentHash, chunkMarkdownProfile, chunkUid, compileKnowledgeCatalog, createProfileEmbedder, documentSlug, extractGezkVerified, generateKnowledgeSigningKeyPair, inspectGezkArchive, knowledgeEmbeddingProfile, knowledgeKeyId, l2Normalize, loadMarkdownCatalog, manifestSigningPayload, openCatalogDatabase, quantizeBinary, quantizeInt8, readGezkManifest, rerankK, rerankScore, signManifest, validateExtractedCatalog, vecChunksDdl, verifyManifestSignature, writeGezkArchive };
|
|
708
|
+
export { type ArchiveEntry, BODY_CODEC_MIN_BYTES, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_APPLICATION_ID, GEZK_ARCHIVE_LIMITS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GezkArchiveError, type GezkArchiveInspection, KNOWLEDGE_EMBEDDING_PROFILES, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type KnowledgeTrustAnchor, type LoadMarkdownCatalogOptions, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, type ManifestSignatureVerdict, type MarkdownCatalogSource, type MarkdownChunk, type ProfileEmbedder, RERANK_FINAL_K, ROUTER_DB_PATH, ROUTER_DDL, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_DDL, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type TokenChunkerOptions, type ValidateCatalogOptions, ZIP_FIXED_MTIME, canonicalizeJson, chunkContentHash, chunkMarkdownProfile, chunkUid, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, documentSlug, extractGezkVerified, fetchKnowledgeRegistry, findRegistryEntry, generateKnowledgeSigningKeyPair, inspectGezkArchive, knowledgeEmbeddingProfile, knowledgeKeyId, l2Normalize, loadMarkdownCatalog, manifestSigningPayload, newerRegistryEntries, openCatalogDatabase, quantizeBinary, quantizeInt8, readGezkManifest, rerankK, rerankScore, signManifest, signRegistryIndex, validateExtractedCatalog, vecChunksDdl, verifyManifestSignature, verifyRegistryIndex, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -350,6 +350,27 @@ var nodeRequire2 = createRequire2(import.meta.url);
|
|
|
350
350
|
var sqlite = nodeRequire2("node:sqlite");
|
|
351
351
|
var DatabaseSync = sqlite.DatabaseSync;
|
|
352
352
|
|
|
353
|
+
// src/reader/fts-query.ts
|
|
354
|
+
function sanitizeFtsQuery(query) {
|
|
355
|
+
const tokens = [
|
|
356
|
+
...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
|
|
357
|
+
];
|
|
358
|
+
if (tokens.length === 0) return null;
|
|
359
|
+
return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
|
|
360
|
+
}
|
|
361
|
+
function documentFtsTopIds(db, match, limit) {
|
|
362
|
+
return db.prepare(
|
|
363
|
+
"SELECT document_id FROM fts_documents WHERE fts_documents MATCH ? ORDER BY rank LIMIT ?"
|
|
364
|
+
).all(match, limit).map((row) => row.document_id);
|
|
365
|
+
}
|
|
366
|
+
var SMOKE_QUERY_TOP_N = 10;
|
|
367
|
+
function documentSmokeQueryMisses(db, smoke) {
|
|
368
|
+
const match = sanitizeFtsQuery(smoke.query);
|
|
369
|
+
if (!match) return [...smoke.expectedDocumentIds];
|
|
370
|
+
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N);
|
|
371
|
+
return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
|
|
372
|
+
}
|
|
373
|
+
|
|
353
374
|
// src/compiler/vec-load.ts
|
|
354
375
|
import { createRequire as createRequire3 } from "module";
|
|
355
376
|
var nodeRequire3 = createRequire3(import.meta.url);
|
|
@@ -404,7 +425,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
404
425
|
(a, b) => a.topicPathKey === b.topicPathKey ? a.doc.id < b.doc.id ? -1 : a.doc.id > b.doc.id ? 1 : 0 : a.topicPathKey < b.topicPathKey ? -1 : 1
|
|
405
426
|
);
|
|
406
427
|
const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
|
|
407
|
-
const
|
|
428
|
+
const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
|
|
429
|
+
const embedded = totalChunks <= shardTarget;
|
|
408
430
|
const shardOf = /* @__PURE__ */ new Map();
|
|
409
431
|
let shardCount = 1;
|
|
410
432
|
if (embedded) {
|
|
@@ -413,7 +435,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
413
435
|
let current = 0;
|
|
414
436
|
let filled = 0;
|
|
415
437
|
for (const p of prepared) {
|
|
416
|
-
if (filled > 0 && filled + p.chunks.length >
|
|
438
|
+
if (filled > 0 && filled + p.chunks.length > shardTarget) {
|
|
417
439
|
current++;
|
|
418
440
|
filled = 0;
|
|
419
441
|
}
|
|
@@ -690,6 +712,35 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
690
712
|
}
|
|
691
713
|
router.close();
|
|
692
714
|
for (const db of shardDbs.values()) db.close();
|
|
715
|
+
let smokeQueries = opts.smokeQueries;
|
|
716
|
+
if (smokeQueries && smokeQueries.length > 0) {
|
|
717
|
+
const routerFinal = new DatabaseSync(join(finalDir, ROUTER_DB_PATH), { readOnly: true });
|
|
718
|
+
try {
|
|
719
|
+
const verdicts = smokeQueries.map((smoke) => ({
|
|
720
|
+
smoke,
|
|
721
|
+
missing: documentSmokeQueryMisses(routerFinal, smoke)
|
|
722
|
+
}));
|
|
723
|
+
if ((opts.smokeQueryPolicy ?? "require") === "select") {
|
|
724
|
+
smokeQueries = verdicts.filter((v) => v.missing.length === 0).slice(0, 3).map((v) => v.smoke);
|
|
725
|
+
if (smokeQueries.length === 0) {
|
|
726
|
+
throw new Error(
|
|
727
|
+
`no smoke-query candidate passes against the built index (tried ${verdicts.length}: ${verdicts.map((v) => `"${v.smoke.query}"`).join(", ")}) \u2014 the fts_documents index cannot surface any candidate's expected document in the top ${SMOKE_QUERY_TOP_N}`
|
|
728
|
+
);
|
|
729
|
+
}
|
|
730
|
+
} else {
|
|
731
|
+
const failed = verdicts.filter((v) => v.missing.length > 0);
|
|
732
|
+
if (failed.length > 0) {
|
|
733
|
+
throw new Error(
|
|
734
|
+
`smoke queries failed against the built index: ${failed.map((v) => `"${v.smoke.query}" (missing ${v.missing.join(", ")})`).join(
|
|
735
|
+
"; "
|
|
736
|
+
)} \u2014 every recorded sanity query must return its expected documents in the top ${SMOKE_QUERY_TOP_N} fts_documents results; pick more specific queries (distinctive titles beat common words in the rank race) or pass smokeQueryPolicy: 'select' with candidates`
|
|
737
|
+
);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
} finally {
|
|
741
|
+
routerFinal.close();
|
|
742
|
+
}
|
|
743
|
+
}
|
|
693
744
|
const files = [];
|
|
694
745
|
const routerShardStats = [];
|
|
695
746
|
for (const f of vacuumed) {
|
|
@@ -740,14 +791,14 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
740
791
|
chunking: opts.chunkingProfile,
|
|
741
792
|
topics: opts.topics.map((t) => ({ id: t.id, name: t.name })),
|
|
742
793
|
router: {
|
|
743
|
-
shardTargetChunks:
|
|
794
|
+
shardTargetChunks: shardTarget,
|
|
744
795
|
shards: routerShardStats.sort((a, b) => a.id - b.id),
|
|
745
796
|
totalCentroids: routerShardStats.reduce((sum, s) => sum + s.centroids, 0)
|
|
746
797
|
},
|
|
747
798
|
counts: { documents: prepared.length, chunks: totalChunks, shards: shardCount },
|
|
748
799
|
files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
749
800
|
compatibility: { maximumIndexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION },
|
|
750
|
-
...
|
|
801
|
+
...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
|
|
751
802
|
...opts.toolchain ? { toolchain: opts.toolchain } : {}
|
|
752
803
|
});
|
|
753
804
|
const manifest = opts.finalizeManifest ? KnowledgeCatalogManifestSchema.parse(opts.finalizeManifest(unsignedManifest)) : unsignedManifest;
|
|
@@ -1375,13 +1426,26 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1375
1426
|
const match = ftsQuery(query);
|
|
1376
1427
|
if (!match) return [];
|
|
1377
1428
|
try {
|
|
1378
|
-
return this.router.db.
|
|
1379
|
-
|
|
1380
|
-
|
|
1429
|
+
return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
|
|
1430
|
+
documentId,
|
|
1431
|
+
rank: i
|
|
1432
|
+
}));
|
|
1381
1433
|
} catch {
|
|
1382
1434
|
return [];
|
|
1383
1435
|
}
|
|
1384
1436
|
}
|
|
1437
|
+
/**
|
|
1438
|
+
* Router-level shape stats: topic-tree size and total route centroids.
|
|
1439
|
+
* Read-only counts for inspectors and recall/routing diagnostics — a
|
|
1440
|
+
* single-topic catalog fills shards in documentId order, which is the
|
|
1441
|
+
* first thing to check when centroid routing underperforms.
|
|
1442
|
+
*/
|
|
1443
|
+
routerStats() {
|
|
1444
|
+
const count = (table) => Number(
|
|
1445
|
+
this.router.db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get().c
|
|
1446
|
+
);
|
|
1447
|
+
return { topics: count("topics"), routeCentroids: count("route_centroids") };
|
|
1448
|
+
}
|
|
1385
1449
|
/**
|
|
1386
1450
|
* Score every shard against the (unit float32) query: shard score = max
|
|
1387
1451
|
* cosine over its centroids (§3.4). This is the cross-catalog routing
|
|
@@ -1549,13 +1613,7 @@ function parseHeadingPath(raw) {
|
|
|
1549
1613
|
return [];
|
|
1550
1614
|
}
|
|
1551
1615
|
}
|
|
1552
|
-
|
|
1553
|
-
const tokens = [
|
|
1554
|
-
...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
|
|
1555
|
-
];
|
|
1556
|
-
if (tokens.length === 0) return null;
|
|
1557
|
-
return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
|
|
1558
|
-
}
|
|
1616
|
+
var ftsQuery = sanitizeFtsQuery;
|
|
1559
1617
|
|
|
1560
1618
|
// src/reader/validate.ts
|
|
1561
1619
|
import { readFile } from "fs/promises";
|
|
@@ -1695,12 +1753,12 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1695
1753
|
check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
|
|
1696
1754
|
}
|
|
1697
1755
|
for (const smoke of manifest.smokeQueries ?? []) {
|
|
1698
|
-
const hits = handle.searchDocumentsFts(smoke.query,
|
|
1756
|
+
const hits = handle.searchDocumentsFts(smoke.query, SMOKE_QUERY_TOP_N).map((h) => h.documentId);
|
|
1699
1757
|
const missing = smoke.expectedDocumentIds.filter((id) => !hits.includes(id));
|
|
1700
1758
|
check(
|
|
1701
1759
|
`smoke:${smoke.query}`,
|
|
1702
1760
|
missing.length === 0,
|
|
1703
|
-
missing.length > 0 ? `missing ${missing.join(", ")}` : void 0
|
|
1761
|
+
missing.length > 0 ? `the catalog's built-in sanity query "${smoke.query}" did not return its expected document${missing.length > 1 ? "s" : ""} (${missing.join(", ")}) in the top ${SMOKE_QUERY_TOP_N} results \u2014 the search index cannot answer queries the publisher recorded as guaranteed, so the archive is corrupt or was built by a broken toolchain; re-download it or report it to the publisher` : void 0
|
|
1704
1762
|
);
|
|
1705
1763
|
}
|
|
1706
1764
|
}
|
|
@@ -1793,7 +1851,10 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
1793
1851
|
transformers.env.useFSCache = true;
|
|
1794
1852
|
transformers.env.allowRemoteModels = true;
|
|
1795
1853
|
}
|
|
1796
|
-
const modelOpts = {
|
|
1854
|
+
const modelOpts = {
|
|
1855
|
+
revision: profile.model.revision,
|
|
1856
|
+
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
|
|
1857
|
+
};
|
|
1797
1858
|
const [pipe, tokenizer] = await Promise.all([
|
|
1798
1859
|
transformers.pipeline("feature-extraction", profile.model.repo, modelOpts),
|
|
1799
1860
|
transformers.AutoTokenizer.from_pretrained(profile.model.repo, modelOpts)
|
|
@@ -1959,6 +2020,20 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
1959
2020
|
return { topics, documents };
|
|
1960
2021
|
}
|
|
1961
2022
|
|
|
2023
|
+
// src/registry-client/fetch.ts
|
|
2024
|
+
import { awakeTimeoutSignal } from "@bendyline/gezel";
|
|
2025
|
+
import { KnowledgeRegistryIndexSchema } from "@bendyline/gezel";
|
|
2026
|
+
|
|
2027
|
+
// src/signatures/signing.ts
|
|
2028
|
+
import {
|
|
2029
|
+
createHash as createHash5,
|
|
2030
|
+
createPrivateKey,
|
|
2031
|
+
createPublicKey,
|
|
2032
|
+
sign as cryptoSign,
|
|
2033
|
+
verify as cryptoVerify,
|
|
2034
|
+
generateKeyPairSync
|
|
2035
|
+
} from "crypto";
|
|
2036
|
+
|
|
1962
2037
|
// src/signatures/jcs.ts
|
|
1963
2038
|
function canonicalizeJson(value) {
|
|
1964
2039
|
return serialize(value, "$");
|
|
@@ -1994,14 +2069,6 @@ function compareUtf16(a, b) {
|
|
|
1994
2069
|
}
|
|
1995
2070
|
|
|
1996
2071
|
// src/signatures/signing.ts
|
|
1997
|
-
import {
|
|
1998
|
-
createHash as createHash5,
|
|
1999
|
-
createPrivateKey,
|
|
2000
|
-
createPublicKey,
|
|
2001
|
-
sign as cryptoSign,
|
|
2002
|
-
verify as cryptoVerify,
|
|
2003
|
-
generateKeyPairSync
|
|
2004
|
-
} from "crypto";
|
|
2005
2072
|
function knowledgeKeyId(publicKeyPem) {
|
|
2006
2073
|
const der = createPublicKey(publicKeyPem).export({ type: "spki", format: "der" });
|
|
2007
2074
|
return createHash5("sha256").update(der).digest("hex").slice(0, 16);
|
|
@@ -2034,7 +2101,30 @@ function signManifest(manifest, privateKeyPem) {
|
|
|
2034
2101
|
};
|
|
2035
2102
|
}
|
|
2036
2103
|
function verifyManifestSignature(manifest, anchors) {
|
|
2037
|
-
|
|
2104
|
+
return verifySignedDocument(manifest, anchors);
|
|
2105
|
+
}
|
|
2106
|
+
function signRegistryIndex(index, privateKeyPem) {
|
|
2107
|
+
const key = createPrivateKey(privateKeyPem);
|
|
2108
|
+
const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
|
|
2109
|
+
const { signature: _omitted, ...unsigned } = index;
|
|
2110
|
+
const value = cryptoSign(null, Buffer.from(canonicalizeJson(unsigned), "utf8"), key).toString(
|
|
2111
|
+
"base64"
|
|
2112
|
+
);
|
|
2113
|
+
return {
|
|
2114
|
+
...index,
|
|
2115
|
+
signature: {
|
|
2116
|
+
algorithm: "ed25519",
|
|
2117
|
+
keyId: knowledgeKeyId(publicKeyPem),
|
|
2118
|
+
canonicalization: "rfc8785",
|
|
2119
|
+
value
|
|
2120
|
+
}
|
|
2121
|
+
};
|
|
2122
|
+
}
|
|
2123
|
+
function verifyRegistryIndex(index, anchors) {
|
|
2124
|
+
return verifySignedDocument(index, anchors);
|
|
2125
|
+
}
|
|
2126
|
+
function verifySignedDocument(document, anchors) {
|
|
2127
|
+
const signature = document.signature;
|
|
2038
2128
|
if (!signature) return { ok: false, reason: "unsigned" };
|
|
2039
2129
|
const anchor = anchors.find((a) => a.keyId === signature.keyId);
|
|
2040
2130
|
if (!anchor) {
|
|
@@ -2044,9 +2134,10 @@ function verifyManifestSignature(manifest, anchors) {
|
|
|
2044
2134
|
if (knowledgeKeyId(anchor.publicKeyPem) !== signature.keyId) {
|
|
2045
2135
|
return { ok: false, reason: "unknown-key", detail: "anchor keyId does not match its key" };
|
|
2046
2136
|
}
|
|
2137
|
+
const { signature: _omitted, ...unsigned } = document;
|
|
2047
2138
|
const ok = cryptoVerify(
|
|
2048
2139
|
null,
|
|
2049
|
-
|
|
2140
|
+
Buffer.from(canonicalizeJson(unsigned), "utf8"),
|
|
2050
2141
|
createPublicKey(anchor.publicKeyPem),
|
|
2051
2142
|
Buffer.from(signature.value, "base64")
|
|
2052
2143
|
);
|
|
@@ -2055,8 +2146,106 @@ function verifyManifestSignature(manifest, anchors) {
|
|
|
2055
2146
|
return { ok: false, reason: "error", detail: err instanceof Error ? err.message : String(err) };
|
|
2056
2147
|
}
|
|
2057
2148
|
}
|
|
2149
|
+
|
|
2150
|
+
// src/registry-client/fetch.ts
|
|
2151
|
+
var DEFAULT_MAX_REGISTRY_BYTES = 4 * 1024 * 1024;
|
|
2152
|
+
var DEFAULT_TIMEOUT_MS = 3e4;
|
|
2153
|
+
var KnowledgeRegistryFetchError = class extends Error {
|
|
2154
|
+
code;
|
|
2155
|
+
constructor(code, message) {
|
|
2156
|
+
super(message);
|
|
2157
|
+
this.name = "KnowledgeRegistryFetchError";
|
|
2158
|
+
this.code = code;
|
|
2159
|
+
}
|
|
2160
|
+
};
|
|
2161
|
+
async function fetchKnowledgeRegistry(url, opts) {
|
|
2162
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
2163
|
+
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_REGISTRY_BYTES;
|
|
2164
|
+
let response;
|
|
2165
|
+
try {
|
|
2166
|
+
response = await fetchImpl(url, {
|
|
2167
|
+
signal: awakeTimeoutSignal(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
|
2168
|
+
redirect: "follow",
|
|
2169
|
+
headers: { accept: "application/json" }
|
|
2170
|
+
});
|
|
2171
|
+
} catch (err) {
|
|
2172
|
+
throw new KnowledgeRegistryFetchError(
|
|
2173
|
+
"http",
|
|
2174
|
+
`registry fetch failed: ${err instanceof Error ? err.message : String(err)}`
|
|
2175
|
+
);
|
|
2176
|
+
}
|
|
2177
|
+
if (!response.ok) {
|
|
2178
|
+
throw new KnowledgeRegistryFetchError("http", `registry fetch failed: HTTP ${response.status}`);
|
|
2179
|
+
}
|
|
2180
|
+
const declared = Number(response.headers.get("content-length") ?? "0");
|
|
2181
|
+
if (declared > maxBytes) {
|
|
2182
|
+
throw new KnowledgeRegistryFetchError(
|
|
2183
|
+
"too-large",
|
|
2184
|
+
`registry declares ${declared} bytes (cap ${maxBytes})`
|
|
2185
|
+
);
|
|
2186
|
+
}
|
|
2187
|
+
const body = Buffer.from(await response.arrayBuffer());
|
|
2188
|
+
if (body.byteLength > maxBytes) {
|
|
2189
|
+
throw new KnowledgeRegistryFetchError(
|
|
2190
|
+
"too-large",
|
|
2191
|
+
`registry body is ${body.byteLength} bytes (cap ${maxBytes})`
|
|
2192
|
+
);
|
|
2193
|
+
}
|
|
2194
|
+
let registry;
|
|
2195
|
+
try {
|
|
2196
|
+
registry = KnowledgeRegistryIndexSchema.parse(JSON.parse(body.toString("utf8")));
|
|
2197
|
+
} catch (err) {
|
|
2198
|
+
throw new KnowledgeRegistryFetchError(
|
|
2199
|
+
"invalid",
|
|
2200
|
+
`registry is not a valid gezel-knowledge-registry document: ${err instanceof Error ? err.message : String(err)}`
|
|
2201
|
+
);
|
|
2202
|
+
}
|
|
2203
|
+
const verdict = verifyRegistryIndex(registry, opts.anchors);
|
|
2204
|
+
if (!verdict.ok) {
|
|
2205
|
+
throw new KnowledgeRegistryFetchError(
|
|
2206
|
+
"unsigned-or-untrusted",
|
|
2207
|
+
`registry signature rejected (${verdict.reason}${verdict.detail ? `: ${verdict.detail}` : ""})`
|
|
2208
|
+
);
|
|
2209
|
+
}
|
|
2210
|
+
return { registry, keyId: verdict.keyId };
|
|
2211
|
+
}
|
|
2212
|
+
function findRegistryEntry(registry, ref) {
|
|
2213
|
+
if (registry.publisher.id !== ref.publisherId) return null;
|
|
2214
|
+
const entry = registry.catalogs.find(
|
|
2215
|
+
(c) => c.catalogId === ref.catalogId && c.version === ref.version
|
|
2216
|
+
);
|
|
2217
|
+
if (!entry) return null;
|
|
2218
|
+
if (ref.contentDigest && entry.contentDigest !== ref.contentDigest.toLowerCase()) return null;
|
|
2219
|
+
return entry;
|
|
2220
|
+
}
|
|
2221
|
+
function compareCatalogVersions(a, b) {
|
|
2222
|
+
const as = a.split(".");
|
|
2223
|
+
const bs = b.split(".");
|
|
2224
|
+
const len = Math.max(as.length, bs.length);
|
|
2225
|
+
for (let i = 0; i < len; i++) {
|
|
2226
|
+
const av = as[i] ?? "";
|
|
2227
|
+
const bv = bs[i] ?? "";
|
|
2228
|
+
const an = /^\d+$/.test(av) ? Number(av) : null;
|
|
2229
|
+
const bn = /^\d+$/.test(bv) ? Number(bv) : null;
|
|
2230
|
+
if (an !== null && bn !== null) {
|
|
2231
|
+
if (an !== bn) return an < bn ? -1 : 1;
|
|
2232
|
+
} else if (av !== bv) {
|
|
2233
|
+
return av < bv ? -1 : 1;
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
return 0;
|
|
2237
|
+
}
|
|
2238
|
+
function newerRegistryEntries(registry, installed) {
|
|
2239
|
+
return registry.catalogs.filter(
|
|
2240
|
+
(c) => c.catalogId === installed.catalogId && compareCatalogVersions(c.version, installed.version) > 0
|
|
2241
|
+
).sort((x, y) => compareCatalogVersions(y.version, x.version));
|
|
2242
|
+
}
|
|
2243
|
+
|
|
2244
|
+
// src/signatures/anchors.ts
|
|
2245
|
+
var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
|
|
2058
2246
|
export {
|
|
2059
2247
|
BODY_CODEC_MIN_BYTES,
|
|
2248
|
+
BUILTIN_KNOWLEDGE_TRUST_ANCHORS,
|
|
2060
2249
|
CENTROID_CHUNKS_PER,
|
|
2061
2250
|
CENTROID_KMEANS_MAX_ITER,
|
|
2062
2251
|
CENTROID_MAX_PER_SHARD,
|
|
@@ -2073,6 +2262,7 @@ export {
|
|
|
2073
2262
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
2074
2263
|
GezkArchiveError,
|
|
2075
2264
|
KNOWLEDGE_EMBEDDING_PROFILES,
|
|
2265
|
+
KnowledgeRegistryFetchError,
|
|
2076
2266
|
MANIFEST_PATH,
|
|
2077
2267
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
2078
2268
|
RERANK_FINAL_K,
|
|
@@ -2088,10 +2278,13 @@ export {
|
|
|
2088
2278
|
chunkContentHash,
|
|
2089
2279
|
chunkMarkdownProfile,
|
|
2090
2280
|
chunkUid,
|
|
2281
|
+
compareCatalogVersions,
|
|
2091
2282
|
compileKnowledgeCatalog,
|
|
2092
2283
|
createProfileEmbedder,
|
|
2093
2284
|
documentSlug,
|
|
2094
2285
|
extractGezkVerified,
|
|
2286
|
+
fetchKnowledgeRegistry,
|
|
2287
|
+
findRegistryEntry,
|
|
2095
2288
|
generateKnowledgeSigningKeyPair,
|
|
2096
2289
|
inspectGezkArchive,
|
|
2097
2290
|
knowledgeEmbeddingProfile,
|
|
@@ -2099,6 +2292,7 @@ export {
|
|
|
2099
2292
|
l2Normalize,
|
|
2100
2293
|
loadMarkdownCatalog,
|
|
2101
2294
|
manifestSigningPayload,
|
|
2295
|
+
newerRegistryEntries,
|
|
2102
2296
|
openCatalogDatabase,
|
|
2103
2297
|
quantizeBinary,
|
|
2104
2298
|
quantizeInt8,
|
|
@@ -2106,8 +2300,10 @@ export {
|
|
|
2106
2300
|
rerankK,
|
|
2107
2301
|
rerankScore,
|
|
2108
2302
|
signManifest,
|
|
2303
|
+
signRegistryIndex,
|
|
2109
2304
|
validateExtractedCatalog,
|
|
2110
2305
|
vecChunksDdl,
|
|
2111
2306
|
verifyManifestSignature,
|
|
2307
|
+
verifyRegistryIndex,
|
|
2112
2308
|
writeGezkArchive
|
|
2113
2309
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/gezel-knowledge",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.7",
|
|
4
4
|
"description": "Gezel knowledge catalogs: the .gezk compiler, archive safety, read-only reader, routing, and signatures.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|
|
@@ -47,11 +47,11 @@
|
|
|
47
47
|
"!dist/**/*.map"
|
|
48
48
|
],
|
|
49
49
|
"dependencies": {
|
|
50
|
+
"@bendyline/gezel": "1.0.7",
|
|
50
51
|
"sqlite-vec": "^0.1.9",
|
|
51
52
|
"yauzl": "^3.4.0",
|
|
52
53
|
"yazl": "^3.3.1",
|
|
53
|
-
"zod": "^4.4.3"
|
|
54
|
-
"@bendyline/gezel": "1.0.6"
|
|
54
|
+
"zod": "^4.4.3"
|
|
55
55
|
},
|
|
56
56
|
"devDependencies": {
|
|
57
57
|
"@huggingface/transformers": "^3.8.1",
|