@bendyline/gezk 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +122 -12
- package/dist/index.js +259 -12
- package/dist/node.d.ts +1 -1
- package/dist/node.js +258 -11
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -154,8 +154,11 @@ declare const KnowledgeSignatureSchema: z.ZodObject<{
|
|
|
154
154
|
type KnowledgeSignature = z.infer<typeof KnowledgeSignatureSchema>;
|
|
155
155
|
declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
156
156
|
kind: z.ZodLiteral<"gezk-catalog">;
|
|
157
|
-
formatVersion: z.
|
|
158
|
-
|
|
157
|
+
formatVersion: z.ZodEnum<{
|
|
158
|
+
0.6: "0.6";
|
|
159
|
+
0.5: "0.5";
|
|
160
|
+
}>;
|
|
161
|
+
indexSchemaVersion: z.ZodUnion<readonly [z.ZodLiteral<2>, z.ZodLiteral<3>]>;
|
|
159
162
|
id: z.ZodString;
|
|
160
163
|
version: z.ZodString;
|
|
161
164
|
name: z.ZodString;
|
|
@@ -239,6 +242,9 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
|
239
242
|
topics: z.ZodArray<z.ZodObject<{
|
|
240
243
|
id: z.ZodString;
|
|
241
244
|
name: z.ZodString;
|
|
245
|
+
parentId: z.ZodOptional<z.ZodString>;
|
|
246
|
+
sortKey: z.ZodOptional<z.ZodString>;
|
|
247
|
+
description: z.ZodOptional<z.ZodString>;
|
|
242
248
|
}, z.core.$strip>>;
|
|
243
249
|
router: z.ZodObject<{
|
|
244
250
|
shardTargetChunks: z.ZodNumber;
|
|
@@ -256,6 +262,7 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
|
256
262
|
documents: z.ZodNumber;
|
|
257
263
|
chunks: z.ZodNumber;
|
|
258
264
|
shards: z.ZodNumber;
|
|
265
|
+
assets: z.ZodOptional<z.ZodNumber>;
|
|
259
266
|
}, z.core.$strip>;
|
|
260
267
|
files: z.ZodArray<z.ZodObject<{
|
|
261
268
|
path: z.ZodString;
|
|
@@ -263,7 +270,10 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
|
263
270
|
sha256: z.ZodString;
|
|
264
271
|
}, z.core.$strip>>;
|
|
265
272
|
requires: z.ZodObject<{
|
|
266
|
-
formatVersion: z.
|
|
273
|
+
formatVersion: z.ZodEnum<{
|
|
274
|
+
0.6: "0.6";
|
|
275
|
+
0.5: "0.5";
|
|
276
|
+
}>;
|
|
267
277
|
features: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
268
278
|
}, z.core.$strip>;
|
|
269
279
|
smokeQueries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
@@ -287,6 +297,8 @@ declare const KnowledgeCatalogManifestSchema: z.ZodObject<{
|
|
|
287
297
|
}, z.core.$strip>>;
|
|
288
298
|
}, z.core.$strip>;
|
|
289
299
|
type KnowledgeCatalogManifest = z.infer<typeof KnowledgeCatalogManifestSchema>;
|
|
300
|
+
/** The (formatVersion, indexSchemaVersion) pairs the manifest schema admits. */
|
|
301
|
+
declare const KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS: [GezkIndexSchemaVersion, ...GezkIndexSchemaVersion[]];
|
|
290
302
|
/** `LICENSES/source-notices.json` — per-source attribution for combined catalogs. */
|
|
291
303
|
declare const SourceNoticesSchema: z.ZodObject<{
|
|
292
304
|
sources: z.ZodArray<z.ZodObject<{
|
|
@@ -300,6 +312,16 @@ declare const SourceNoticesSchema: z.ZodObject<{
|
|
|
300
312
|
}, z.core.$strip>;
|
|
301
313
|
type SourceNotices = z.infer<typeof SourceNoticesSchema>;
|
|
302
314
|
|
|
315
|
+
/** A signed 32-bit listing position; unordered documents sort after ordered ones. */
|
|
316
|
+
declare const KnowledgeOrdinalSchema: z.ZodNumber;
|
|
317
|
+
/**
|
|
318
|
+
* Producer-defined document metadata, opaque to readers. Stored canonically
|
|
319
|
+
* (RFC 8785) in `documents.meta_json`, bounded by
|
|
320
|
+
* `MAX_KNOWLEDGE_DOCUMENT_META_BYTES`. Keys beginning with `gezk.` are
|
|
321
|
+
* reserved for future versions of the format.
|
|
322
|
+
*/
|
|
323
|
+
declare const KnowledgeDocumentMetaSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
324
|
+
type KnowledgeDocumentMeta = z.infer<typeof KnowledgeDocumentMetaSchema>;
|
|
303
325
|
/** One normalized document streamed into the compiler. */
|
|
304
326
|
declare const CatalogDocumentSchema: z.ZodObject<{
|
|
305
327
|
id: z.ZodString;
|
|
@@ -314,6 +336,8 @@ declare const CatalogDocumentSchema: z.ZodObject<{
|
|
|
314
336
|
sourceUpdatedAt: z.ZodOptional<z.ZodString>;
|
|
315
337
|
attribution: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
316
338
|
aliases: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
339
|
+
ordinal: z.ZodOptional<z.ZodNumber>;
|
|
340
|
+
meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
317
341
|
}, z.core.$strip>;
|
|
318
342
|
type CatalogDocument = z.infer<typeof CatalogDocumentSchema>;
|
|
319
343
|
|
|
@@ -349,7 +373,10 @@ type KnowledgeRegistryEntry = z.infer<typeof KnowledgeRegistryEntrySchema>;
|
|
|
349
373
|
*/
|
|
350
374
|
declare const KnowledgeRegistryIndexSchema: z.ZodObject<{
|
|
351
375
|
kind: z.ZodLiteral<"gezk-registry">;
|
|
352
|
-
formatVersion: z.
|
|
376
|
+
formatVersion: z.ZodEnum<{
|
|
377
|
+
0.6: "0.6";
|
|
378
|
+
0.5: "0.5";
|
|
379
|
+
}>;
|
|
353
380
|
publisher: z.ZodObject<{
|
|
354
381
|
id: z.ZodString;
|
|
355
382
|
name: z.ZodString;
|
|
@@ -407,12 +434,29 @@ declare function parseKnowledgeUri(raw: string): KnowledgeUri | null;
|
|
|
407
434
|
|
|
408
435
|
/** Container-level constants of the `.gezk` format. */
|
|
409
436
|
/**
|
|
410
|
-
* Public format version. `0.x` is preliminary:
|
|
411
|
-
* any minor until 1.0, and a reader supports
|
|
437
|
+
* Public format version this implementation writes. `0.x` is preliminary:
|
|
438
|
+
* breaking changes may land in any minor until 1.0, and a reader supports
|
|
439
|
+
* exactly the versions it names (see {@link GEZK_FORMAT_GENERATIONS}).
|
|
412
440
|
*/
|
|
413
|
-
declare const GEZK_FORMAT_VERSION = "0.
|
|
414
|
-
/** SQLite schema generation (`PRAGMA user_version`
|
|
415
|
-
declare const GEZK_INDEX_SCHEMA_VERSION =
|
|
441
|
+
declare const GEZK_FORMAT_VERSION = "0.6";
|
|
442
|
+
/** SQLite schema generation this implementation writes (`PRAGMA user_version`). */
|
|
443
|
+
declare const GEZK_INDEX_SCHEMA_VERSION = 3;
|
|
444
|
+
/**
|
|
445
|
+
* Every (formatVersion, indexSchemaVersion) pairing this implementation
|
|
446
|
+
* reads. The writer always emits the current pair; older generations stay
|
|
447
|
+
* readable so a catalog published under an earlier 0.x keeps opening after
|
|
448
|
+
* a reader upgrade. A manifest that pairs the two differently is corrupt.
|
|
449
|
+
*/
|
|
450
|
+
declare const GEZK_FORMAT_GENERATIONS: {
|
|
451
|
+
readonly '0.5': 2;
|
|
452
|
+
readonly '0.6': 3;
|
|
453
|
+
};
|
|
454
|
+
type GezkFormatVersion = keyof typeof GEZK_FORMAT_GENERATIONS;
|
|
455
|
+
type GezkIndexSchemaVersion = (typeof GEZK_FORMAT_GENERATIONS)[GezkFormatVersion];
|
|
456
|
+
declare const GEZK_SUPPORTED_FORMAT_VERSIONS: [GezkFormatVersion, ...GezkFormatVersion[]];
|
|
457
|
+
declare const GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS: [GezkIndexSchemaVersion, ...GezkIndexSchemaVersion[]];
|
|
458
|
+
declare function isSupportedFormatVersion(value: unknown): value is GezkFormatVersion;
|
|
459
|
+
declare function isSupportedIndexSchemaVersion(value: unknown): value is GezkIndexSchemaVersion;
|
|
416
460
|
/** `PRAGMA application_id` for every database in a catalog: 'GEZK'. */
|
|
417
461
|
declare const GEZK_APPLICATION_ID = 1195727435;
|
|
418
462
|
/**
|
|
@@ -431,15 +475,19 @@ declare const ROUTER_DB_PATH = "index/router.db";
|
|
|
431
475
|
declare const BODY_CODEC_MIN_BYTES = 512;
|
|
432
476
|
/** Maximum decompressed Markdown body returned from one catalog document. */
|
|
433
477
|
declare const MAX_KNOWLEDGE_DOCUMENT_BYTES: number;
|
|
478
|
+
/** Maximum canonical UTF-8 size of one document's `meta_json` object. */
|
|
479
|
+
declare const MAX_KNOWLEDGE_DOCUMENT_META_BYTES: number;
|
|
480
|
+
/** Maximum depth of the topic forest (root = 1). */
|
|
481
|
+
declare const MAX_KNOWLEDGE_TOPIC_DEPTH = 16;
|
|
434
482
|
/** Fixed ZIP entry timestamp for deterministic archives. */
|
|
435
483
|
declare const ZIP_FIXED_MTIME: Date;
|
|
436
484
|
|
|
437
485
|
/**
|
|
438
|
-
* Index schema
|
|
486
|
+
* Index schema 3 DDL. Readers never migrate these. Every table is plain
|
|
439
487
|
* SQLite (FTS5 is the only virtual-table module used), so any SQLite client
|
|
440
488
|
* can read a catalog without extensions.
|
|
441
489
|
*/
|
|
442
|
-
declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
|
|
490
|
+
declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n ordinal INTEGER,\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n meta_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, ordinal, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
|
|
443
491
|
/**
|
|
444
492
|
* Per-shard tables. Vectors are plain BLOB columns keyed by chunk id:
|
|
445
493
|
* `chunk_vectors_bit.v` holds ceil(dim/8) bytes of sign bits (LSB-first) and
|
|
@@ -451,6 +499,68 @@ declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT
|
|
|
451
499
|
*/
|
|
452
500
|
declare const SHARD_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE chunks(\n id INTEGER PRIMARY KEY,\n chunk_uid TEXT NOT NULL,\n document_id TEXT NOT NULL,\n ordinal INTEGER NOT NULL,\n title TEXT NOT NULL,\n heading_path TEXT NOT NULL,\n heading_text TEXT NOT NULL,\n line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,\n token_count INTEGER NOT NULL,\n content_hash TEXT NOT NULL,\n text TEXT NOT NULL\n);\nCREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);\nCREATE INDEX chunks_document ON chunks(document_id, ordinal);\n\nCREATE VIRTUAL TABLE fts_chunks USING fts5(\n title, heading_text, text,\n content = 'chunks', content_rowid = 'id',\n tokenize = 'unicode61 remove_diacritics 2'\n);\n\nCREATE TABLE chunk_vectors_bit(\n chunk_id INTEGER PRIMARY KEY,\n v BLOB NOT NULL\n);\n\nCREATE TABLE chunk_vectors_int8(\n chunk_id INTEGER PRIMARY KEY,\n v BLOB NOT NULL\n);\n";
|
|
453
501
|
|
|
502
|
+
/**
|
|
503
|
+
* Assets: images a catalog ships under `assets/` for its document bodies.
|
|
504
|
+
* A document references one by its archive path (``),
|
|
505
|
+
* and a reader resolves that against the catalog, never the network. The
|
|
506
|
+
* rules here are the format's: which files may be assets, how large, and
|
|
507
|
+
* the inertness an SVG must prove before a reader serves it — an SVG is a
|
|
508
|
+
* document, not a bitmap, so the format refuses anything that could run.
|
|
509
|
+
*/
|
|
510
|
+
|
|
511
|
+
declare const ASSETS_PREFIX = "assets/";
|
|
512
|
+
declare const KNOWLEDGE_ASSET_TYPES: {
|
|
513
|
+
readonly png: "image/png";
|
|
514
|
+
readonly jpg: "image/jpeg";
|
|
515
|
+
readonly jpeg: "image/jpeg";
|
|
516
|
+
readonly gif: "image/gif";
|
|
517
|
+
readonly webp: "image/webp";
|
|
518
|
+
readonly svg: "image/svg+xml";
|
|
519
|
+
};
|
|
520
|
+
type KnowledgeAssetExtension = keyof typeof KNOWLEDGE_ASSET_TYPES;
|
|
521
|
+
type KnowledgeAssetKind = 'png' | 'jpeg' | 'gif' | 'webp' | 'svg';
|
|
522
|
+
declare const MAX_KNOWLEDGE_ASSET_BYTES: number;
|
|
523
|
+
declare const MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES: number;
|
|
524
|
+
declare const MAX_KNOWLEDGE_ASSET_COUNT = 8192;
|
|
525
|
+
declare const MAX_KNOWLEDGE_ASSET_PATH_LENGTH = 512;
|
|
526
|
+
/**
|
|
527
|
+
* `assets/(<dir>/)*<name>.<ext>` — up to 15 directory segments, each
|
|
528
|
+
* segment `[A-Za-z0-9._-]` and never starting with a dot (so `.`, `..` and
|
|
529
|
+
* hidden files are impossible by construction), extension from the table.
|
|
530
|
+
*/
|
|
531
|
+
declare const KNOWLEDGE_ASSET_PATH_PATTERN: RegExp;
|
|
532
|
+
declare const KnowledgeAssetPathSchema: z.ZodString;
|
|
533
|
+
declare function isKnowledgeAssetPath(path: string): boolean;
|
|
534
|
+
declare function assetExtension(path: string): KnowledgeAssetExtension | null;
|
|
535
|
+
/** The media type an asset path implies, from its extension. */
|
|
536
|
+
declare function assetContentType(path: string): string | null;
|
|
537
|
+
/** The kind an extension declares, normalized (`jpg` and `jpeg` are one kind). */
|
|
538
|
+
declare function assetKindForExtension(ext: KnowledgeAssetExtension): KnowledgeAssetKind;
|
|
539
|
+
/**
|
|
540
|
+
* What the leading bytes say the file is. Rasters are recognized by their
|
|
541
|
+
* magic numbers; an SVG is UTF-8 text whose first element, after any BOM,
|
|
542
|
+
* whitespace, XML declaration, comments and DOCTYPE, is `<svg`.
|
|
543
|
+
*/
|
|
544
|
+
declare function sniffAssetType(bytes: Uint8Array): KnowledgeAssetKind | null;
|
|
545
|
+
/**
|
|
546
|
+
* Why an SVG is not inert, or `null` when it is. Scripts, event handlers,
|
|
547
|
+
* foreign content, entity declarations and any reference that leaves the
|
|
548
|
+
* file (other than embedded raster data) all disqualify it: a reader may
|
|
549
|
+
* hand the bytes to an image element, but it may also let a person open
|
|
550
|
+
* them directly, and an SVG that can run is a page, not a picture.
|
|
551
|
+
*/
|
|
552
|
+
declare function svgInertnessProblem(bytes: Uint8Array): string | null;
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* Topic ordering keys. `topics.sort_key` is a TEXT column compared bytewise
|
|
556
|
+
* by every reader (`ORDER BY sort_key, id`), so a producer that orders topics
|
|
557
|
+
* numerically must encode the number so that byte order equals numeric
|
|
558
|
+
* order. Offsetting by 2^31 makes every int32 non-negative and ten digits
|
|
559
|
+
* wide; zero-padding then makes the string comparison agree with the
|
|
560
|
+
* integer comparison, negatives included.
|
|
561
|
+
*/
|
|
562
|
+
declare function topicSortKeyForOrder(order: number): string;
|
|
563
|
+
|
|
454
564
|
/**
|
|
455
565
|
* Frozen quantization formulas (the gezk spec §6.2). Deliberately computed
|
|
456
566
|
* here in TypeScript rather than via sqlite-vec's `vec_quantize_*` scalar
|
|
@@ -489,4 +599,4 @@ declare function canonicalizeJson(value: unknown): string;
|
|
|
489
599
|
/** URL-ish slug for a document title (display/dedup aid, not an identity). */
|
|
490
600
|
declare function documentSlug(title: string): string;
|
|
491
601
|
|
|
492
|
-
export { ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, type CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_VERSION_PATTERN, type KnowledgeCatalogManifest, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, type KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, type KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, type KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, type KnowledgeRegistryIndex, KnowledgeRegistryIndexSchema, type KnowledgeSignature, KnowledgeSignatureSchema, type KnowledgeUri, type KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, type SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace };
|
|
602
|
+
export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, type CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, type GezkFormatVersion, type GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, type KnowledgeAssetExtension, type KnowledgeAssetKind, KnowledgeAssetPathSchema, type KnowledgeCatalogManifest, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, type KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, type KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, type KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, type KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, type KnowledgeRegistryIndex, KnowledgeRegistryIndexSchema, type KnowledgeSignature, KnowledgeSignatureSchema, type KnowledgeUri, type KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, type SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder };
|
package/dist/index.js
CHANGED
|
@@ -104,8 +104,17 @@ var KnowledgeChunkingProfileSchema = z2.object({
|
|
|
104
104
|
import { z as z3 } from "zod";
|
|
105
105
|
|
|
106
106
|
// src/format/constants.ts
|
|
107
|
-
var GEZK_FORMAT_VERSION = "0.
|
|
108
|
-
var GEZK_INDEX_SCHEMA_VERSION =
|
|
107
|
+
var GEZK_FORMAT_VERSION = "0.6";
|
|
108
|
+
var GEZK_INDEX_SCHEMA_VERSION = 3;
|
|
109
|
+
var GEZK_FORMAT_GENERATIONS = { "0.5": 2, "0.6": 3 };
|
|
110
|
+
var GEZK_SUPPORTED_FORMAT_VERSIONS = Object.keys(GEZK_FORMAT_GENERATIONS);
|
|
111
|
+
var GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS = Object.values(GEZK_FORMAT_GENERATIONS);
|
|
112
|
+
function isSupportedFormatVersion(value) {
|
|
113
|
+
return typeof value === "string" && Object.hasOwn(GEZK_FORMAT_GENERATIONS, value);
|
|
114
|
+
}
|
|
115
|
+
function isSupportedIndexSchemaVersion(value) {
|
|
116
|
+
return typeof value === "number" && GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.includes(value);
|
|
117
|
+
}
|
|
109
118
|
var GEZK_APPLICATION_ID = 1195727435;
|
|
110
119
|
var GEZK_MIME_TYPE = "application/vnd.gezk+zip";
|
|
111
120
|
var MIMETYPE_PATH = "mimetype";
|
|
@@ -116,6 +125,8 @@ var SOURCE_NOTICES_PATH = "LICENSES/source-notices.json";
|
|
|
116
125
|
var ROUTER_DB_PATH = "index/router.db";
|
|
117
126
|
var BODY_CODEC_MIN_BYTES = 512;
|
|
118
127
|
var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
|
|
128
|
+
var MAX_KNOWLEDGE_DOCUMENT_META_BYTES = 16 * 1024;
|
|
129
|
+
var MAX_KNOWLEDGE_TOPIC_DEPTH = 16;
|
|
119
130
|
var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
|
|
120
131
|
|
|
121
132
|
// src/schemas/manifest.ts
|
|
@@ -134,8 +145,8 @@ var KnowledgeSignatureSchema = z3.object({
|
|
|
134
145
|
});
|
|
135
146
|
var KnowledgeCatalogManifestSchema = z3.object({
|
|
136
147
|
kind: z3.literal(GEZK_MANIFEST_KIND),
|
|
137
|
-
formatVersion: z3.
|
|
138
|
-
indexSchemaVersion: z3.literal(
|
|
148
|
+
formatVersion: z3.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
149
|
+
indexSchemaVersion: z3.union([z3.literal(2), z3.literal(3)]),
|
|
139
150
|
id: KnowledgeIdSchema,
|
|
140
151
|
version: KnowledgeVersionSchema,
|
|
141
152
|
name: z3.string().min(1),
|
|
@@ -162,7 +173,17 @@ var KnowledgeCatalogManifestSchema = z3.object({
|
|
|
162
173
|
}),
|
|
163
174
|
embedding: KnowledgeEmbeddingProfileSchema,
|
|
164
175
|
chunking: KnowledgeChunkingProfileSchema,
|
|
165
|
-
topics: z3.array(
|
|
176
|
+
topics: z3.array(
|
|
177
|
+
z3.object({
|
|
178
|
+
id: KnowledgeIdSchema,
|
|
179
|
+
name: z3.string().min(1),
|
|
180
|
+
/** Absent for a root topic; names another entry of this list. */
|
|
181
|
+
parentId: KnowledgeIdSchema.optional(),
|
|
182
|
+
/** Sibling ordering key, identical to `topics.sort_key` in the router. */
|
|
183
|
+
sortKey: z3.string().min(1).optional(),
|
|
184
|
+
description: z3.string().optional()
|
|
185
|
+
})
|
|
186
|
+
).min(1, "a catalog must ship a table of contents (at least one topic)"),
|
|
166
187
|
router: z3.object({
|
|
167
188
|
shardTargetChunks: z3.number().int().positive(),
|
|
168
189
|
shards: z3.array(
|
|
@@ -180,12 +201,14 @@ var KnowledgeCatalogManifestSchema = z3.object({
|
|
|
180
201
|
counts: z3.object({
|
|
181
202
|
documents: z3.number().int().nonnegative(),
|
|
182
203
|
chunks: z3.number().int().nonnegative(),
|
|
183
|
-
shards: z3.number().int().positive()
|
|
204
|
+
shards: z3.number().int().positive(),
|
|
205
|
+
/** Files under `assets/` (0.6+). */
|
|
206
|
+
assets: z3.number().int().nonnegative().optional()
|
|
184
207
|
}),
|
|
185
208
|
files: z3.array(KnowledgeManifestFileSchema).min(1),
|
|
186
209
|
/** What a reader must implement to open this catalog. */
|
|
187
210
|
requires: z3.object({
|
|
188
|
-
formatVersion: z3.
|
|
211
|
+
formatVersion: z3.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
189
212
|
features: z3.array(z3.string()).optional()
|
|
190
213
|
}),
|
|
191
214
|
smokeQueries: z3.array(
|
|
@@ -205,7 +228,50 @@ var KnowledgeCatalogManifestSchema = z3.object({
|
|
|
205
228
|
tokenizerDigest: z3.string().optional()
|
|
206
229
|
}).optional(),
|
|
207
230
|
signature: KnowledgeSignatureSchema.optional()
|
|
231
|
+
}).superRefine((manifest, ctx) => {
|
|
232
|
+
if (GEZK_FORMAT_GENERATIONS[manifest.formatVersion] !== manifest.indexSchemaVersion) {
|
|
233
|
+
ctx.addIssue({
|
|
234
|
+
code: "custom",
|
|
235
|
+
path: ["indexSchemaVersion"],
|
|
236
|
+
message: `format ${manifest.formatVersion} pairs with index schema ${GEZK_FORMAT_GENERATIONS[manifest.formatVersion]}, not ${manifest.indexSchemaVersion}`
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
if (manifest.requires.formatVersion !== manifest.formatVersion) {
|
|
240
|
+
ctx.addIssue({
|
|
241
|
+
code: "custom",
|
|
242
|
+
path: ["requires", "formatVersion"],
|
|
243
|
+
message: "requires.formatVersion must equal formatVersion"
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
if (manifest.formatVersion === "0.5" && manifest.counts.assets !== void 0) {
|
|
247
|
+
ctx.addIssue({
|
|
248
|
+
code: "custom",
|
|
249
|
+
path: ["counts", "assets"],
|
|
250
|
+
message: "counts.assets is a 0.6 field"
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
const topicIds = /* @__PURE__ */ new Set();
|
|
254
|
+
manifest.topics.forEach((topic, index) => {
|
|
255
|
+
if (topicIds.has(topic.id)) {
|
|
256
|
+
ctx.addIssue({
|
|
257
|
+
code: "custom",
|
|
258
|
+
path: ["topics", index, "id"],
|
|
259
|
+
message: `duplicate topic id '${topic.id}'`
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
topicIds.add(topic.id);
|
|
263
|
+
});
|
|
264
|
+
manifest.topics.forEach((topic, index) => {
|
|
265
|
+
if (topic.parentId !== void 0 && !topicIds.has(topic.parentId)) {
|
|
266
|
+
ctx.addIssue({
|
|
267
|
+
code: "custom",
|
|
268
|
+
path: ["topics", index, "parentId"],
|
|
269
|
+
message: `topic '${topic.id}' names an undeclared parent '${topic.parentId}'`
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
});
|
|
208
273
|
});
|
|
274
|
+
var KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS = GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS;
|
|
209
275
|
var SourceNoticesSchema = z3.object({
|
|
210
276
|
sources: z3.array(
|
|
211
277
|
z3.object({
|
|
@@ -223,20 +289,28 @@ var SourceNoticesSchema = z3.object({
|
|
|
223
289
|
|
|
224
290
|
// src/schemas/document.ts
|
|
225
291
|
import { z as z4 } from "zod";
|
|
292
|
+
var KnowledgeOrdinalSchema = z4.number().int().min(-2147483648).max(2147483647);
|
|
293
|
+
var KnowledgeDocumentMetaSchema = z4.record(z4.string(), z4.unknown());
|
|
226
294
|
var CatalogDocumentSchema = z4.object({
|
|
227
295
|
id: KnowledgeDocumentIdSchema,
|
|
228
296
|
title: z4.string().min(1),
|
|
229
297
|
slug: z4.string().min(1),
|
|
230
298
|
summary: z4.string().optional(),
|
|
231
299
|
language: z4.string().min(2),
|
|
232
|
-
/**
|
|
300
|
+
/**
|
|
301
|
+
* Root→leaf topic id path. Every segment must be a declared topic and each
|
|
302
|
+
* must be the parent of the next; the document is filed at the last one.
|
|
303
|
+
*/
|
|
233
304
|
topicPath: z4.array(KnowledgeIdSchema).min(1),
|
|
234
305
|
markdown: z4.string(),
|
|
235
306
|
sourceUrl: z4.string().optional(),
|
|
236
307
|
sourceRevision: z4.string().optional(),
|
|
237
308
|
sourceUpdatedAt: z4.string().optional(),
|
|
238
309
|
attribution: z4.record(z4.string(), z4.string()).optional(),
|
|
239
|
-
aliases: z4.array(z4.string()).optional()
|
|
310
|
+
aliases: z4.array(z4.string()).optional(),
|
|
311
|
+
/** Explicit position among the documents of its topic in a listing. */
|
|
312
|
+
ordinal: KnowledgeOrdinalSchema.optional(),
|
|
313
|
+
meta: KnowledgeDocumentMetaSchema.optional()
|
|
240
314
|
});
|
|
241
315
|
|
|
242
316
|
// src/schemas/registry.ts
|
|
@@ -259,7 +333,7 @@ var KnowledgeRegistryEntrySchema = z5.object({
|
|
|
259
333
|
});
|
|
260
334
|
var KnowledgeRegistryIndexSchema = z5.object({
|
|
261
335
|
kind: z5.literal(GEZK_REGISTRY_KIND),
|
|
262
|
-
formatVersion: z5.
|
|
336
|
+
formatVersion: z5.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
263
337
|
publisher: z5.object({
|
|
264
338
|
id: KnowledgeIdSchema,
|
|
265
339
|
name: z5.string().min(1),
|
|
@@ -338,14 +412,16 @@ CREATE TABLE documents(
|
|
|
338
412
|
id TEXT PRIMARY KEY,
|
|
339
413
|
title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
|
|
340
414
|
language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
|
|
415
|
+
ordinal INTEGER,
|
|
341
416
|
shard_id INTEGER NOT NULL,
|
|
342
417
|
chunk_count INTEGER NOT NULL,
|
|
343
418
|
source_url TEXT, source_revision TEXT, source_updated_at TEXT,
|
|
344
419
|
attribution_json TEXT,
|
|
420
|
+
meta_json TEXT,
|
|
345
421
|
body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
|
|
346
422
|
body_blob BLOB NOT NULL
|
|
347
423
|
);
|
|
348
|
-
CREATE INDEX documents_topic ON documents(topic_id, slug);
|
|
424
|
+
CREATE INDEX documents_topic ON documents(topic_id, ordinal, slug);
|
|
349
425
|
CREATE INDEX documents_shard ON documents(shard_id);
|
|
350
426
|
|
|
351
427
|
CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
|
|
@@ -408,6 +484,152 @@ CREATE TABLE chunk_vectors_int8(
|
|
|
408
484
|
);
|
|
409
485
|
`;
|
|
410
486
|
|
|
487
|
+
// src/format/assets.ts
|
|
488
|
+
import { z as z6 } from "zod";
|
|
489
|
+
var ASSETS_PREFIX = "assets/";
|
|
490
|
+
var KNOWLEDGE_ASSET_TYPES = {
|
|
491
|
+
png: "image/png",
|
|
492
|
+
jpg: "image/jpeg",
|
|
493
|
+
jpeg: "image/jpeg",
|
|
494
|
+
gif: "image/gif",
|
|
495
|
+
webp: "image/webp",
|
|
496
|
+
svg: "image/svg+xml"
|
|
497
|
+
};
|
|
498
|
+
var MAX_KNOWLEDGE_ASSET_BYTES = 8 * 1024 * 1024;
|
|
499
|
+
var MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES = 256 * 1024 * 1024;
|
|
500
|
+
var MAX_KNOWLEDGE_ASSET_COUNT = 8192;
|
|
501
|
+
var MAX_KNOWLEDGE_ASSET_PATH_LENGTH = 512;
|
|
502
|
+
var KNOWLEDGE_ASSET_PATH_PATTERN = /^assets\/(?:[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\/){0,15}[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\.(?:png|jpe?g|gif|webp|svg)$/i;
|
|
503
|
+
var KnowledgeAssetPathSchema = z6.string().max(MAX_KNOWLEDGE_ASSET_PATH_LENGTH).regex(KNOWLEDGE_ASSET_PATH_PATTERN, "not a valid asset path");
|
|
504
|
+
function isKnowledgeAssetPath(path) {
|
|
505
|
+
return path.length <= MAX_KNOWLEDGE_ASSET_PATH_LENGTH && KNOWLEDGE_ASSET_PATH_PATTERN.test(path);
|
|
506
|
+
}
|
|
507
|
+
function assetExtension(path) {
|
|
508
|
+
const dot = path.lastIndexOf(".");
|
|
509
|
+
if (dot < 0) return null;
|
|
510
|
+
const ext = path.slice(dot + 1).toLowerCase();
|
|
511
|
+
return Object.hasOwn(KNOWLEDGE_ASSET_TYPES, ext) ? ext : null;
|
|
512
|
+
}
|
|
513
|
+
function assetContentType(path) {
|
|
514
|
+
const ext = assetExtension(path);
|
|
515
|
+
return ext ? KNOWLEDGE_ASSET_TYPES[ext] : null;
|
|
516
|
+
}
|
|
517
|
+
function assetKindForExtension(ext) {
|
|
518
|
+
return ext === "jpg" ? "jpeg" : ext;
|
|
519
|
+
}
|
|
520
|
+
var PNG_MAGIC = [137, 80, 78, 71, 13, 10, 26, 10];
|
|
521
|
+
var JPEG_MAGIC = [255, 216, 255];
|
|
522
|
+
var GIF87 = [71, 73, 70, 56, 55, 97];
|
|
523
|
+
var GIF89 = [71, 73, 70, 56, 57, 97];
|
|
524
|
+
var RIFF = [82, 73, 70, 70];
|
|
525
|
+
var WEBP = [87, 69, 66, 80];
|
|
526
|
+
function startsWith(bytes, magic, offset = 0) {
|
|
527
|
+
if (bytes.length < offset + magic.length) return false;
|
|
528
|
+
for (let i = 0; i < magic.length; i++) {
|
|
529
|
+
if (bytes[offset + i] !== magic[i]) return false;
|
|
530
|
+
}
|
|
531
|
+
return true;
|
|
532
|
+
}
|
|
533
|
+
function sniffAssetType(bytes) {
|
|
534
|
+
if (startsWith(bytes, PNG_MAGIC)) return "png";
|
|
535
|
+
if (startsWith(bytes, JPEG_MAGIC)) return "jpeg";
|
|
536
|
+
if (startsWith(bytes, GIF87) || startsWith(bytes, GIF89)) return "gif";
|
|
537
|
+
if (startsWith(bytes, RIFF) && startsWith(bytes, WEBP, 8)) return "webp";
|
|
538
|
+
return looksLikeSvg(decodeText(bytes)) ? "svg" : null;
|
|
539
|
+
}
|
|
540
|
+
function decodeText(bytes) {
|
|
541
|
+
try {
|
|
542
|
+
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes);
|
|
543
|
+
} catch {
|
|
544
|
+
return "";
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
function looksLikeSvg(text) {
|
|
548
|
+
let i = 0;
|
|
549
|
+
const n = text.length;
|
|
550
|
+
while (i < n) {
|
|
551
|
+
const ch = text.charCodeAt(i);
|
|
552
|
+
if (ch === 32 || ch === 9 || ch === 10 || ch === 13 || ch === 65279) {
|
|
553
|
+
i++;
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
556
|
+
if (text.startsWith("<?", i)) {
|
|
557
|
+
const end = text.indexOf("?>", i);
|
|
558
|
+
if (end < 0) return false;
|
|
559
|
+
i = end + 2;
|
|
560
|
+
continue;
|
|
561
|
+
}
|
|
562
|
+
if (text.startsWith("<!--", i)) {
|
|
563
|
+
const end = text.indexOf("-->", i);
|
|
564
|
+
if (end < 0) return false;
|
|
565
|
+
i = end + 3;
|
|
566
|
+
continue;
|
|
567
|
+
}
|
|
568
|
+
if (/^<!doctype/i.test(text.slice(i, i + 9))) {
|
|
569
|
+
const bracket = text.indexOf("[", i);
|
|
570
|
+
const close = text.indexOf(">", i);
|
|
571
|
+
if (close < 0) return false;
|
|
572
|
+
if (bracket >= 0 && bracket < close) {
|
|
573
|
+
const end = text.indexOf("]>", bracket);
|
|
574
|
+
if (end < 0) return false;
|
|
575
|
+
i = end + 2;
|
|
576
|
+
} else {
|
|
577
|
+
i = close + 1;
|
|
578
|
+
}
|
|
579
|
+
continue;
|
|
580
|
+
}
|
|
581
|
+
return /^<svg[\s>/]/i.test(text.slice(i, i + 5));
|
|
582
|
+
}
|
|
583
|
+
return false;
|
|
584
|
+
}
|
|
585
|
+
var SVG_ACTIVE_PATTERNS = [
|
|
586
|
+
[/<script[\s>/]/i, "a <script> element"],
|
|
587
|
+
[/<foreignobject[\s>/]/i, "a <foreignObject> element"],
|
|
588
|
+
[/<!entity/i, "an entity declaration"],
|
|
589
|
+
[/\son[a-z]+\s*=/i, "an event-handler attribute"],
|
|
590
|
+
[/javascript\s*:/i, "a javascript: reference"],
|
|
591
|
+
[/data\s*:\s*text\/html/i, "a data:text/html reference"],
|
|
592
|
+
[/@import/i, "a CSS @import"]
|
|
593
|
+
];
|
|
594
|
+
var HREF_ATTRIBUTE = /(?:xlink:)?href\s*=\s*["']\s*([^"']*)["']/gi;
|
|
595
|
+
var CSS_URL = /url\(\s*["']?\s*([^"')]*)/gi;
|
|
596
|
+
function externalReferenceProblem(target) {
|
|
597
|
+
const value = target.trim();
|
|
598
|
+
if (value === "" || value.startsWith("#")) return null;
|
|
599
|
+
const scheme = /^([a-z][a-z0-9+.-]*):/i.exec(value);
|
|
600
|
+
if (!scheme) return null;
|
|
601
|
+
if (/^data:image\/(?:png|jpeg|gif|webp)[;,]/i.test(value)) return null;
|
|
602
|
+
return `a ${scheme[1]}: reference`;
|
|
603
|
+
}
|
|
604
|
+
function svgInertnessProblem(bytes) {
|
|
605
|
+
const text = decodeText(bytes);
|
|
606
|
+
if (text === "") return "not valid UTF-8";
|
|
607
|
+
if (!looksLikeSvg(text)) return "does not start with an <svg> element";
|
|
608
|
+
for (const [pattern, why] of SVG_ACTIVE_PATTERNS) {
|
|
609
|
+
if (pattern.test(text)) return `contains ${why}`;
|
|
610
|
+
}
|
|
611
|
+
for (const match of text.matchAll(HREF_ATTRIBUTE)) {
|
|
612
|
+
const problem = externalReferenceProblem(match[1] ?? "");
|
|
613
|
+
if (problem) return `contains ${problem} in an href`;
|
|
614
|
+
}
|
|
615
|
+
for (const match of text.matchAll(CSS_URL)) {
|
|
616
|
+
const problem = externalReferenceProblem(match[1] ?? "");
|
|
617
|
+
if (problem) return `contains ${problem} in a CSS url()`;
|
|
618
|
+
}
|
|
619
|
+
return null;
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
// src/format/sort-key.ts
|
|
623
|
+
var INT32_MIN = -2147483648;
|
|
624
|
+
var INT32_MAX = 2147483647;
|
|
625
|
+
var OFFSET = 2147483648;
|
|
626
|
+
function topicSortKeyForOrder(order) {
|
|
627
|
+
if (!Number.isInteger(order) || order < INT32_MIN || order > INT32_MAX) {
|
|
628
|
+
throw new RangeError(`topic order must be an int32: ${String(order)}`);
|
|
629
|
+
}
|
|
630
|
+
return String(order + OFFSET).padStart(10, "0");
|
|
631
|
+
}
|
|
632
|
+
|
|
411
633
|
// src/format/quantize.ts
|
|
412
634
|
function quantizeInt8(unitVector) {
|
|
413
635
|
const out = new Int8Array(unitVector.length);
|
|
@@ -497,26 +719,36 @@ function documentSlug(title) {
|
|
|
497
719
|
return slug || "untitled";
|
|
498
720
|
}
|
|
499
721
|
export {
|
|
722
|
+
ASSETS_PREFIX,
|
|
500
723
|
ArtifactDigestSchema,
|
|
501
724
|
BODY_CODEC_MIN_BYTES,
|
|
502
725
|
CatalogDocumentSchema,
|
|
503
726
|
DEFAULT_EMBEDDING_ONNX_FILE,
|
|
504
727
|
DEFAULT_EMBEDDING_TOKENIZER_FILE,
|
|
505
728
|
GEZK_APPLICATION_ID,
|
|
729
|
+
GEZK_FORMAT_GENERATIONS,
|
|
506
730
|
GEZK_FORMAT_VERSION,
|
|
507
731
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
508
732
|
GEZK_MANIFEST_KIND,
|
|
509
733
|
GEZK_MIME_TYPE,
|
|
510
734
|
GEZK_REGISTRY_KIND,
|
|
735
|
+
GEZK_SUPPORTED_FORMAT_VERSIONS,
|
|
736
|
+
GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
|
|
737
|
+
KNOWLEDGE_ASSET_PATH_PATTERN,
|
|
738
|
+
KNOWLEDGE_ASSET_TYPES,
|
|
511
739
|
KNOWLEDGE_ID_PATTERN,
|
|
740
|
+
KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS,
|
|
512
741
|
KNOWLEDGE_VERSION_PATTERN,
|
|
742
|
+
KnowledgeAssetPathSchema,
|
|
513
743
|
KnowledgeCatalogManifestSchema,
|
|
514
744
|
KnowledgeChunkUidSchema,
|
|
515
745
|
KnowledgeChunkingProfileSchema,
|
|
516
746
|
KnowledgeDocumentIdSchema,
|
|
747
|
+
KnowledgeDocumentMetaSchema,
|
|
517
748
|
KnowledgeEmbeddingProfileSchema,
|
|
518
749
|
KnowledgeIdSchema,
|
|
519
750
|
KnowledgeManifestFileSchema,
|
|
751
|
+
KnowledgeOrdinalSchema,
|
|
520
752
|
KnowledgeRegistryEntrySchema,
|
|
521
753
|
KnowledgeRegistryIndexSchema,
|
|
522
754
|
KnowledgeSignatureSchema,
|
|
@@ -524,7 +756,13 @@ export {
|
|
|
524
756
|
KnowledgeVersionSchema,
|
|
525
757
|
LICENSE_NOTICE_PATH,
|
|
526
758
|
MANIFEST_PATH,
|
|
759
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
|
|
760
|
+
MAX_KNOWLEDGE_ASSET_BYTES,
|
|
761
|
+
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
762
|
+
MAX_KNOWLEDGE_ASSET_PATH_LENGTH,
|
|
527
763
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
764
|
+
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
765
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
528
766
|
MIMETYPE_PATH,
|
|
529
767
|
README_PATH,
|
|
530
768
|
ROUTER_DB_PATH,
|
|
@@ -535,14 +773,23 @@ export {
|
|
|
535
773
|
Sha256HexSchema,
|
|
536
774
|
SourceNoticesSchema,
|
|
537
775
|
ZIP_FIXED_MTIME,
|
|
776
|
+
assetContentType,
|
|
777
|
+
assetExtension,
|
|
778
|
+
assetKindForExtension,
|
|
538
779
|
canonicalizeJson,
|
|
539
780
|
documentSlug,
|
|
540
781
|
embeddingProfileArtifacts,
|
|
541
782
|
formatKnowledgeUri,
|
|
783
|
+
isKnowledgeAssetPath,
|
|
784
|
+
isSupportedFormatVersion,
|
|
785
|
+
isSupportedIndexSchemaVersion,
|
|
542
786
|
l2Normalize,
|
|
543
787
|
parseKnowledgeUri,
|
|
544
788
|
quantizeBinary,
|
|
545
789
|
quantizeInt8,
|
|
546
790
|
rerankScore,
|
|
547
|
-
sameVectorSpace
|
|
791
|
+
sameVectorSpace,
|
|
792
|
+
sniffAssetType,
|
|
793
|
+
svgInertnessProblem,
|
|
794
|
+
topicSortKeyForOrder
|
|
548
795
|
};
|
package/dist/node.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { KnowledgeCatalogManifest, KnowledgeRegistryIndex } from './index.js';
|
|
2
|
-
export { ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_VERSION_PATTERN, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, KnowledgeRegistryIndexSchema, KnowledgeSignature, KnowledgeSignatureSchema, KnowledgeUri, KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace } from './index.js';
|
|
2
|
+
export { ASSETS_PREFIX, ArtifactDigestSchema, BODY_CODEC_MIN_BYTES, CatalogDocument, CatalogDocumentSchema, DEFAULT_EMBEDDING_ONNX_FILE, DEFAULT_EMBEDDING_TOKENIZER_FILE, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MANIFEST_KIND, GEZK_MIME_TYPE, GEZK_REGISTRY_KIND, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KNOWLEDGE_ASSET_PATH_PATTERN, KNOWLEDGE_ASSET_TYPES, KNOWLEDGE_ID_PATTERN, KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS, KNOWLEDGE_VERSION_PATTERN, KnowledgeAssetExtension, KnowledgeAssetKind, KnowledgeAssetPathSchema, KnowledgeCatalogManifestSchema, KnowledgeChunkUidSchema, KnowledgeChunkingProfile, KnowledgeChunkingProfileSchema, KnowledgeDocumentIdSchema, KnowledgeDocumentMeta, KnowledgeDocumentMetaSchema, KnowledgeEmbeddingProfile, KnowledgeEmbeddingProfileSchema, KnowledgeIdSchema, KnowledgeManifestFileSchema, KnowledgeOrdinalSchema, KnowledgeRegistryEntry, KnowledgeRegistryEntrySchema, KnowledgeRegistryIndexSchema, KnowledgeSignature, KnowledgeSignatureSchema, KnowledgeUri, KnowledgeVectorEncoding, KnowledgeVectorEncodingSchema, KnowledgeVersionSchema, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES, MAX_KNOWLEDGE_ASSET_BYTES, MAX_KNOWLEDGE_ASSET_COUNT, MAX_KNOWLEDGE_ASSET_PATH_LENGTH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, RepoRelativePathSchema, SHARD_DDL, SOURCE_NOTICES_PATH, Sha256HexSchema, SourceNotices, SourceNoticesSchema, ZIP_FIXED_MTIME, assetContentType, assetExtension, assetKindForExtension, canonicalizeJson, documentSlug, embeddingProfileArtifacts, formatKnowledgeUri, isKnowledgeAssetPath, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, parseKnowledgeUri, quantizeBinary, quantizeInt8, rerankScore, sameVectorSpace, sniffAssetType, svgInertnessProblem, topicSortKeyForOrder } from './index.js';
|
|
3
3
|
import 'zod';
|
|
4
4
|
|
|
5
5
|
/**
|
package/dist/node.js
CHANGED
|
@@ -104,8 +104,17 @@ var KnowledgeChunkingProfileSchema = z2.object({
|
|
|
104
104
|
import { z as z3 } from "zod";
|
|
105
105
|
|
|
106
106
|
// src/format/constants.ts
|
|
107
|
-
var GEZK_FORMAT_VERSION = "0.
|
|
108
|
-
var GEZK_INDEX_SCHEMA_VERSION =
|
|
107
|
+
var GEZK_FORMAT_VERSION = "0.6";
|
|
108
|
+
var GEZK_INDEX_SCHEMA_VERSION = 3;
|
|
109
|
+
var GEZK_FORMAT_GENERATIONS = { "0.5": 2, "0.6": 3 };
|
|
110
|
+
var GEZK_SUPPORTED_FORMAT_VERSIONS = Object.keys(GEZK_FORMAT_GENERATIONS);
|
|
111
|
+
var GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS = Object.values(GEZK_FORMAT_GENERATIONS);
|
|
112
|
+
function isSupportedFormatVersion(value) {
|
|
113
|
+
return typeof value === "string" && Object.hasOwn(GEZK_FORMAT_GENERATIONS, value);
|
|
114
|
+
}
|
|
115
|
+
function isSupportedIndexSchemaVersion(value) {
|
|
116
|
+
return typeof value === "number" && GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.includes(value);
|
|
117
|
+
}
|
|
109
118
|
var GEZK_APPLICATION_ID = 1195727435;
|
|
110
119
|
var GEZK_MIME_TYPE = "application/vnd.gezk+zip";
|
|
111
120
|
var MIMETYPE_PATH = "mimetype";
|
|
@@ -116,6 +125,8 @@ var SOURCE_NOTICES_PATH = "LICENSES/source-notices.json";
|
|
|
116
125
|
var ROUTER_DB_PATH = "index/router.db";
|
|
117
126
|
var BODY_CODEC_MIN_BYTES = 512;
|
|
118
127
|
var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
|
|
128
|
+
var MAX_KNOWLEDGE_DOCUMENT_META_BYTES = 16 * 1024;
|
|
129
|
+
var MAX_KNOWLEDGE_TOPIC_DEPTH = 16;
|
|
119
130
|
var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
|
|
120
131
|
|
|
121
132
|
// src/schemas/manifest.ts
|
|
@@ -134,8 +145,8 @@ var KnowledgeSignatureSchema = z3.object({
|
|
|
134
145
|
});
|
|
135
146
|
var KnowledgeCatalogManifestSchema = z3.object({
|
|
136
147
|
kind: z3.literal(GEZK_MANIFEST_KIND),
|
|
137
|
-
formatVersion: z3.
|
|
138
|
-
indexSchemaVersion: z3.literal(
|
|
148
|
+
formatVersion: z3.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
149
|
+
indexSchemaVersion: z3.union([z3.literal(2), z3.literal(3)]),
|
|
139
150
|
id: KnowledgeIdSchema,
|
|
140
151
|
version: KnowledgeVersionSchema,
|
|
141
152
|
name: z3.string().min(1),
|
|
@@ -162,7 +173,17 @@ var KnowledgeCatalogManifestSchema = z3.object({
|
|
|
162
173
|
}),
|
|
163
174
|
embedding: KnowledgeEmbeddingProfileSchema,
|
|
164
175
|
chunking: KnowledgeChunkingProfileSchema,
|
|
165
|
-
topics: z3.array(
|
|
176
|
+
topics: z3.array(
|
|
177
|
+
z3.object({
|
|
178
|
+
id: KnowledgeIdSchema,
|
|
179
|
+
name: z3.string().min(1),
|
|
180
|
+
/** Absent for a root topic; names another entry of this list. */
|
|
181
|
+
parentId: KnowledgeIdSchema.optional(),
|
|
182
|
+
/** Sibling ordering key, identical to `topics.sort_key` in the router. */
|
|
183
|
+
sortKey: z3.string().min(1).optional(),
|
|
184
|
+
description: z3.string().optional()
|
|
185
|
+
})
|
|
186
|
+
).min(1, "a catalog must ship a table of contents (at least one topic)"),
|
|
166
187
|
router: z3.object({
|
|
167
188
|
shardTargetChunks: z3.number().int().positive(),
|
|
168
189
|
shards: z3.array(
|
|
@@ -180,12 +201,14 @@ var KnowledgeCatalogManifestSchema = z3.object({
|
|
|
180
201
|
counts: z3.object({
|
|
181
202
|
documents: z3.number().int().nonnegative(),
|
|
182
203
|
chunks: z3.number().int().nonnegative(),
|
|
183
|
-
shards: z3.number().int().positive()
|
|
204
|
+
shards: z3.number().int().positive(),
|
|
205
|
+
/** Files under `assets/` (0.6+). */
|
|
206
|
+
assets: z3.number().int().nonnegative().optional()
|
|
184
207
|
}),
|
|
185
208
|
files: z3.array(KnowledgeManifestFileSchema).min(1),
|
|
186
209
|
/** What a reader must implement to open this catalog. */
|
|
187
210
|
requires: z3.object({
|
|
188
|
-
formatVersion: z3.
|
|
211
|
+
formatVersion: z3.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
189
212
|
features: z3.array(z3.string()).optional()
|
|
190
213
|
}),
|
|
191
214
|
smokeQueries: z3.array(
|
|
@@ -205,7 +228,50 @@ var KnowledgeCatalogManifestSchema = z3.object({
|
|
|
205
228
|
tokenizerDigest: z3.string().optional()
|
|
206
229
|
}).optional(),
|
|
207
230
|
signature: KnowledgeSignatureSchema.optional()
|
|
231
|
+
}).superRefine((manifest, ctx) => {
|
|
232
|
+
if (GEZK_FORMAT_GENERATIONS[manifest.formatVersion] !== manifest.indexSchemaVersion) {
|
|
233
|
+
ctx.addIssue({
|
|
234
|
+
code: "custom",
|
|
235
|
+
path: ["indexSchemaVersion"],
|
|
236
|
+
message: `format ${manifest.formatVersion} pairs with index schema ${GEZK_FORMAT_GENERATIONS[manifest.formatVersion]}, not ${manifest.indexSchemaVersion}`
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
if (manifest.requires.formatVersion !== manifest.formatVersion) {
|
|
240
|
+
ctx.addIssue({
|
|
241
|
+
code: "custom",
|
|
242
|
+
path: ["requires", "formatVersion"],
|
|
243
|
+
message: "requires.formatVersion must equal formatVersion"
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
if (manifest.formatVersion === "0.5" && manifest.counts.assets !== void 0) {
|
|
247
|
+
ctx.addIssue({
|
|
248
|
+
code: "custom",
|
|
249
|
+
path: ["counts", "assets"],
|
|
250
|
+
message: "counts.assets is a 0.6 field"
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
const topicIds = /* @__PURE__ */ new Set();
|
|
254
|
+
manifest.topics.forEach((topic, index) => {
|
|
255
|
+
if (topicIds.has(topic.id)) {
|
|
256
|
+
ctx.addIssue({
|
|
257
|
+
code: "custom",
|
|
258
|
+
path: ["topics", index, "id"],
|
|
259
|
+
message: `duplicate topic id '${topic.id}'`
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
topicIds.add(topic.id);
|
|
263
|
+
});
|
|
264
|
+
manifest.topics.forEach((topic, index) => {
|
|
265
|
+
if (topic.parentId !== void 0 && !topicIds.has(topic.parentId)) {
|
|
266
|
+
ctx.addIssue({
|
|
267
|
+
code: "custom",
|
|
268
|
+
path: ["topics", index, "parentId"],
|
|
269
|
+
message: `topic '${topic.id}' names an undeclared parent '${topic.parentId}'`
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
});
|
|
208
273
|
});
|
|
274
|
+
var KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS = GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS;
|
|
209
275
|
var SourceNoticesSchema = z3.object({
|
|
210
276
|
sources: z3.array(
|
|
211
277
|
z3.object({
|
|
@@ -223,20 +289,28 @@ var SourceNoticesSchema = z3.object({
|
|
|
223
289
|
|
|
224
290
|
// src/schemas/document.ts
|
|
225
291
|
import { z as z4 } from "zod";
|
|
292
|
+
var KnowledgeOrdinalSchema = z4.number().int().min(-2147483648).max(2147483647);
|
|
293
|
+
var KnowledgeDocumentMetaSchema = z4.record(z4.string(), z4.unknown());
|
|
226
294
|
var CatalogDocumentSchema = z4.object({
|
|
227
295
|
id: KnowledgeDocumentIdSchema,
|
|
228
296
|
title: z4.string().min(1),
|
|
229
297
|
slug: z4.string().min(1),
|
|
230
298
|
summary: z4.string().optional(),
|
|
231
299
|
language: z4.string().min(2),
|
|
232
|
-
/**
|
|
300
|
+
/**
|
|
301
|
+
* Root→leaf topic id path. Every segment must be a declared topic and each
|
|
302
|
+
* must be the parent of the next; the document is filed at the last one.
|
|
303
|
+
*/
|
|
233
304
|
topicPath: z4.array(KnowledgeIdSchema).min(1),
|
|
234
305
|
markdown: z4.string(),
|
|
235
306
|
sourceUrl: z4.string().optional(),
|
|
236
307
|
sourceRevision: z4.string().optional(),
|
|
237
308
|
sourceUpdatedAt: z4.string().optional(),
|
|
238
309
|
attribution: z4.record(z4.string(), z4.string()).optional(),
|
|
239
|
-
aliases: z4.array(z4.string()).optional()
|
|
310
|
+
aliases: z4.array(z4.string()).optional(),
|
|
311
|
+
/** Explicit position among the documents of its topic in a listing. */
|
|
312
|
+
ordinal: KnowledgeOrdinalSchema.optional(),
|
|
313
|
+
meta: KnowledgeDocumentMetaSchema.optional()
|
|
240
314
|
});
|
|
241
315
|
|
|
242
316
|
// src/schemas/registry.ts
|
|
@@ -259,7 +333,7 @@ var KnowledgeRegistryEntrySchema = z5.object({
|
|
|
259
333
|
});
|
|
260
334
|
var KnowledgeRegistryIndexSchema = z5.object({
|
|
261
335
|
kind: z5.literal(GEZK_REGISTRY_KIND),
|
|
262
|
-
formatVersion: z5.
|
|
336
|
+
formatVersion: z5.enum(GEZK_SUPPORTED_FORMAT_VERSIONS),
|
|
263
337
|
publisher: z5.object({
|
|
264
338
|
id: KnowledgeIdSchema,
|
|
265
339
|
name: z5.string().min(1),
|
|
@@ -338,14 +412,16 @@ CREATE TABLE documents(
|
|
|
338
412
|
id TEXT PRIMARY KEY,
|
|
339
413
|
title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
|
|
340
414
|
language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
|
|
415
|
+
ordinal INTEGER,
|
|
341
416
|
shard_id INTEGER NOT NULL,
|
|
342
417
|
chunk_count INTEGER NOT NULL,
|
|
343
418
|
source_url TEXT, source_revision TEXT, source_updated_at TEXT,
|
|
344
419
|
attribution_json TEXT,
|
|
420
|
+
meta_json TEXT,
|
|
345
421
|
body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
|
|
346
422
|
body_blob BLOB NOT NULL
|
|
347
423
|
);
|
|
348
|
-
CREATE INDEX documents_topic ON documents(topic_id, slug);
|
|
424
|
+
CREATE INDEX documents_topic ON documents(topic_id, ordinal, slug);
|
|
349
425
|
CREATE INDEX documents_shard ON documents(shard_id);
|
|
350
426
|
|
|
351
427
|
CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
|
|
@@ -408,6 +484,152 @@ CREATE TABLE chunk_vectors_int8(
|
|
|
408
484
|
);
|
|
409
485
|
`;
|
|
410
486
|
|
|
487
|
+
// src/format/assets.ts
|
|
488
|
+
import { z as z6 } from "zod";
|
|
489
|
+
var ASSETS_PREFIX = "assets/";
|
|
490
|
+
var KNOWLEDGE_ASSET_TYPES = {
|
|
491
|
+
png: "image/png",
|
|
492
|
+
jpg: "image/jpeg",
|
|
493
|
+
jpeg: "image/jpeg",
|
|
494
|
+
gif: "image/gif",
|
|
495
|
+
webp: "image/webp",
|
|
496
|
+
svg: "image/svg+xml"
|
|
497
|
+
};
|
|
498
|
+
var MAX_KNOWLEDGE_ASSET_BYTES = 8 * 1024 * 1024;
|
|
499
|
+
var MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES = 256 * 1024 * 1024;
|
|
500
|
+
var MAX_KNOWLEDGE_ASSET_COUNT = 8192;
|
|
501
|
+
var MAX_KNOWLEDGE_ASSET_PATH_LENGTH = 512;
|
|
502
|
+
var KNOWLEDGE_ASSET_PATH_PATTERN = /^assets\/(?:[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\/){0,15}[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}\.(?:png|jpe?g|gif|webp|svg)$/i;
|
|
503
|
+
var KnowledgeAssetPathSchema = z6.string().max(MAX_KNOWLEDGE_ASSET_PATH_LENGTH).regex(KNOWLEDGE_ASSET_PATH_PATTERN, "not a valid asset path");
|
|
504
|
+
function isKnowledgeAssetPath(path) {
|
|
505
|
+
return path.length <= MAX_KNOWLEDGE_ASSET_PATH_LENGTH && KNOWLEDGE_ASSET_PATH_PATTERN.test(path);
|
|
506
|
+
}
|
|
507
|
+
function assetExtension(path) {
|
|
508
|
+
const dot = path.lastIndexOf(".");
|
|
509
|
+
if (dot < 0) return null;
|
|
510
|
+
const ext = path.slice(dot + 1).toLowerCase();
|
|
511
|
+
return Object.hasOwn(KNOWLEDGE_ASSET_TYPES, ext) ? ext : null;
|
|
512
|
+
}
|
|
513
|
+
function assetContentType(path) {
|
|
514
|
+
const ext = assetExtension(path);
|
|
515
|
+
return ext ? KNOWLEDGE_ASSET_TYPES[ext] : null;
|
|
516
|
+
}
|
|
517
|
+
function assetKindForExtension(ext) {
|
|
518
|
+
return ext === "jpg" ? "jpeg" : ext;
|
|
519
|
+
}
|
|
520
|
+
var PNG_MAGIC = [137, 80, 78, 71, 13, 10, 26, 10];
|
|
521
|
+
var JPEG_MAGIC = [255, 216, 255];
|
|
522
|
+
var GIF87 = [71, 73, 70, 56, 55, 97];
|
|
523
|
+
var GIF89 = [71, 73, 70, 56, 57, 97];
|
|
524
|
+
var RIFF = [82, 73, 70, 70];
|
|
525
|
+
var WEBP = [87, 69, 66, 80];
|
|
526
|
+
function startsWith(bytes, magic, offset = 0) {
|
|
527
|
+
if (bytes.length < offset + magic.length) return false;
|
|
528
|
+
for (let i = 0; i < magic.length; i++) {
|
|
529
|
+
if (bytes[offset + i] !== magic[i]) return false;
|
|
530
|
+
}
|
|
531
|
+
return true;
|
|
532
|
+
}
|
|
533
|
+
function sniffAssetType(bytes) {
|
|
534
|
+
if (startsWith(bytes, PNG_MAGIC)) return "png";
|
|
535
|
+
if (startsWith(bytes, JPEG_MAGIC)) return "jpeg";
|
|
536
|
+
if (startsWith(bytes, GIF87) || startsWith(bytes, GIF89)) return "gif";
|
|
537
|
+
if (startsWith(bytes, RIFF) && startsWith(bytes, WEBP, 8)) return "webp";
|
|
538
|
+
return looksLikeSvg(decodeText(bytes)) ? "svg" : null;
|
|
539
|
+
}
|
|
540
|
+
function decodeText(bytes) {
|
|
541
|
+
try {
|
|
542
|
+
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes);
|
|
543
|
+
} catch {
|
|
544
|
+
return "";
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
function looksLikeSvg(text) {
|
|
548
|
+
let i = 0;
|
|
549
|
+
const n = text.length;
|
|
550
|
+
while (i < n) {
|
|
551
|
+
const ch = text.charCodeAt(i);
|
|
552
|
+
if (ch === 32 || ch === 9 || ch === 10 || ch === 13 || ch === 65279) {
|
|
553
|
+
i++;
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
556
|
+
if (text.startsWith("<?", i)) {
|
|
557
|
+
const end = text.indexOf("?>", i);
|
|
558
|
+
if (end < 0) return false;
|
|
559
|
+
i = end + 2;
|
|
560
|
+
continue;
|
|
561
|
+
}
|
|
562
|
+
if (text.startsWith("<!--", i)) {
|
|
563
|
+
const end = text.indexOf("-->", i);
|
|
564
|
+
if (end < 0) return false;
|
|
565
|
+
i = end + 3;
|
|
566
|
+
continue;
|
|
567
|
+
}
|
|
568
|
+
if (/^<!doctype/i.test(text.slice(i, i + 9))) {
|
|
569
|
+
const bracket = text.indexOf("[", i);
|
|
570
|
+
const close = text.indexOf(">", i);
|
|
571
|
+
if (close < 0) return false;
|
|
572
|
+
if (bracket >= 0 && bracket < close) {
|
|
573
|
+
const end = text.indexOf("]>", bracket);
|
|
574
|
+
if (end < 0) return false;
|
|
575
|
+
i = end + 2;
|
|
576
|
+
} else {
|
|
577
|
+
i = close + 1;
|
|
578
|
+
}
|
|
579
|
+
continue;
|
|
580
|
+
}
|
|
581
|
+
return /^<svg[\s>/]/i.test(text.slice(i, i + 5));
|
|
582
|
+
}
|
|
583
|
+
return false;
|
|
584
|
+
}
|
|
585
|
+
var SVG_ACTIVE_PATTERNS = [
|
|
586
|
+
[/<script[\s>/]/i, "a <script> element"],
|
|
587
|
+
[/<foreignobject[\s>/]/i, "a <foreignObject> element"],
|
|
588
|
+
[/<!entity/i, "an entity declaration"],
|
|
589
|
+
[/\son[a-z]+\s*=/i, "an event-handler attribute"],
|
|
590
|
+
[/javascript\s*:/i, "a javascript: reference"],
|
|
591
|
+
[/data\s*:\s*text\/html/i, "a data:text/html reference"],
|
|
592
|
+
[/@import/i, "a CSS @import"]
|
|
593
|
+
];
|
|
594
|
+
var HREF_ATTRIBUTE = /(?:xlink:)?href\s*=\s*["']\s*([^"']*)["']/gi;
|
|
595
|
+
var CSS_URL = /url\(\s*["']?\s*([^"')]*)/gi;
|
|
596
|
+
function externalReferenceProblem(target) {
|
|
597
|
+
const value = target.trim();
|
|
598
|
+
if (value === "" || value.startsWith("#")) return null;
|
|
599
|
+
const scheme = /^([a-z][a-z0-9+.-]*):/i.exec(value);
|
|
600
|
+
if (!scheme) return null;
|
|
601
|
+
if (/^data:image\/(?:png|jpeg|gif|webp)[;,]/i.test(value)) return null;
|
|
602
|
+
return `a ${scheme[1]}: reference`;
|
|
603
|
+
}
|
|
604
|
+
function svgInertnessProblem(bytes) {
|
|
605
|
+
const text = decodeText(bytes);
|
|
606
|
+
if (text === "") return "not valid UTF-8";
|
|
607
|
+
if (!looksLikeSvg(text)) return "does not start with an <svg> element";
|
|
608
|
+
for (const [pattern, why] of SVG_ACTIVE_PATTERNS) {
|
|
609
|
+
if (pattern.test(text)) return `contains ${why}`;
|
|
610
|
+
}
|
|
611
|
+
for (const match of text.matchAll(HREF_ATTRIBUTE)) {
|
|
612
|
+
const problem = externalReferenceProblem(match[1] ?? "");
|
|
613
|
+
if (problem) return `contains ${problem} in an href`;
|
|
614
|
+
}
|
|
615
|
+
for (const match of text.matchAll(CSS_URL)) {
|
|
616
|
+
const problem = externalReferenceProblem(match[1] ?? "");
|
|
617
|
+
if (problem) return `contains ${problem} in a CSS url()`;
|
|
618
|
+
}
|
|
619
|
+
return null;
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
// src/format/sort-key.ts
|
|
623
|
+
var INT32_MIN = -2147483648;
|
|
624
|
+
var INT32_MAX = 2147483647;
|
|
625
|
+
var OFFSET = 2147483648;
|
|
626
|
+
function topicSortKeyForOrder(order) {
|
|
627
|
+
if (!Number.isInteger(order) || order < INT32_MIN || order > INT32_MAX) {
|
|
628
|
+
throw new RangeError(`topic order must be an int32: ${String(order)}`);
|
|
629
|
+
}
|
|
630
|
+
return String(order + OFFSET).padStart(10, "0");
|
|
631
|
+
}
|
|
632
|
+
|
|
411
633
|
// src/format/quantize.ts
|
|
412
634
|
function quantizeInt8(unitVector) {
|
|
413
635
|
const out = new Int8Array(unitVector.length);
|
|
@@ -617,26 +839,36 @@ function verifySignedDocument(document, anchors) {
|
|
|
617
839
|
}
|
|
618
840
|
}
|
|
619
841
|
export {
|
|
842
|
+
ASSETS_PREFIX,
|
|
620
843
|
ArtifactDigestSchema,
|
|
621
844
|
BODY_CODEC_MIN_BYTES,
|
|
622
845
|
CatalogDocumentSchema,
|
|
623
846
|
DEFAULT_EMBEDDING_ONNX_FILE,
|
|
624
847
|
DEFAULT_EMBEDDING_TOKENIZER_FILE,
|
|
625
848
|
GEZK_APPLICATION_ID,
|
|
849
|
+
GEZK_FORMAT_GENERATIONS,
|
|
626
850
|
GEZK_FORMAT_VERSION,
|
|
627
851
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
628
852
|
GEZK_MANIFEST_KIND,
|
|
629
853
|
GEZK_MIME_TYPE,
|
|
630
854
|
GEZK_REGISTRY_KIND,
|
|
855
|
+
GEZK_SUPPORTED_FORMAT_VERSIONS,
|
|
856
|
+
GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
|
|
857
|
+
KNOWLEDGE_ASSET_PATH_PATTERN,
|
|
858
|
+
KNOWLEDGE_ASSET_TYPES,
|
|
631
859
|
KNOWLEDGE_ID_PATTERN,
|
|
860
|
+
KNOWLEDGE_MANIFEST_INDEX_SCHEMA_VERSIONS,
|
|
632
861
|
KNOWLEDGE_VERSION_PATTERN,
|
|
862
|
+
KnowledgeAssetPathSchema,
|
|
633
863
|
KnowledgeCatalogManifestSchema,
|
|
634
864
|
KnowledgeChunkUidSchema,
|
|
635
865
|
KnowledgeChunkingProfileSchema,
|
|
636
866
|
KnowledgeDocumentIdSchema,
|
|
867
|
+
KnowledgeDocumentMetaSchema,
|
|
637
868
|
KnowledgeEmbeddingProfileSchema,
|
|
638
869
|
KnowledgeIdSchema,
|
|
639
870
|
KnowledgeManifestFileSchema,
|
|
871
|
+
KnowledgeOrdinalSchema,
|
|
640
872
|
KnowledgeRegistryEntrySchema,
|
|
641
873
|
KnowledgeRegistryIndexSchema,
|
|
642
874
|
KnowledgeSignatureSchema,
|
|
@@ -644,7 +876,13 @@ export {
|
|
|
644
876
|
KnowledgeVersionSchema,
|
|
645
877
|
LICENSE_NOTICE_PATH,
|
|
646
878
|
MANIFEST_PATH,
|
|
879
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
|
|
880
|
+
MAX_KNOWLEDGE_ASSET_BYTES,
|
|
881
|
+
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
882
|
+
MAX_KNOWLEDGE_ASSET_PATH_LENGTH,
|
|
647
883
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
884
|
+
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
885
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
648
886
|
MIMETYPE_PATH,
|
|
649
887
|
README_PATH,
|
|
650
888
|
ROUTER_DB_PATH,
|
|
@@ -655,6 +893,9 @@ export {
|
|
|
655
893
|
Sha256HexSchema,
|
|
656
894
|
SourceNoticesSchema,
|
|
657
895
|
ZIP_FIXED_MTIME,
|
|
896
|
+
assetContentType,
|
|
897
|
+
assetExtension,
|
|
898
|
+
assetKindForExtension,
|
|
658
899
|
canonicalizeJson,
|
|
659
900
|
chunkContentHash,
|
|
660
901
|
chunkUid,
|
|
@@ -663,6 +904,9 @@ export {
|
|
|
663
904
|
formatKnowledgeUri,
|
|
664
905
|
generateKnowledgeSigningKeyPair,
|
|
665
906
|
hashFileStreaming,
|
|
907
|
+
isKnowledgeAssetPath,
|
|
908
|
+
isSupportedFormatVersion,
|
|
909
|
+
isSupportedIndexSchemaVersion,
|
|
666
910
|
knowledgeKeyId,
|
|
667
911
|
l2Normalize,
|
|
668
912
|
manifestSigningPayload,
|
|
@@ -673,6 +917,9 @@ export {
|
|
|
673
917
|
sameVectorSpace,
|
|
674
918
|
signManifest,
|
|
675
919
|
signRegistryIndex,
|
|
920
|
+
sniffAssetType,
|
|
921
|
+
svgInertnessProblem,
|
|
922
|
+
topicSortKeyForOrder,
|
|
676
923
|
verifyManifestSignature,
|
|
677
924
|
verifyRegistryIndex
|
|
678
925
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/gezk",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"description": "The gezk knowledge-catalog format: manifest and registry schemas, id grammars, knowledge:// references, vector quantization, SQLite DDL, canonical JSON, and Ed25519 manifest signatures.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|