@bendyline/gezel-knowledge 1.2.6 → 1.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +49 -4
- package/dist/index.js +155 -69
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -475,8 +475,8 @@ declare function openCatalogDatabase(absPath: string): CatalogDb;
|
|
|
475
475
|
* Read-only document point index. The router's ordinary SQLite latitude and
|
|
476
476
|
* longitude indexes provide conservative candidates; the portable gezk
|
|
477
477
|
* spherical predicate decides membership before ranking or pagination.
|
|
478
|
-
*
|
|
479
|
-
*
|
|
478
|
+
* Archives without a `document_locations` table (before 0.7) have no
|
|
479
|
+
* locations: opaque producer metadata is never read as coordinates.
|
|
480
480
|
*/
|
|
481
481
|
|
|
482
482
|
interface SpatialMatch {
|
|
@@ -517,6 +517,14 @@ interface CatalogDocumentMeta {
|
|
|
517
517
|
meta: Record<string, unknown> | null;
|
|
518
518
|
locations?: KnowledgeLocation[];
|
|
519
519
|
}
|
|
520
|
+
/** One slug initial of a listing: where it first appears and how many documents share it. */
|
|
521
|
+
interface CatalogDocumentInitial {
|
|
522
|
+
/** The slug's first character, as stored (slugs are usually lowercase ASCII). */
|
|
523
|
+
initial: string;
|
|
524
|
+
/** Zero-based position of its first document in `documentsPage` order. */
|
|
525
|
+
offset: number;
|
|
526
|
+
count: number;
|
|
527
|
+
}
|
|
520
528
|
interface CatalogAssetInfo {
|
|
521
529
|
/** Archive path under `assets/`. */
|
|
522
530
|
path: string;
|
|
@@ -671,6 +679,26 @@ declare class CatalogHandle {
|
|
|
671
679
|
documents: CatalogDocumentMeta[];
|
|
672
680
|
total: number;
|
|
673
681
|
};
|
|
682
|
+
/**
|
|
683
|
+
* Where each slug initial first appears in `documentsPage` order, for an
|
|
684
|
+
* A–Z index over the listing. Only the alphabetical part counts: documents
|
|
685
|
+
* with an ordinal are listed first in their curated order, so a fully
|
|
686
|
+
* curated listing has no initials.
|
|
687
|
+
*/
|
|
688
|
+
documentInitials(opts?: {
|
|
689
|
+
topicId?: string;
|
|
690
|
+
descendants?: boolean;
|
|
691
|
+
}): CatalogDocumentInitial[];
|
|
692
|
+
/** A document's zero-based position in `documentsPage` order; null when the listing lacks it. */
|
|
693
|
+
documentPosition(documentId: string, opts?: {
|
|
694
|
+
topicId?: string;
|
|
695
|
+
descendants?: boolean;
|
|
696
|
+
}): number | null;
|
|
697
|
+
/** A listing's documents as `id, slug, ordinal` rows, ordered by LISTING_ORDER. */
|
|
698
|
+
private listingRows;
|
|
699
|
+
private plainScope;
|
|
700
|
+
/** The topic's placements (schema 4) as `scoped`; binds the topic id once. */
|
|
701
|
+
private sharedScope;
|
|
674
702
|
/** A reference is a placement, not a duplicate result or a redirect page. */
|
|
675
703
|
private sharedDocumentsPage;
|
|
676
704
|
/** Integrity facts kept separate from browse counts, which include shared references. */
|
|
@@ -957,7 +985,7 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
|
957
985
|
* bytes.
|
|
958
986
|
*/
|
|
959
987
|
|
|
960
|
-
/** 4: document-locations companion
|
|
988
|
+
/** 4: document-locations companion. */
|
|
961
989
|
declare const PARQUET_EXPORT_VERSION = 4;
|
|
962
990
|
type ParquetExportSource = {
|
|
963
991
|
archivePath: string;
|
|
@@ -1062,6 +1090,8 @@ declare const EMBEDDINGGEMMA_2_512_1: KnowledgeEmbeddingProfile;
|
|
|
1062
1090
|
/** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
|
|
1063
1091
|
declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
|
|
1064
1092
|
declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
|
|
1093
|
+
/** What a profile's query model costs to download, or null for an unregistered profile. */
|
|
1094
|
+
declare function knowledgeProfileQueryModelBytes(id: string): number | null;
|
|
1065
1095
|
/** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
|
|
1066
1096
|
declare const MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
|
|
1067
1097
|
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
@@ -1274,6 +1304,8 @@ declare class EmbedderArtifactError extends Error {
|
|
|
1274
1304
|
}
|
|
1275
1305
|
/** The transformers.js FileCache key for one repo file. */
|
|
1276
1306
|
declare function transformersCachePath(cacheDir: string, repo: string, revision: string, file: string): string;
|
|
1307
|
+
/** `sha256:<hex>` of a file, or null when it is absent. Memoized by path, size and mtime. */
|
|
1308
|
+
declare function fileDigest(path: string): Promise<string | null>;
|
|
1277
1309
|
/** The pinned files of a profile's media encoders for the given modalities (video reuses image). */
|
|
1278
1310
|
declare function mediaEncoderPins(profile: KnowledgeEmbeddingProfile, modalities: ReadonlyArray<'image' | 'video' | 'audio'>): Array<{
|
|
1279
1311
|
file: string;
|
|
@@ -1505,6 +1537,19 @@ declare function prefetchExternalData(profile: KnowledgeEmbeddingProfile, opts:
|
|
|
1505
1537
|
revision?: string;
|
|
1506
1538
|
fetchImpl?: typeof fetch;
|
|
1507
1539
|
}): Promise<string[]>;
|
|
1540
|
+
/**
|
|
1541
|
+
* Remove the cached pinned files whose bytes do not match their pins, and
|
|
1542
|
+
* return their paths. transformers.js streams a download straight into its
|
|
1543
|
+
* cache path, so one that is cut off stays there as a truncated file that
|
|
1544
|
+
* every later load fails to parse, until someone deletes it by hand. Only for
|
|
1545
|
+
* a load that already failed that way: it hashes every pinned file present,
|
|
1546
|
+
* which a healthy load never pays for.
|
|
1547
|
+
*/
|
|
1548
|
+
declare function removeMismatchedPinnedFiles(profile: KnowledgeEmbeddingProfile, opts: {
|
|
1549
|
+
cacheDir: string;
|
|
1550
|
+
modalities?: readonly MediaModality[];
|
|
1551
|
+
revision?: string;
|
|
1552
|
+
}): Promise<string[]>;
|
|
1508
1553
|
|
|
1509
1554
|
/**
|
|
1510
1555
|
* Tables of contents that documentation trees already carry. Each parser
|
|
@@ -1763,4 +1808,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1763
1808
|
|
|
1764
1809
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1765
1810
|
|
|
1766
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
|
|
1811
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentInitial, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, fileDigest, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, knowledgeProfileQueryModelBytes, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, removeMismatchedPinnedFiles, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -2195,23 +2195,19 @@ import {
|
|
|
2195
2195
|
KnowledgeLocationsSchema as KnowledgeLocationsSchema2,
|
|
2196
2196
|
KnowledgeRadiusSchema,
|
|
2197
2197
|
locationDistanceMeters,
|
|
2198
|
-
locationInBounds,
|
|
2199
2198
|
normalizeLongitude as normalizeLongitude2,
|
|
2200
2199
|
radiusBounds,
|
|
2201
2200
|
spatialManifest as spatialManifest2
|
|
2202
2201
|
} from "@bendyline/gezk";
|
|
2203
2202
|
var DocumentSpatialIndex = class {
|
|
2204
|
-
constructor(db
|
|
2203
|
+
constructor(db) {
|
|
2205
2204
|
this.db = db;
|
|
2206
|
-
this.allowLegacy = allowLegacy;
|
|
2207
2205
|
this.hasTable = Boolean(
|
|
2208
2206
|
db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='document_locations'").get()
|
|
2209
2207
|
);
|
|
2210
2208
|
}
|
|
2211
2209
|
db;
|
|
2212
|
-
allowLegacy;
|
|
2213
2210
|
hasTable;
|
|
2214
|
-
legacy;
|
|
2215
2211
|
decode(row) {
|
|
2216
2212
|
const provenance = row.provenance_json;
|
|
2217
2213
|
if (typeof provenance === "string" && Buffer.byteLength(provenance) > 16384)
|
|
@@ -2227,39 +2223,13 @@ var DocumentSpatialIndex = class {
|
|
|
2227
2223
|
})
|
|
2228
2224
|
};
|
|
2229
2225
|
}
|
|
2230
|
-
legacyRows() {
|
|
2231
|
-
if (this.legacy) return this.legacy;
|
|
2232
|
-
this.legacy = [];
|
|
2233
|
-
if (!this.allowLegacy) return this.legacy;
|
|
2234
|
-
for (const row of this.db.prepare(
|
|
2235
|
-
"SELECT id, meta_json FROM documents WHERE id LIKE 'qualla-%' AND meta_json IS NOT NULL"
|
|
2236
|
-
).all()) {
|
|
2237
|
-
try {
|
|
2238
|
-
const meta = JSON.parse(String(row.meta_json));
|
|
2239
|
-
if (typeof meta.region !== "string" || !Array.isArray(meta.quallaArticleIds) || !meta.coordinates)
|
|
2240
|
-
continue;
|
|
2241
|
-
const point = KnowledgeLocationSchema.safeParse({
|
|
2242
|
-
id: "qualla-coordinate",
|
|
2243
|
-
role: "subject",
|
|
2244
|
-
latitude: meta.coordinates.lat,
|
|
2245
|
-
longitude: meta.coordinates.lng,
|
|
2246
|
-
provenance: { adapter: "qualla-regional-meta@1", articleIds: meta.quallaArticleIds }
|
|
2247
|
-
});
|
|
2248
|
-
if (point.success)
|
|
2249
|
-
this.legacy.push({
|
|
2250
|
-
documentId: String(row.id),
|
|
2251
|
-
location: { ...point.data, longitude: normalizeLongitude2(point.data.longitude) }
|
|
2252
|
-
});
|
|
2253
|
-
} catch {
|
|
2254
|
-
}
|
|
2255
|
-
}
|
|
2256
|
-
return this.legacy;
|
|
2257
|
-
}
|
|
2258
2226
|
rows() {
|
|
2259
|
-
|
|
2227
|
+
if (!this.hasTable) return [];
|
|
2228
|
+
return this.db.prepare("SELECT * FROM document_locations ORDER BY document_id, location_id").all().map((row) => this.decode(row));
|
|
2260
2229
|
}
|
|
2261
2230
|
locations(documentId) {
|
|
2262
|
-
|
|
2231
|
+
if (!this.hasTable) return [];
|
|
2232
|
+
return this.db.prepare("SELECT * FROM document_locations WHERE document_id = ? ORDER BY location_id").all(documentId).map((row) => this.decode(row).location);
|
|
2263
2233
|
}
|
|
2264
2234
|
integrity() {
|
|
2265
2235
|
const rows = this.rows();
|
|
@@ -2278,14 +2248,12 @@ var DocumentSpatialIndex = class {
|
|
|
2278
2248
|
}
|
|
2279
2249
|
matches(raw) {
|
|
2280
2250
|
const radius = KnowledgeRadiusSchema.parse(raw);
|
|
2281
|
-
const
|
|
2282
|
-
|
|
2251
|
+
const matches = /* @__PURE__ */ new Map();
|
|
2252
|
+
if (!this.hasTable) return matches;
|
|
2253
|
+
const candidates = radiusBounds(radius).flatMap(
|
|
2283
2254
|
(box) => this.db.prepare(`SELECT * FROM document_locations
|
|
2284
2255
|
WHERE role = 'subject' AND latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ?`).all(box.south, box.north, box.west, box.east).map((row) => this.decode(row))
|
|
2285
|
-
) : this.legacyRows().filter(
|
|
2286
|
-
(row) => row.location.role === "subject" && boxes.some((box) => locationInBounds(row.location, box))
|
|
2287
2256
|
);
|
|
2288
|
-
const matches = /* @__PURE__ */ new Map();
|
|
2289
2257
|
for (const { documentId, location } of candidates) {
|
|
2290
2258
|
const distanceMeters = locationDistanceMeters(radius, location);
|
|
2291
2259
|
if (distanceMeters > radius.radiusMeters) continue;
|
|
@@ -2300,6 +2268,11 @@ var DocumentSpatialIndex = class {
|
|
|
2300
2268
|
|
|
2301
2269
|
// src/reader/catalog-handle.ts
|
|
2302
2270
|
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
2271
|
+
var LISTING_ORDER = "(ordinal IS NULL), ordinal, slug, id";
|
|
2272
|
+
var SHARED_PLACEMENT_RANK = `ranked AS (
|
|
2273
|
+
SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
|
|
2274
|
+
ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
|
|
2275
|
+
)`;
|
|
2303
2276
|
var CHUNK_COLUMNS = "chunk_uid, document_id, title, heading_path, line_start, line_end, text";
|
|
2304
2277
|
var MEDIA_COLUMNS = "modality, asset_path, mime_type, width, height, start_ms, end_ms, thumbnail_path, attribution_json";
|
|
2305
2278
|
function mediaOf(row) {
|
|
@@ -2342,7 +2315,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2342
2315
|
this.rootDir = rootDir;
|
|
2343
2316
|
this.router = router;
|
|
2344
2317
|
this.schemaVersion = router.schemaVersion;
|
|
2345
|
-
this.spatial = new DocumentSpatialIndex(router.db
|
|
2318
|
+
this.spatial = new DocumentSpatialIndex(router.db);
|
|
2346
2319
|
this.meta = {};
|
|
2347
2320
|
for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
|
|
2348
2321
|
this.meta[row.key] = row.value;
|
|
@@ -2690,22 +2663,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2690
2663
|
if (this.schemaVersion >= 4 && opts.topicId) {
|
|
2691
2664
|
return this.sharedDocumentsPage(opts.topicId, descendants, limit, offset);
|
|
2692
2665
|
}
|
|
2693
|
-
|
|
2694
|
-
let where = "";
|
|
2695
|
-
const params = [];
|
|
2696
|
-
if (opts.topicId && descendants) {
|
|
2697
|
-
scope = `WITH RECURSIVE sub(id, depth) AS (
|
|
2698
|
-
SELECT ?, 0
|
|
2699
|
-
UNION ALL
|
|
2700
|
-
SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
|
|
2701
|
-
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
2702
|
-
)`;
|
|
2703
|
-
where = "WHERE topic_id IN (SELECT id FROM sub)";
|
|
2704
|
-
params.push(opts.topicId);
|
|
2705
|
-
} else if (opts.topicId) {
|
|
2706
|
-
where = "WHERE topic_id = ?";
|
|
2707
|
-
params.push(opts.topicId);
|
|
2708
|
-
}
|
|
2666
|
+
const { scope, where, params } = this.plainScope(opts.topicId, descendants);
|
|
2709
2667
|
const total = Number(
|
|
2710
2668
|
this.router.db.prepare(`${scope} SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
|
|
2711
2669
|
);
|
|
@@ -2715,23 +2673,88 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2715
2673
|
).all(...params, limit, offset).map((row) => this.documentMeta(row));
|
|
2716
2674
|
return { documents, total };
|
|
2717
2675
|
}
|
|
2718
|
-
/**
|
|
2719
|
-
|
|
2676
|
+
/**
|
|
2677
|
+
* Where each slug initial first appears in `documentsPage` order, for an
|
|
2678
|
+
* A–Z index over the listing. Only the alphabetical part counts: documents
|
|
2679
|
+
* with an ordinal are listed first in their curated order, so a fully
|
|
2680
|
+
* curated listing has no initials.
|
|
2681
|
+
*/
|
|
2682
|
+
documentInitials(opts = {}) {
|
|
2683
|
+
const { sql, params } = this.listingRows(opts.topicId, opts.descendants ?? true);
|
|
2684
|
+
const rows = this.router.db.prepare(
|
|
2685
|
+
`SELECT initial, MIN(position) AS position, COUNT(*) AS n FROM (
|
|
2686
|
+
SELECT substr(slug, 1, 1) AS initial, ordinal,
|
|
2687
|
+
ROW_NUMBER() OVER (ORDER BY ${LISTING_ORDER}) - 1 AS position
|
|
2688
|
+
FROM (${sql})
|
|
2689
|
+
) WHERE ordinal IS NULL GROUP BY initial ORDER BY position`
|
|
2690
|
+
).all(...params);
|
|
2691
|
+
return rows.map((r) => ({
|
|
2692
|
+
initial: r.initial,
|
|
2693
|
+
offset: Number(r.position),
|
|
2694
|
+
count: Number(r.n)
|
|
2695
|
+
}));
|
|
2696
|
+
}
|
|
2697
|
+
/** A document's zero-based position in `documentsPage` order; null when the listing lacks it. */
|
|
2698
|
+
documentPosition(documentId, opts = {}) {
|
|
2699
|
+
const { sql, params } = this.listingRows(opts.topicId, opts.descendants ?? true);
|
|
2700
|
+
const row = this.router.db.prepare(
|
|
2701
|
+
`SELECT position FROM (
|
|
2702
|
+
SELECT id, ROW_NUMBER() OVER (ORDER BY ${LISTING_ORDER}) - 1 AS position FROM (${sql})
|
|
2703
|
+
) WHERE id = ?`
|
|
2704
|
+
).get(...params, documentId);
|
|
2705
|
+
return row ? Number(row.position) : null;
|
|
2706
|
+
}
|
|
2707
|
+
/** A listing's documents as `id, slug, ordinal` rows, ordered by LISTING_ORDER. */
|
|
2708
|
+
listingRows(topicId, descendants) {
|
|
2709
|
+
if (this.schemaVersion >= 4 && topicId) {
|
|
2710
|
+
return {
|
|
2711
|
+
sql: `${this.sharedScope(descendants)}, ${SHARED_PLACEMENT_RANK}
|
|
2712
|
+
SELECT d.id AS id, d.slug AS slug, p.ordinal AS ordinal
|
|
2713
|
+
FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1`,
|
|
2714
|
+
params: [topicId]
|
|
2715
|
+
};
|
|
2716
|
+
}
|
|
2717
|
+
const { scope, where, params } = this.plainScope(topicId, descendants);
|
|
2718
|
+
const ordinal = this.schemaVersion >= 3 ? "ordinal" : "NULL";
|
|
2719
|
+
return {
|
|
2720
|
+
sql: `${scope} SELECT id, slug, ${ordinal} AS ordinal FROM documents ${where}`,
|
|
2721
|
+
params
|
|
2722
|
+
};
|
|
2723
|
+
}
|
|
2724
|
+
plainScope(topicId, descendants) {
|
|
2725
|
+
if (topicId && descendants) {
|
|
2726
|
+
return {
|
|
2727
|
+
scope: `WITH RECURSIVE sub(id, depth) AS (
|
|
2728
|
+
SELECT ?, 0
|
|
2729
|
+
UNION ALL
|
|
2730
|
+
SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
|
|
2731
|
+
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
2732
|
+
)`,
|
|
2733
|
+
where: "WHERE topic_id IN (SELECT id FROM sub)",
|
|
2734
|
+
params: [topicId]
|
|
2735
|
+
};
|
|
2736
|
+
}
|
|
2737
|
+
if (topicId) return { scope: "", where: "WHERE topic_id = ?", params: [topicId] };
|
|
2738
|
+
return { scope: "", where: "", params: [] };
|
|
2739
|
+
}
|
|
2740
|
+
/** The topic's placements (schema 4) as `scoped`; binds the topic id once. */
|
|
2741
|
+
sharedScope(descendants) {
|
|
2720
2742
|
const sub = descendants ? `SELECT ?, 0 UNION ALL SELECT t.id, sub.depth + 1 FROM topics t
|
|
2721
2743
|
JOIN sub ON t.parent_id = sub.id WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}` : "SELECT ?, 0";
|
|
2722
|
-
|
|
2744
|
+
return `WITH RECURSIVE sub(id, depth) AS (${sub}), scoped AS (
|
|
2723
2745
|
SELECT td.document_id, td.topic_id, td.ordinal, t.sort_key
|
|
2724
2746
|
FROM topic_documents td JOIN topics t ON t.id = td.topic_id
|
|
2725
2747
|
WHERE td.topic_id IN (SELECT id FROM sub)
|
|
2726
2748
|
)`;
|
|
2749
|
+
}
|
|
2750
|
+
/** A reference is a placement, not a duplicate result or a redirect page. */
|
|
2751
|
+
sharedDocumentsPage(topicId, descendants, limit, offset) {
|
|
2752
|
+
const scope = this.sharedScope(descendants);
|
|
2727
2753
|
const total = Number(
|
|
2728
2754
|
this.router.db.prepare(`${scope} SELECT COUNT(DISTINCT document_id) AS n FROM scoped`).get(topicId).n
|
|
2729
2755
|
);
|
|
2730
2756
|
const documents = this.router.db.prepare(
|
|
2731
|
-
`${scope},
|
|
2732
|
-
SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
|
|
2733
|
-
ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
|
|
2734
|
-
) SELECT ${this.documentColumns("d", "p")}
|
|
2757
|
+
`${scope}, ${SHARED_PLACEMENT_RANK} SELECT ${this.documentColumns("d", "p")}
|
|
2735
2758
|
FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1
|
|
2736
2759
|
ORDER BY (p.ordinal IS NULL), p.ordinal, d.slug, d.id LIMIT ? OFFSET ?`
|
|
2737
2760
|
).all(topicId, limit, offset).map((row) => this.documentMeta(row));
|
|
@@ -3725,9 +3748,9 @@ async function exportCatalogParquet(opts) {
|
|
|
3725
3748
|
onProgress: opts.onProgress
|
|
3726
3749
|
})
|
|
3727
3750
|
);
|
|
3728
|
-
const spatial = new DocumentSpatialIndex(router.db
|
|
3751
|
+
const spatial = new DocumentSpatialIndex(router.db);
|
|
3729
3752
|
const locations = spatial.rows();
|
|
3730
|
-
if (spatial.hasTable
|
|
3753
|
+
if (spatial.hasTable) {
|
|
3731
3754
|
const path = join4(staging, "document-locations.ndjson");
|
|
3732
3755
|
const out = ndjsonWriter(path);
|
|
3733
3756
|
for (const { documentId, location } of locations)
|
|
@@ -3741,7 +3764,10 @@ async function exportCatalogParquet(opts) {
|
|
|
3741
3764
|
});
|
|
3742
3765
|
await out.close();
|
|
3743
3766
|
const columns = "{document_id:'VARCHAR',location_id:'VARCHAR',latitude:'DOUBLE',longitude:'DOUBLE',role:'VARCHAR',provenance_json:'VARCHAR'}";
|
|
3744
|
-
const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` :
|
|
3767
|
+
const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : (
|
|
3768
|
+
// `role` is a DuckDB keyword: a bare alias is a parser error.
|
|
3769
|
+
'(SELECT NULL::VARCHAR AS "document_id", NULL::VARCHAR AS "location_id", NULL::DOUBLE AS "latitude", NULL::DOUBLE AS "longitude", NULL::VARCHAR AS "role", NULL::VARCHAR AS "provenance_json" WHERE false)'
|
|
3770
|
+
);
|
|
3745
3771
|
files.push(
|
|
3746
3772
|
await writeParquet(opts.duckdb, {
|
|
3747
3773
|
outDir,
|
|
@@ -5183,6 +5209,15 @@ var KNOWLEDGE_EMBEDDING_PROFILES = [
|
|
|
5183
5209
|
function knowledgeEmbeddingProfile(id) {
|
|
5184
5210
|
return KNOWLEDGE_EMBEDDING_PROFILES.find((p) => p.id === id) ?? null;
|
|
5185
5211
|
}
|
|
5212
|
+
var QUERY_MODEL_DOWNLOAD_BYTES = {
|
|
5213
|
+
"embeddinggemma-2-512@1": 346397233,
|
|
5214
|
+
"multilingual-e5-small@2": 487352364,
|
|
5215
|
+
"multilingual-e5-small@1": 487352364,
|
|
5216
|
+
"bge-small-en-v1.5@1": 133805935
|
|
5217
|
+
};
|
|
5218
|
+
function knowledgeProfileQueryModelBytes(id) {
|
|
5219
|
+
return QUERY_MODEL_DOWNLOAD_BYTES[id] ?? null;
|
|
5220
|
+
}
|
|
5186
5221
|
var MARKDOWN_CHUNKS_2 = {
|
|
5187
5222
|
id: "markdown-chunks@2",
|
|
5188
5223
|
unit: "tokens",
|
|
@@ -5405,6 +5440,19 @@ async function prefetchExternalData(profile, opts) {
|
|
|
5405
5440
|
}
|
|
5406
5441
|
return fetched;
|
|
5407
5442
|
}
|
|
5443
|
+
async function removeMismatchedPinnedFiles(profile, opts) {
|
|
5444
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
5445
|
+
const removed = [];
|
|
5446
|
+
for (const file of pinnedLoadFiles(profile, opts.modalities)) {
|
|
5447
|
+
if (!file.digest) continue;
|
|
5448
|
+
const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
|
|
5449
|
+
const digest = await fileDigest(path);
|
|
5450
|
+
if (digest === null || digest === file.digest) continue;
|
|
5451
|
+
await rm2(path, { force: true });
|
|
5452
|
+
removed.push(file.path);
|
|
5453
|
+
}
|
|
5454
|
+
return removed;
|
|
5455
|
+
}
|
|
5408
5456
|
|
|
5409
5457
|
// src/embedding/transformers-import.ts
|
|
5410
5458
|
var TRANSFORMERS_MODULE = "@huggingface/transformers";
|
|
@@ -5534,6 +5582,24 @@ function encoderRows(outputs, attentionMask) {
|
|
|
5534
5582
|
return sum.map((v) => v / Math.max(1, count));
|
|
5535
5583
|
});
|
|
5536
5584
|
}
|
|
5585
|
+
function isCorruptCacheFailure(error) {
|
|
5586
|
+
const messages = [];
|
|
5587
|
+
let current = error;
|
|
5588
|
+
for (let depth = 0; depth < 6 && current; depth++) {
|
|
5589
|
+
messages.push(current instanceof Error ? current.message : String(current));
|
|
5590
|
+
current = current instanceof Error ? current.cause : void 0;
|
|
5591
|
+
}
|
|
5592
|
+
return /(?:protobuf parsing failed|failed to parse protobuf|invalid protobuf|unexpected end of (?:file|data|json input)|failed to load model|load model .* failed|invalid wire type)/i.test(
|
|
5593
|
+
messages.join(" ")
|
|
5594
|
+
);
|
|
5595
|
+
}
|
|
5596
|
+
async function settleBoth(runner, tokenizer) {
|
|
5597
|
+
const [r, t] = await Promise.allSettled([runner, tokenizer]);
|
|
5598
|
+
if (r.status === "fulfilled" && t.status === "fulfilled") return [r.value, t.value];
|
|
5599
|
+
if (r.status === "fulfilled") await r.value.dispose?.().catch(() => {
|
|
5600
|
+
});
|
|
5601
|
+
throw r.status === "rejected" ? r.reason : t.reason;
|
|
5602
|
+
}
|
|
5537
5603
|
async function loadTransformers() {
|
|
5538
5604
|
try {
|
|
5539
5605
|
const specifier = TRANSFORMERS_MODULE;
|
|
@@ -5601,7 +5667,7 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
5601
5667
|
);
|
|
5602
5668
|
}
|
|
5603
5669
|
const externalData = pinnedExternalDataChunks(profile);
|
|
5604
|
-
const
|
|
5670
|
+
const loadSessions = () => settleBoth(
|
|
5605
5671
|
multimodal ? transformers.AutoModel.from_pretrained(
|
|
5606
5672
|
profile.model.repo,
|
|
5607
5673
|
{
|
|
@@ -5624,7 +5690,24 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
5624
5690
|
...localOnly,
|
|
5625
5691
|
...progress
|
|
5626
5692
|
})
|
|
5627
|
-
|
|
5693
|
+
);
|
|
5694
|
+
let runner;
|
|
5695
|
+
let tokenizer;
|
|
5696
|
+
try {
|
|
5697
|
+
[runner, tokenizer] = await loadSessions();
|
|
5698
|
+
} catch (err) {
|
|
5699
|
+
const repairDir = cacheDir ?? transformers.env.cacheDir;
|
|
5700
|
+
if (!repairDir || !isCorruptCacheFailure(err)) throw err;
|
|
5701
|
+
const removed = await removeMismatchedPinnedFiles(profile, { cacheDir: repairDir });
|
|
5702
|
+
if (removed.length === 0) throw err;
|
|
5703
|
+
if (opts.localFilesOnly) {
|
|
5704
|
+
throw new EmbedderUnavailableError(
|
|
5705
|
+
`the cached model files for profile ${profile.id} were damaged (${removed.join(", ")}) and have been removed; they download again on the next load that may use the network`
|
|
5706
|
+
);
|
|
5707
|
+
}
|
|
5708
|
+
if (!opts.transformers) await preparePinnedLoad(profile, { cacheDir: repairDir });
|
|
5709
|
+
[runner, tokenizer] = await loadSessions();
|
|
5710
|
+
}
|
|
5628
5711
|
let verification;
|
|
5629
5712
|
try {
|
|
5630
5713
|
verification = await verifyProfileArtifacts(profile, {
|
|
@@ -7248,6 +7331,7 @@ export {
|
|
|
7248
7331
|
exportCatalogParquet,
|
|
7249
7332
|
extractGezkVerified,
|
|
7250
7333
|
fetchKnowledgeRegistry,
|
|
7334
|
+
fileDigest,
|
|
7251
7335
|
findDuckdbBinary,
|
|
7252
7336
|
findRegistryEntry,
|
|
7253
7337
|
formatAtLeast,
|
|
@@ -7261,6 +7345,7 @@ export {
|
|
|
7261
7345
|
isSupportedIndexSchemaVersion,
|
|
7262
7346
|
knowledgeEmbeddingProfile,
|
|
7263
7347
|
knowledgeKeyId,
|
|
7348
|
+
knowledgeProfileQueryModelBytes,
|
|
7264
7349
|
l2Normalize,
|
|
7265
7350
|
loadMarkdownCatalog,
|
|
7266
7351
|
manifestSigningPayload,
|
|
@@ -7284,6 +7369,7 @@ export {
|
|
|
7284
7369
|
quantizeInt8,
|
|
7285
7370
|
readGezkManifest,
|
|
7286
7371
|
readMkdocsDocsDir,
|
|
7372
|
+
removeMismatchedPinnedFiles,
|
|
7287
7373
|
rerankK,
|
|
7288
7374
|
rerankScore,
|
|
7289
7375
|
resolveOutlinePath,
|
package/package.json
CHANGED