@bendyline/gezel-knowledge 1.2.6 → 1.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -475,8 +475,8 @@ declare function openCatalogDatabase(absPath: string): CatalogDb;
475
475
  * Read-only document point index. The router's ordinary SQLite latitude and
476
476
  * longitude indexes provide conservative candidates; the portable gezk
477
477
  * spherical predicate decides membership before ranking or pagination.
478
- * The explicit Qualla adapter keeps already-published 0.6 regional archives
479
- * useful without interpreting arbitrary producer metadata as coordinates.
478
+ * Archives without a `document_locations` table (before 0.7) have no
479
+ * locations: opaque producer metadata is never read as coordinates.
480
480
  */
481
481
 
482
482
  interface SpatialMatch {
@@ -517,6 +517,14 @@ interface CatalogDocumentMeta {
517
517
  meta: Record<string, unknown> | null;
518
518
  locations?: KnowledgeLocation[];
519
519
  }
520
+ /** One slug initial of a listing: where it first appears and how many documents share it. */
521
+ interface CatalogDocumentInitial {
522
+ /** The slug's first character, as stored (slugs are usually lowercase ASCII). */
523
+ initial: string;
524
+ /** Zero-based position of its first document in `documentsPage` order. */
525
+ offset: number;
526
+ count: number;
527
+ }
520
528
  interface CatalogAssetInfo {
521
529
  /** Archive path under `assets/`. */
522
530
  path: string;
@@ -671,6 +679,26 @@ declare class CatalogHandle {
671
679
  documents: CatalogDocumentMeta[];
672
680
  total: number;
673
681
  };
682
+ /**
683
+ * Where each slug initial first appears in `documentsPage` order, for an
684
+ * A–Z index over the listing. Only the alphabetical part counts: documents
685
+ * with an ordinal are listed first in their curated order, so a fully
686
+ * curated listing has no initials.
687
+ */
688
+ documentInitials(opts?: {
689
+ topicId?: string;
690
+ descendants?: boolean;
691
+ }): CatalogDocumentInitial[];
692
+ /** A document's zero-based position in `documentsPage` order; null when the listing lacks it. */
693
+ documentPosition(documentId: string, opts?: {
694
+ topicId?: string;
695
+ descendants?: boolean;
696
+ }): number | null;
697
+ /** A listing's documents as `id, slug, ordinal` rows, ordered by LISTING_ORDER. */
698
+ private listingRows;
699
+ private plainScope;
700
+ /** The topic's placements (schema 4) as `scoped`; binds the topic id once. */
701
+ private sharedScope;
674
702
  /** A reference is a placement, not a duplicate result or a redirect page. */
675
703
  private sharedDocumentsPage;
676
704
  /** Integrity facts kept separate from browse counts, which include shared references. */
@@ -957,7 +985,7 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
957
985
  * bytes.
958
986
  */
959
987
 
960
- /** 4: document-locations companion, including the explicit legacy Qualla adapter. */
988
+ /** 4: document-locations companion. */
961
989
  declare const PARQUET_EXPORT_VERSION = 4;
962
990
  type ParquetExportSource = {
963
991
  archivePath: string;
@@ -1062,6 +1090,8 @@ declare const EMBEDDINGGEMMA_2_512_1: KnowledgeEmbeddingProfile;
1062
1090
  /** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
1063
1091
  declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
1064
1092
  declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
1093
+ /** What a profile's query model costs to download, or null for an unregistered profile. */
1094
+ declare function knowledgeProfileQueryModelBytes(id: string): number | null;
1065
1095
  /** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
1066
1096
  declare const MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
1067
1097
  /** @deprecated gezk 0.4 names; removed with the next minor. */
@@ -1274,6 +1304,8 @@ declare class EmbedderArtifactError extends Error {
1274
1304
  }
1275
1305
  /** The transformers.js FileCache key for one repo file. */
1276
1306
  declare function transformersCachePath(cacheDir: string, repo: string, revision: string, file: string): string;
1307
+ /** `sha256:<hex>` of a file, or null when it is absent. Memoized by path, size and mtime. */
1308
+ declare function fileDigest(path: string): Promise<string | null>;
1277
1309
  /** The pinned files of a profile's media encoders for the given modalities (video reuses image). */
1278
1310
  declare function mediaEncoderPins(profile: KnowledgeEmbeddingProfile, modalities: ReadonlyArray<'image' | 'video' | 'audio'>): Array<{
1279
1311
  file: string;
@@ -1505,6 +1537,19 @@ declare function prefetchExternalData(profile: KnowledgeEmbeddingProfile, opts:
1505
1537
  revision?: string;
1506
1538
  fetchImpl?: typeof fetch;
1507
1539
  }): Promise<string[]>;
1540
+ /**
1541
+ * Remove the cached pinned files whose bytes do not match their pins, and
1542
+ * return their paths. transformers.js streams a download straight into its
1543
+ * cache path, so one that is cut off stays there as a truncated file that
1544
+ * every later load fails to parse, until someone deletes it by hand. Only for
1545
+ * a load that already failed that way: it hashes every pinned file present,
1546
+ * which a healthy load never pays for.
1547
+ */
1548
+ declare function removeMismatchedPinnedFiles(profile: KnowledgeEmbeddingProfile, opts: {
1549
+ cacheDir: string;
1550
+ modalities?: readonly MediaModality[];
1551
+ revision?: string;
1552
+ }): Promise<string[]>;
1508
1553
 
1509
1554
  /**
1510
1555
  * Tables of contents that documentation trees already carry. Each parser
@@ -1763,4 +1808,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
1763
1808
 
1764
1809
  declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
1765
1810
 
1766
- export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
1811
+ export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentInitial, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, fileDigest, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, knowledgeProfileQueryModelBytes, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, removeMismatchedPinnedFiles, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
package/dist/index.js CHANGED
@@ -2195,23 +2195,19 @@ import {
2195
2195
  KnowledgeLocationsSchema as KnowledgeLocationsSchema2,
2196
2196
  KnowledgeRadiusSchema,
2197
2197
  locationDistanceMeters,
2198
- locationInBounds,
2199
2198
  normalizeLongitude as normalizeLongitude2,
2200
2199
  radiusBounds,
2201
2200
  spatialManifest as spatialManifest2
2202
2201
  } from "@bendyline/gezk";
2203
2202
  var DocumentSpatialIndex = class {
2204
- constructor(db, allowLegacy) {
2203
+ constructor(db) {
2205
2204
  this.db = db;
2206
- this.allowLegacy = allowLegacy;
2207
2205
  this.hasTable = Boolean(
2208
2206
  db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='document_locations'").get()
2209
2207
  );
2210
2208
  }
2211
2209
  db;
2212
- allowLegacy;
2213
2210
  hasTable;
2214
- legacy;
2215
2211
  decode(row) {
2216
2212
  const provenance = row.provenance_json;
2217
2213
  if (typeof provenance === "string" && Buffer.byteLength(provenance) > 16384)
@@ -2227,39 +2223,13 @@ var DocumentSpatialIndex = class {
2227
2223
  })
2228
2224
  };
2229
2225
  }
2230
- legacyRows() {
2231
- if (this.legacy) return this.legacy;
2232
- this.legacy = [];
2233
- if (!this.allowLegacy) return this.legacy;
2234
- for (const row of this.db.prepare(
2235
- "SELECT id, meta_json FROM documents WHERE id LIKE 'qualla-%' AND meta_json IS NOT NULL"
2236
- ).all()) {
2237
- try {
2238
- const meta = JSON.parse(String(row.meta_json));
2239
- if (typeof meta.region !== "string" || !Array.isArray(meta.quallaArticleIds) || !meta.coordinates)
2240
- continue;
2241
- const point = KnowledgeLocationSchema.safeParse({
2242
- id: "qualla-coordinate",
2243
- role: "subject",
2244
- latitude: meta.coordinates.lat,
2245
- longitude: meta.coordinates.lng,
2246
- provenance: { adapter: "qualla-regional-meta@1", articleIds: meta.quallaArticleIds }
2247
- });
2248
- if (point.success)
2249
- this.legacy.push({
2250
- documentId: String(row.id),
2251
- location: { ...point.data, longitude: normalizeLongitude2(point.data.longitude) }
2252
- });
2253
- } catch {
2254
- }
2255
- }
2256
- return this.legacy;
2257
- }
2258
2226
  rows() {
2259
- return this.hasTable ? this.db.prepare("SELECT * FROM document_locations ORDER BY document_id, location_id").all().map((row) => this.decode(row)) : this.legacyRows();
2227
+ if (!this.hasTable) return [];
2228
+ return this.db.prepare("SELECT * FROM document_locations ORDER BY document_id, location_id").all().map((row) => this.decode(row));
2260
2229
  }
2261
2230
  locations(documentId) {
2262
- return this.hasTable ? this.db.prepare("SELECT * FROM document_locations WHERE document_id = ? ORDER BY location_id").all(documentId).map((row) => this.decode(row).location) : this.legacyRows().filter((row) => row.documentId === documentId).map((row) => row.location);
2231
+ if (!this.hasTable) return [];
2232
+ return this.db.prepare("SELECT * FROM document_locations WHERE document_id = ? ORDER BY location_id").all(documentId).map((row) => this.decode(row).location);
2263
2233
  }
2264
2234
  integrity() {
2265
2235
  const rows = this.rows();
@@ -2278,14 +2248,12 @@ var DocumentSpatialIndex = class {
2278
2248
  }
2279
2249
  matches(raw) {
2280
2250
  const radius = KnowledgeRadiusSchema.parse(raw);
2281
- const boxes = radiusBounds(radius);
2282
- const candidates = this.hasTable ? boxes.flatMap(
2251
+ const matches = /* @__PURE__ */ new Map();
2252
+ if (!this.hasTable) return matches;
2253
+ const candidates = radiusBounds(radius).flatMap(
2283
2254
  (box) => this.db.prepare(`SELECT * FROM document_locations
2284
2255
  WHERE role = 'subject' AND latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ?`).all(box.south, box.north, box.west, box.east).map((row) => this.decode(row))
2285
- ) : this.legacyRows().filter(
2286
- (row) => row.location.role === "subject" && boxes.some((box) => locationInBounds(row.location, box))
2287
2256
  );
2288
- const matches = /* @__PURE__ */ new Map();
2289
2257
  for (const { documentId, location } of candidates) {
2290
2258
  const distanceMeters = locationDistanceMeters(radius, location);
2291
2259
  if (distanceMeters > radius.radiusMeters) continue;
@@ -2300,6 +2268,11 @@ var DocumentSpatialIndex = class {
2300
2268
 
2301
2269
  // src/reader/catalog-handle.ts
2302
2270
  var TOPIC_WALK_MAX_DEPTH = 32;
2271
+ var LISTING_ORDER = "(ordinal IS NULL), ordinal, slug, id";
2272
+ var SHARED_PLACEMENT_RANK = `ranked AS (
2273
+ SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
2274
+ ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
2275
+ )`;
2303
2276
  var CHUNK_COLUMNS = "chunk_uid, document_id, title, heading_path, line_start, line_end, text";
2304
2277
  var MEDIA_COLUMNS = "modality, asset_path, mime_type, width, height, start_ms, end_ms, thumbnail_path, attribution_json";
2305
2278
  function mediaOf(row) {
@@ -2342,7 +2315,7 @@ var CatalogHandle = class _CatalogHandle {
2342
2315
  this.rootDir = rootDir;
2343
2316
  this.router = router;
2344
2317
  this.schemaVersion = router.schemaVersion;
2345
- this.spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
2318
+ this.spatial = new DocumentSpatialIndex(router.db);
2346
2319
  this.meta = {};
2347
2320
  for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
2348
2321
  this.meta[row.key] = row.value;
@@ -2690,22 +2663,7 @@ var CatalogHandle = class _CatalogHandle {
2690
2663
  if (this.schemaVersion >= 4 && opts.topicId) {
2691
2664
  return this.sharedDocumentsPage(opts.topicId, descendants, limit, offset);
2692
2665
  }
2693
- let scope = "";
2694
- let where = "";
2695
- const params = [];
2696
- if (opts.topicId && descendants) {
2697
- scope = `WITH RECURSIVE sub(id, depth) AS (
2698
- SELECT ?, 0
2699
- UNION ALL
2700
- SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
2701
- WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
2702
- )`;
2703
- where = "WHERE topic_id IN (SELECT id FROM sub)";
2704
- params.push(opts.topicId);
2705
- } else if (opts.topicId) {
2706
- where = "WHERE topic_id = ?";
2707
- params.push(opts.topicId);
2708
- }
2666
+ const { scope, where, params } = this.plainScope(opts.topicId, descendants);
2709
2667
  const total = Number(
2710
2668
  this.router.db.prepare(`${scope} SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
2711
2669
  );
@@ -2715,23 +2673,88 @@ var CatalogHandle = class _CatalogHandle {
2715
2673
  ).all(...params, limit, offset).map((row) => this.documentMeta(row));
2716
2674
  return { documents, total };
2717
2675
  }
2718
- /** A reference is a placement, not a duplicate result or a redirect page. */
2719
- sharedDocumentsPage(topicId, descendants, limit, offset) {
2676
+ /**
2677
+ * Where each slug initial first appears in `documentsPage` order, for an
2678
+ * A–Z index over the listing. Only the alphabetical part counts: documents
2679
+ * with an ordinal are listed first in their curated order, so a fully
2680
+ * curated listing has no initials.
2681
+ */
2682
+ documentInitials(opts = {}) {
2683
+ const { sql, params } = this.listingRows(opts.topicId, opts.descendants ?? true);
2684
+ const rows = this.router.db.prepare(
2685
+ `SELECT initial, MIN(position) AS position, COUNT(*) AS n FROM (
2686
+ SELECT substr(slug, 1, 1) AS initial, ordinal,
2687
+ ROW_NUMBER() OVER (ORDER BY ${LISTING_ORDER}) - 1 AS position
2688
+ FROM (${sql})
2689
+ ) WHERE ordinal IS NULL GROUP BY initial ORDER BY position`
2690
+ ).all(...params);
2691
+ return rows.map((r) => ({
2692
+ initial: r.initial,
2693
+ offset: Number(r.position),
2694
+ count: Number(r.n)
2695
+ }));
2696
+ }
2697
+ /** A document's zero-based position in `documentsPage` order; null when the listing lacks it. */
2698
+ documentPosition(documentId, opts = {}) {
2699
+ const { sql, params } = this.listingRows(opts.topicId, opts.descendants ?? true);
2700
+ const row = this.router.db.prepare(
2701
+ `SELECT position FROM (
2702
+ SELECT id, ROW_NUMBER() OVER (ORDER BY ${LISTING_ORDER}) - 1 AS position FROM (${sql})
2703
+ ) WHERE id = ?`
2704
+ ).get(...params, documentId);
2705
+ return row ? Number(row.position) : null;
2706
+ }
2707
+ /** A listing's documents as `id, slug, ordinal` rows, ordered by LISTING_ORDER. */
2708
+ listingRows(topicId, descendants) {
2709
+ if (this.schemaVersion >= 4 && topicId) {
2710
+ return {
2711
+ sql: `${this.sharedScope(descendants)}, ${SHARED_PLACEMENT_RANK}
2712
+ SELECT d.id AS id, d.slug AS slug, p.ordinal AS ordinal
2713
+ FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1`,
2714
+ params: [topicId]
2715
+ };
2716
+ }
2717
+ const { scope, where, params } = this.plainScope(topicId, descendants);
2718
+ const ordinal = this.schemaVersion >= 3 ? "ordinal" : "NULL";
2719
+ return {
2720
+ sql: `${scope} SELECT id, slug, ${ordinal} AS ordinal FROM documents ${where}`,
2721
+ params
2722
+ };
2723
+ }
2724
+ plainScope(topicId, descendants) {
2725
+ if (topicId && descendants) {
2726
+ return {
2727
+ scope: `WITH RECURSIVE sub(id, depth) AS (
2728
+ SELECT ?, 0
2729
+ UNION ALL
2730
+ SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
2731
+ WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
2732
+ )`,
2733
+ where: "WHERE topic_id IN (SELECT id FROM sub)",
2734
+ params: [topicId]
2735
+ };
2736
+ }
2737
+ if (topicId) return { scope: "", where: "WHERE topic_id = ?", params: [topicId] };
2738
+ return { scope: "", where: "", params: [] };
2739
+ }
2740
+ /** The topic's placements (schema 4) as `scoped`; binds the topic id once. */
2741
+ sharedScope(descendants) {
2720
2742
  const sub = descendants ? `SELECT ?, 0 UNION ALL SELECT t.id, sub.depth + 1 FROM topics t
2721
2743
  JOIN sub ON t.parent_id = sub.id WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}` : "SELECT ?, 0";
2722
- const scope = `WITH RECURSIVE sub(id, depth) AS (${sub}), scoped AS (
2744
+ return `WITH RECURSIVE sub(id, depth) AS (${sub}), scoped AS (
2723
2745
  SELECT td.document_id, td.topic_id, td.ordinal, t.sort_key
2724
2746
  FROM topic_documents td JOIN topics t ON t.id = td.topic_id
2725
2747
  WHERE td.topic_id IN (SELECT id FROM sub)
2726
2748
  )`;
2749
+ }
2750
+ /** A reference is a placement, not a duplicate result or a redirect page. */
2751
+ sharedDocumentsPage(topicId, descendants, limit, offset) {
2752
+ const scope = this.sharedScope(descendants);
2727
2753
  const total = Number(
2728
2754
  this.router.db.prepare(`${scope} SELECT COUNT(DISTINCT document_id) AS n FROM scoped`).get(topicId).n
2729
2755
  );
2730
2756
  const documents = this.router.db.prepare(
2731
- `${scope}, ranked AS (
2732
- SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
2733
- ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
2734
- ) SELECT ${this.documentColumns("d", "p")}
2757
+ `${scope}, ${SHARED_PLACEMENT_RANK} SELECT ${this.documentColumns("d", "p")}
2735
2758
  FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1
2736
2759
  ORDER BY (p.ordinal IS NULL), p.ordinal, d.slug, d.id LIMIT ? OFFSET ?`
2737
2760
  ).all(topicId, limit, offset).map((row) => this.documentMeta(row));
@@ -3725,9 +3748,9 @@ async function exportCatalogParquet(opts) {
3725
3748
  onProgress: opts.onProgress
3726
3749
  })
3727
3750
  );
3728
- const spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
3751
+ const spatial = new DocumentSpatialIndex(router.db);
3729
3752
  const locations = spatial.rows();
3730
- if (spatial.hasTable || locations.length) {
3753
+ if (spatial.hasTable) {
3731
3754
  const path = join4(staging, "document-locations.ndjson");
3732
3755
  const out = ndjsonWriter(path);
3733
3756
  for (const { documentId, location } of locations)
@@ -3741,7 +3764,10 @@ async function exportCatalogParquet(opts) {
3741
3764
  });
3742
3765
  await out.close();
3743
3766
  const columns = "{document_id:'VARCHAR',location_id:'VARCHAR',latitude:'DOUBLE',longitude:'DOUBLE',role:'VARCHAR',provenance_json:'VARCHAR'}";
3744
- const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : "(SELECT NULL::VARCHAR document_id, NULL::VARCHAR location_id, NULL::DOUBLE latitude, NULL::DOUBLE longitude, NULL::VARCHAR role, NULL::VARCHAR provenance_json WHERE false)";
3767
+ const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : (
3768
+ // `role` is a DuckDB keyword: a bare alias is a parser error.
3769
+ '(SELECT NULL::VARCHAR AS "document_id", NULL::VARCHAR AS "location_id", NULL::DOUBLE AS "latitude", NULL::DOUBLE AS "longitude", NULL::VARCHAR AS "role", NULL::VARCHAR AS "provenance_json" WHERE false)'
3770
+ );
3745
3771
  files.push(
3746
3772
  await writeParquet(opts.duckdb, {
3747
3773
  outDir,
@@ -5183,6 +5209,15 @@ var KNOWLEDGE_EMBEDDING_PROFILES = [
5183
5209
  function knowledgeEmbeddingProfile(id) {
5184
5210
  return KNOWLEDGE_EMBEDDING_PROFILES.find((p) => p.id === id) ?? null;
5185
5211
  }
5212
+ var QUERY_MODEL_DOWNLOAD_BYTES = {
5213
+ "embeddinggemma-2-512@1": 346397233,
5214
+ "multilingual-e5-small@2": 487352364,
5215
+ "multilingual-e5-small@1": 487352364,
5216
+ "bge-small-en-v1.5@1": 133805935
5217
+ };
5218
+ function knowledgeProfileQueryModelBytes(id) {
5219
+ return QUERY_MODEL_DOWNLOAD_BYTES[id] ?? null;
5220
+ }
5186
5221
  var MARKDOWN_CHUNKS_2 = {
5187
5222
  id: "markdown-chunks@2",
5188
5223
  unit: "tokens",
@@ -5405,6 +5440,19 @@ async function prefetchExternalData(profile, opts) {
5405
5440
  }
5406
5441
  return fetched;
5407
5442
  }
5443
+ async function removeMismatchedPinnedFiles(profile, opts) {
5444
+ const revision = opts.revision ?? profile.model.revision;
5445
+ const removed = [];
5446
+ for (const file of pinnedLoadFiles(profile, opts.modalities)) {
5447
+ if (!file.digest) continue;
5448
+ const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
5449
+ const digest = await fileDigest(path);
5450
+ if (digest === null || digest === file.digest) continue;
5451
+ await rm2(path, { force: true });
5452
+ removed.push(file.path);
5453
+ }
5454
+ return removed;
5455
+ }
5408
5456
 
5409
5457
  // src/embedding/transformers-import.ts
5410
5458
  var TRANSFORMERS_MODULE = "@huggingface/transformers";
@@ -5534,6 +5582,24 @@ function encoderRows(outputs, attentionMask) {
5534
5582
  return sum.map((v) => v / Math.max(1, count));
5535
5583
  });
5536
5584
  }
5585
+ function isCorruptCacheFailure(error) {
5586
+ const messages = [];
5587
+ let current = error;
5588
+ for (let depth = 0; depth < 6 && current; depth++) {
5589
+ messages.push(current instanceof Error ? current.message : String(current));
5590
+ current = current instanceof Error ? current.cause : void 0;
5591
+ }
5592
+ return /(?:protobuf parsing failed|failed to parse protobuf|invalid protobuf|unexpected end of (?:file|data|json input)|failed to load model|load model .* failed|invalid wire type)/i.test(
5593
+ messages.join(" ")
5594
+ );
5595
+ }
5596
+ async function settleBoth(runner, tokenizer) {
5597
+ const [r, t] = await Promise.allSettled([runner, tokenizer]);
5598
+ if (r.status === "fulfilled" && t.status === "fulfilled") return [r.value, t.value];
5599
+ if (r.status === "fulfilled") await r.value.dispose?.().catch(() => {
5600
+ });
5601
+ throw r.status === "rejected" ? r.reason : t.reason;
5602
+ }
5537
5603
  async function loadTransformers() {
5538
5604
  try {
5539
5605
  const specifier = TRANSFORMERS_MODULE;
@@ -5601,7 +5667,7 @@ async function createProfileEmbedder(profile, opts = {}) {
5601
5667
  );
5602
5668
  }
5603
5669
  const externalData = pinnedExternalDataChunks(profile);
5604
- const [runner, tokenizer] = await Promise.all([
5670
+ const loadSessions = () => settleBoth(
5605
5671
  multimodal ? transformers.AutoModel.from_pretrained(
5606
5672
  profile.model.repo,
5607
5673
  {
@@ -5624,7 +5690,24 @@ async function createProfileEmbedder(profile, opts = {}) {
5624
5690
  ...localOnly,
5625
5691
  ...progress
5626
5692
  })
5627
- ]);
5693
+ );
5694
+ let runner;
5695
+ let tokenizer;
5696
+ try {
5697
+ [runner, tokenizer] = await loadSessions();
5698
+ } catch (err) {
5699
+ const repairDir = cacheDir ?? transformers.env.cacheDir;
5700
+ if (!repairDir || !isCorruptCacheFailure(err)) throw err;
5701
+ const removed = await removeMismatchedPinnedFiles(profile, { cacheDir: repairDir });
5702
+ if (removed.length === 0) throw err;
5703
+ if (opts.localFilesOnly) {
5704
+ throw new EmbedderUnavailableError(
5705
+ `the cached model files for profile ${profile.id} were damaged (${removed.join(", ")}) and have been removed; they download again on the next load that may use the network`
5706
+ );
5707
+ }
5708
+ if (!opts.transformers) await preparePinnedLoad(profile, { cacheDir: repairDir });
5709
+ [runner, tokenizer] = await loadSessions();
5710
+ }
5628
5711
  let verification;
5629
5712
  try {
5630
5713
  verification = await verifyProfileArtifacts(profile, {
@@ -7248,6 +7331,7 @@ export {
7248
7331
  exportCatalogParquet,
7249
7332
  extractGezkVerified,
7250
7333
  fetchKnowledgeRegistry,
7334
+ fileDigest,
7251
7335
  findDuckdbBinary,
7252
7336
  findRegistryEntry,
7253
7337
  formatAtLeast,
@@ -7261,6 +7345,7 @@ export {
7261
7345
  isSupportedIndexSchemaVersion,
7262
7346
  knowledgeEmbeddingProfile,
7263
7347
  knowledgeKeyId,
7348
+ knowledgeProfileQueryModelBytes,
7264
7349
  l2Normalize,
7265
7350
  loadMarkdownCatalog,
7266
7351
  manifestSigningPayload,
@@ -7284,6 +7369,7 @@ export {
7284
7369
  quantizeInt8,
7285
7370
  readGezkManifest,
7286
7371
  readMkdocsDocsDir,
7372
+ removeMismatchedPinnedFiles,
7287
7373
  rerankK,
7288
7374
  rerankScore,
7289
7375
  resolveOutlinePath,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bendyline/gezel-knowledge",
3
- "version": "1.2.6",
3
+ "version": "1.2.8",
4
4
  "description": "Gezel knowledge catalogs: the .gezk compiler, archive safety, read-only reader, routing, and signatures.",
5
5
  "license": "MIT",
6
6
  "keywords": [