@bendyline/gezel-knowledge 1.2.7 → 1.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -1304,6 +1304,8 @@ declare class EmbedderArtifactError extends Error {
1304
1304
  }
1305
1305
  /** The transformers.js FileCache key for one repo file. */
1306
1306
  declare function transformersCachePath(cacheDir: string, repo: string, revision: string, file: string): string;
1307
+ /** `sha256:<hex>` of a file, or null when it is absent. Memoized by path, size and mtime. */
1308
+ declare function fileDigest(path: string): Promise<string | null>;
1307
1309
  /** The pinned files of a profile's media encoders for the given modalities (video reuses image). */
1308
1310
  declare function mediaEncoderPins(profile: KnowledgeEmbeddingProfile, modalities: ReadonlyArray<'image' | 'video' | 'audio'>): Array<{
1309
1311
  file: string;
@@ -1535,6 +1537,19 @@ declare function prefetchExternalData(profile: KnowledgeEmbeddingProfile, opts:
1535
1537
  revision?: string;
1536
1538
  fetchImpl?: typeof fetch;
1537
1539
  }): Promise<string[]>;
1540
+ /**
1541
+ * Remove the cached pinned files whose bytes do not match their pins, and
1542
+ * return their paths. transformers.js streams a download straight into its
1543
+ * cache path, so one that is cut off stays there as a truncated file that
1544
+ * every later load fails to parse, until someone deletes it by hand. Only for
1545
+ * a load that already failed that way: it hashes every pinned file present,
1546
+ * which a healthy load never pays for.
1547
+ */
1548
+ declare function removeMismatchedPinnedFiles(profile: KnowledgeEmbeddingProfile, opts: {
1549
+ cacheDir: string;
1550
+ modalities?: readonly MediaModality[];
1551
+ revision?: string;
1552
+ }): Promise<string[]>;
1538
1553
 
1539
1554
  /**
1540
1555
  * Tables of contents that documentation trees already carry. Each parser
@@ -1793,4 +1808,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
1793
1808
 
1794
1809
  declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
1795
1810
 
1796
- export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentInitial, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, knowledgeProfileQueryModelBytes, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
1811
+ export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentInitial, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, fileDigest, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, knowledgeProfileQueryModelBytes, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, removeMismatchedPinnedFiles, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
package/dist/index.js CHANGED
@@ -3764,7 +3764,10 @@ async function exportCatalogParquet(opts) {
3764
3764
  });
3765
3765
  await out.close();
3766
3766
  const columns = "{document_id:'VARCHAR',location_id:'VARCHAR',latitude:'DOUBLE',longitude:'DOUBLE',role:'VARCHAR',provenance_json:'VARCHAR'}";
3767
- const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : "(SELECT NULL::VARCHAR document_id, NULL::VARCHAR location_id, NULL::DOUBLE latitude, NULL::DOUBLE longitude, NULL::VARCHAR role, NULL::VARCHAR provenance_json WHERE false)";
3767
+ const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : (
3768
+ // `role` is a DuckDB keyword: a bare alias is a parser error.
3769
+ '(SELECT NULL::VARCHAR AS "document_id", NULL::VARCHAR AS "location_id", NULL::DOUBLE AS "latitude", NULL::DOUBLE AS "longitude", NULL::VARCHAR AS "role", NULL::VARCHAR AS "provenance_json" WHERE false)'
3770
+ );
3768
3771
  files.push(
3769
3772
  await writeParquet(opts.duckdb, {
3770
3773
  outDir,
@@ -5437,6 +5440,19 @@ async function prefetchExternalData(profile, opts) {
5437
5440
  }
5438
5441
  return fetched;
5439
5442
  }
5443
+ async function removeMismatchedPinnedFiles(profile, opts) {
5444
+ const revision = opts.revision ?? profile.model.revision;
5445
+ const removed = [];
5446
+ for (const file of pinnedLoadFiles(profile, opts.modalities)) {
5447
+ if (!file.digest) continue;
5448
+ const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
5449
+ const digest = await fileDigest(path);
5450
+ if (digest === null || digest === file.digest) continue;
5451
+ await rm2(path, { force: true });
5452
+ removed.push(file.path);
5453
+ }
5454
+ return removed;
5455
+ }
5440
5456
 
5441
5457
  // src/embedding/transformers-import.ts
5442
5458
  var TRANSFORMERS_MODULE = "@huggingface/transformers";
@@ -5566,6 +5582,24 @@ function encoderRows(outputs, attentionMask) {
5566
5582
  return sum.map((v) => v / Math.max(1, count));
5567
5583
  });
5568
5584
  }
5585
+ function isCorruptCacheFailure(error) {
5586
+ const messages = [];
5587
+ let current = error;
5588
+ for (let depth = 0; depth < 6 && current; depth++) {
5589
+ messages.push(current instanceof Error ? current.message : String(current));
5590
+ current = current instanceof Error ? current.cause : void 0;
5591
+ }
5592
+ return /(?:protobuf parsing failed|failed to parse protobuf|invalid protobuf|unexpected end of (?:file|data|json input)|failed to load model|load model .* failed|invalid wire type)/i.test(
5593
+ messages.join(" ")
5594
+ );
5595
+ }
5596
+ async function settleBoth(runner, tokenizer) {
5597
+ const [r, t] = await Promise.allSettled([runner, tokenizer]);
5598
+ if (r.status === "fulfilled" && t.status === "fulfilled") return [r.value, t.value];
5599
+ if (r.status === "fulfilled") await r.value.dispose?.().catch(() => {
5600
+ });
5601
+ throw r.status === "rejected" ? r.reason : t.reason;
5602
+ }
5569
5603
  async function loadTransformers() {
5570
5604
  try {
5571
5605
  const specifier = TRANSFORMERS_MODULE;
@@ -5633,7 +5667,7 @@ async function createProfileEmbedder(profile, opts = {}) {
5633
5667
  );
5634
5668
  }
5635
5669
  const externalData = pinnedExternalDataChunks(profile);
5636
- const [runner, tokenizer] = await Promise.all([
5670
+ const loadSessions = () => settleBoth(
5637
5671
  multimodal ? transformers.AutoModel.from_pretrained(
5638
5672
  profile.model.repo,
5639
5673
  {
@@ -5656,7 +5690,24 @@ async function createProfileEmbedder(profile, opts = {}) {
5656
5690
  ...localOnly,
5657
5691
  ...progress
5658
5692
  })
5659
- ]);
5693
+ );
5694
+ let runner;
5695
+ let tokenizer;
5696
+ try {
5697
+ [runner, tokenizer] = await loadSessions();
5698
+ } catch (err) {
5699
+ const repairDir = cacheDir ?? transformers.env.cacheDir;
5700
+ if (!repairDir || !isCorruptCacheFailure(err)) throw err;
5701
+ const removed = await removeMismatchedPinnedFiles(profile, { cacheDir: repairDir });
5702
+ if (removed.length === 0) throw err;
5703
+ if (opts.localFilesOnly) {
5704
+ throw new EmbedderUnavailableError(
5705
+ `the cached model files for profile ${profile.id} were damaged (${removed.join(", ")}) and have been removed; they download again on the next load that may use the network`
5706
+ );
5707
+ }
5708
+ if (!opts.transformers) await preparePinnedLoad(profile, { cacheDir: repairDir });
5709
+ [runner, tokenizer] = await loadSessions();
5710
+ }
5660
5711
  let verification;
5661
5712
  try {
5662
5713
  verification = await verifyProfileArtifacts(profile, {
@@ -7280,6 +7331,7 @@ export {
7280
7331
  exportCatalogParquet,
7281
7332
  extractGezkVerified,
7282
7333
  fetchKnowledgeRegistry,
7334
+ fileDigest,
7283
7335
  findDuckdbBinary,
7284
7336
  findRegistryEntry,
7285
7337
  formatAtLeast,
@@ -7317,6 +7369,7 @@ export {
7317
7369
  quantizeInt8,
7318
7370
  readGezkManifest,
7319
7371
  readMkdocsDocsDir,
7372
+ removeMismatchedPinnedFiles,
7320
7373
  rerankK,
7321
7374
  rerankScore,
7322
7375
  resolveOutlinePath,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bendyline/gezel-knowledge",
3
- "version": "1.2.7",
3
+ "version": "1.2.8",
4
4
  "description": "Gezel knowledge catalogs: the .gezk compiler, archive safety, read-only reader, routing, and signatures.",
5
5
  "license": "MIT",
6
6
  "keywords": [