@bendyline/gezel-knowledge 1.2.7 → 1.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +16 -1
- package/dist/index.js +56 -3
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -1304,6 +1304,8 @@ declare class EmbedderArtifactError extends Error {
|
|
|
1304
1304
|
}
|
|
1305
1305
|
/** The transformers.js FileCache key for one repo file. */
|
|
1306
1306
|
declare function transformersCachePath(cacheDir: string, repo: string, revision: string, file: string): string;
|
|
1307
|
+
/** `sha256:<hex>` of a file, or null when it is absent. Memoized by path, size and mtime. */
|
|
1308
|
+
declare function fileDigest(path: string): Promise<string | null>;
|
|
1307
1309
|
/** The pinned files of a profile's media encoders for the given modalities (video reuses image). */
|
|
1308
1310
|
declare function mediaEncoderPins(profile: KnowledgeEmbeddingProfile, modalities: ReadonlyArray<'image' | 'video' | 'audio'>): Array<{
|
|
1309
1311
|
file: string;
|
|
@@ -1535,6 +1537,19 @@ declare function prefetchExternalData(profile: KnowledgeEmbeddingProfile, opts:
|
|
|
1535
1537
|
revision?: string;
|
|
1536
1538
|
fetchImpl?: typeof fetch;
|
|
1537
1539
|
}): Promise<string[]>;
|
|
1540
|
+
/**
|
|
1541
|
+
* Remove the cached pinned files whose bytes do not match their pins, and
|
|
1542
|
+
* return their paths. transformers.js streams a download straight into its
|
|
1543
|
+
* cache path, so one that is cut off stays there as a truncated file that
|
|
1544
|
+
* every later load fails to parse, until someone deletes it by hand. Only for
|
|
1545
|
+
* a load that already failed that way: it hashes every pinned file present,
|
|
1546
|
+
* which a healthy load never pays for.
|
|
1547
|
+
*/
|
|
1548
|
+
declare function removeMismatchedPinnedFiles(profile: KnowledgeEmbeddingProfile, opts: {
|
|
1549
|
+
cacheDir: string;
|
|
1550
|
+
modalities?: readonly MediaModality[];
|
|
1551
|
+
revision?: string;
|
|
1552
|
+
}): Promise<string[]>;
|
|
1538
1553
|
|
|
1539
1554
|
/**
|
|
1540
1555
|
* Tables of contents that documentation trees already carry. Each parser
|
|
@@ -1793,4 +1808,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1793
1808
|
|
|
1794
1809
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1795
1810
|
|
|
1796
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentInitial, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, knowledgeProfileQueryModelBytes, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
|
|
1811
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentInitial, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, fileDigest, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, knowledgeProfileQueryModelBytes, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, removeMismatchedPinnedFiles, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -3764,7 +3764,10 @@ async function exportCatalogParquet(opts) {
|
|
|
3764
3764
|
});
|
|
3765
3765
|
await out.close();
|
|
3766
3766
|
const columns = "{document_id:'VARCHAR',location_id:'VARCHAR',latitude:'DOUBLE',longitude:'DOUBLE',role:'VARCHAR',provenance_json:'VARCHAR'}";
|
|
3767
|
-
const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` :
|
|
3767
|
+
const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : (
|
|
3768
|
+
// `role` is a DuckDB keyword: a bare alias is a parser error.
|
|
3769
|
+
'(SELECT NULL::VARCHAR AS "document_id", NULL::VARCHAR AS "location_id", NULL::DOUBLE AS "latitude", NULL::DOUBLE AS "longitude", NULL::VARCHAR AS "role", NULL::VARCHAR AS "provenance_json" WHERE false)'
|
|
3770
|
+
);
|
|
3768
3771
|
files.push(
|
|
3769
3772
|
await writeParquet(opts.duckdb, {
|
|
3770
3773
|
outDir,
|
|
@@ -5437,6 +5440,19 @@ async function prefetchExternalData(profile, opts) {
|
|
|
5437
5440
|
}
|
|
5438
5441
|
return fetched;
|
|
5439
5442
|
}
|
|
5443
|
+
async function removeMismatchedPinnedFiles(profile, opts) {
|
|
5444
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
5445
|
+
const removed = [];
|
|
5446
|
+
for (const file of pinnedLoadFiles(profile, opts.modalities)) {
|
|
5447
|
+
if (!file.digest) continue;
|
|
5448
|
+
const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
|
|
5449
|
+
const digest = await fileDigest(path);
|
|
5450
|
+
if (digest === null || digest === file.digest) continue;
|
|
5451
|
+
await rm2(path, { force: true });
|
|
5452
|
+
removed.push(file.path);
|
|
5453
|
+
}
|
|
5454
|
+
return removed;
|
|
5455
|
+
}
|
|
5440
5456
|
|
|
5441
5457
|
// src/embedding/transformers-import.ts
|
|
5442
5458
|
var TRANSFORMERS_MODULE = "@huggingface/transformers";
|
|
@@ -5566,6 +5582,24 @@ function encoderRows(outputs, attentionMask) {
|
|
|
5566
5582
|
return sum.map((v) => v / Math.max(1, count));
|
|
5567
5583
|
});
|
|
5568
5584
|
}
|
|
5585
|
+
function isCorruptCacheFailure(error) {
|
|
5586
|
+
const messages = [];
|
|
5587
|
+
let current = error;
|
|
5588
|
+
for (let depth = 0; depth < 6 && current; depth++) {
|
|
5589
|
+
messages.push(current instanceof Error ? current.message : String(current));
|
|
5590
|
+
current = current instanceof Error ? current.cause : void 0;
|
|
5591
|
+
}
|
|
5592
|
+
return /(?:protobuf parsing failed|failed to parse protobuf|invalid protobuf|unexpected end of (?:file|data|json input)|failed to load model|load model .* failed|invalid wire type)/i.test(
|
|
5593
|
+
messages.join(" ")
|
|
5594
|
+
);
|
|
5595
|
+
}
|
|
5596
|
+
async function settleBoth(runner, tokenizer) {
|
|
5597
|
+
const [r, t] = await Promise.allSettled([runner, tokenizer]);
|
|
5598
|
+
if (r.status === "fulfilled" && t.status === "fulfilled") return [r.value, t.value];
|
|
5599
|
+
if (r.status === "fulfilled") await r.value.dispose?.().catch(() => {
|
|
5600
|
+
});
|
|
5601
|
+
throw r.status === "rejected" ? r.reason : t.reason;
|
|
5602
|
+
}
|
|
5569
5603
|
async function loadTransformers() {
|
|
5570
5604
|
try {
|
|
5571
5605
|
const specifier = TRANSFORMERS_MODULE;
|
|
@@ -5633,7 +5667,7 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
5633
5667
|
);
|
|
5634
5668
|
}
|
|
5635
5669
|
const externalData = pinnedExternalDataChunks(profile);
|
|
5636
|
-
const
|
|
5670
|
+
const loadSessions = () => settleBoth(
|
|
5637
5671
|
multimodal ? transformers.AutoModel.from_pretrained(
|
|
5638
5672
|
profile.model.repo,
|
|
5639
5673
|
{
|
|
@@ -5656,7 +5690,24 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
5656
5690
|
...localOnly,
|
|
5657
5691
|
...progress
|
|
5658
5692
|
})
|
|
5659
|
-
|
|
5693
|
+
);
|
|
5694
|
+
let runner;
|
|
5695
|
+
let tokenizer;
|
|
5696
|
+
try {
|
|
5697
|
+
[runner, tokenizer] = await loadSessions();
|
|
5698
|
+
} catch (err) {
|
|
5699
|
+
const repairDir = cacheDir ?? transformers.env.cacheDir;
|
|
5700
|
+
if (!repairDir || !isCorruptCacheFailure(err)) throw err;
|
|
5701
|
+
const removed = await removeMismatchedPinnedFiles(profile, { cacheDir: repairDir });
|
|
5702
|
+
if (removed.length === 0) throw err;
|
|
5703
|
+
if (opts.localFilesOnly) {
|
|
5704
|
+
throw new EmbedderUnavailableError(
|
|
5705
|
+
`the cached model files for profile ${profile.id} were damaged (${removed.join(", ")}) and have been removed; they download again on the next load that may use the network`
|
|
5706
|
+
);
|
|
5707
|
+
}
|
|
5708
|
+
if (!opts.transformers) await preparePinnedLoad(profile, { cacheDir: repairDir });
|
|
5709
|
+
[runner, tokenizer] = await loadSessions();
|
|
5710
|
+
}
|
|
5660
5711
|
let verification;
|
|
5661
5712
|
try {
|
|
5662
5713
|
verification = await verifyProfileArtifacts(profile, {
|
|
@@ -7280,6 +7331,7 @@ export {
|
|
|
7280
7331
|
exportCatalogParquet,
|
|
7281
7332
|
extractGezkVerified,
|
|
7282
7333
|
fetchKnowledgeRegistry,
|
|
7334
|
+
fileDigest,
|
|
7283
7335
|
findDuckdbBinary,
|
|
7284
7336
|
findRegistryEntry,
|
|
7285
7337
|
formatAtLeast,
|
|
@@ -7317,6 +7369,7 @@ export {
|
|
|
7317
7369
|
quantizeInt8,
|
|
7318
7370
|
readGezkManifest,
|
|
7319
7371
|
readMkdocsDocsDir,
|
|
7372
|
+
removeMismatchedPinnedFiles,
|
|
7320
7373
|
rerankK,
|
|
7321
7374
|
rerankScore,
|
|
7322
7375
|
resolveOutlinePath,
|
package/package.json
CHANGED