@bendyline/gezel-knowledge 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/index.d.ts +43 -3
- package/dist/index.js +57 -19
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -12,13 +12,15 @@ npm install @bendyline/gezel-knowledge
|
|
|
12
12
|
|
|
13
13
|
## What a `.gezk` is
|
|
14
14
|
|
|
15
|
-
An open format (
|
|
15
|
+
An open format (format version 0.6, preliminary until format 1.0; 0.5
|
|
16
|
+
archives still open): a ZIP whose first entry is
|
|
16
17
|
the stored `mimetype` magic (`application/vnd.gezk+zip`), then
|
|
17
18
|
`manifest.json`, `README.md`, `LICENSES/`, a `router.db` (topics, document
|
|
18
19
|
directory, brotli bodies, routing centroids) and shard SQLite databases with
|
|
19
20
|
FTS5 and quantized vectors in plain BLOB tables. Nothing beyond stock SQLite
|
|
20
21
|
is needed to read one. The format definitions are the separate
|
|
21
|
-
[`@bendyline/gezk`](https://www.npmjs.com/package/@bendyline/gezk) package
|
|
22
|
+
[`@bendyline/gezk`](https://www.npmjs.com/package/@bendyline/gezk) package,
|
|
23
|
+
whose npm version is independent of the format version it writes;
|
|
22
24
|
the specification, JSON Schemas, conformance fixtures and a Python reference
|
|
23
25
|
reader live in [bendyline/gezk](https://github.com/bendyline/gezk).
|
|
24
26
|
|
package/dist/index.d.ts
CHANGED
|
@@ -485,7 +485,10 @@ declare class CatalogHandle {
|
|
|
485
485
|
assets(): CatalogAssetInfo[];
|
|
486
486
|
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
487
487
|
readAsset(path: string): CatalogAssetRead | null;
|
|
488
|
-
/**
|
|
488
|
+
/**
|
|
489
|
+
* Catalog-wide title/summary/alias FTS — always runs regardless of routing.
|
|
490
|
+
* Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
|
|
491
|
+
*/
|
|
489
492
|
searchDocumentsFts(query: string, limit?: number): Array<{
|
|
490
493
|
documentId: string;
|
|
491
494
|
rank: number;
|
|
@@ -653,6 +656,13 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
|
653
656
|
* stores them (provably identical, one third of float32's size), with the
|
|
654
657
|
* dequantisation rule recorded in the file metadata (gezk spec §6).
|
|
655
658
|
*
|
|
659
|
+
* The bit rule in that metadata follows the catalog's profile: a `sign`
|
|
660
|
+
* profile stores sign(x), a `centered-sign` profile sign(x − c) with the
|
|
661
|
+
* profile's center c — which the metadata then carries too, so the Parquet
|
|
662
|
+
* files alone are enough to binarise a query the way the archive did. (Export
|
|
663
|
+
* version 2 wrote the plain-sign rule for every profile, which misdescribed
|
|
664
|
+
* centered-sign catalogs; version 3 fixed that.)
|
|
665
|
+
*
|
|
656
666
|
* Determinism: rows are read in primary-key order and written single-
|
|
657
667
|
* threaded with insertion order preserved, a fixed codec and level, a fixed
|
|
658
668
|
* row-group size, and no wall-clock metadata — so two exports of one archive
|
|
@@ -661,7 +671,8 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
|
661
671
|
* bytes.
|
|
662
672
|
*/
|
|
663
673
|
|
|
664
|
-
|
|
674
|
+
/** 3: the bit-rule metadata follows the profile's binary method (was always plain sign). */
|
|
675
|
+
declare const PARQUET_EXPORT_VERSION = 3;
|
|
665
676
|
type ParquetExportSource = {
|
|
666
677
|
archivePath: string;
|
|
667
678
|
} | {
|
|
@@ -712,6 +723,16 @@ interface ParquetExportReport {
|
|
|
712
723
|
}
|
|
713
724
|
declare const PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
714
725
|
declare function exportCatalogParquet(opts: ParquetExportOptions): Promise<ParquetExportReport>;
|
|
726
|
+
/**
|
|
727
|
+
* The key/value metadata every exported file carries. The bit rule is read
|
|
728
|
+
* from the profile — never assumed — because a `centered-sign` profile's bits
|
|
729
|
+
* are meaningless to a reader who binarises a query with plain sign(x). `x`
|
|
730
|
+
* is the float unit vector the bits were taken from, before int8 rounding, so
|
|
731
|
+
* recomputing them from `q / 127` can flip the few dimensions within rounding
|
|
732
|
+
* error of the threshold. Insertion order is the file's key order, so it is
|
|
733
|
+
* part of the deterministic bytes.
|
|
734
|
+
*/
|
|
735
|
+
declare function parquetKvMetadata(manifest: KnowledgeCatalogManifest): Record<string, string>;
|
|
715
736
|
/** Where a test or CLI may find a DuckDB CLI without configuration: env, then a gezel engine cache. */
|
|
716
737
|
declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | null>;
|
|
717
738
|
|
|
@@ -950,6 +971,11 @@ type PipelineFn = ((texts: string[], options: {
|
|
|
950
971
|
interface TokenizerFn {
|
|
951
972
|
encode(text: string): number[];
|
|
952
973
|
}
|
|
974
|
+
/** Bytes fetched so far across every file a profile load pulls, summed. */
|
|
975
|
+
interface ModelDownloadProgress {
|
|
976
|
+
bytesDone: number;
|
|
977
|
+
bytesTotal: number;
|
|
978
|
+
}
|
|
953
979
|
/** The load options that make transformers.js fetch exactly the profile's ONNX graph. */
|
|
954
980
|
interface TransformersModelOptions {
|
|
955
981
|
revision: string;
|
|
@@ -958,6 +984,18 @@ interface TransformersModelOptions {
|
|
|
958
984
|
model_file_name: string;
|
|
959
985
|
}
|
|
960
986
|
declare function resolveTransformersModelOptions(profile: KnowledgeEmbeddingProfile): TransformersModelOptions;
|
|
987
|
+
/**
|
|
988
|
+
* The transformers.js options that select exactly one pinned ONNX graph:
|
|
989
|
+
* its folder, its base name, and the dtype whose suffix transformers.js
|
|
990
|
+
* appends to that name. A file with no recognized suffix loads as fp32 under
|
|
991
|
+
* its full name, which selects the same file. Null for a path that is not a
|
|
992
|
+
* named `.onnx` graph.
|
|
993
|
+
*/
|
|
994
|
+
declare function transformersGraphOptions(onnxFile: string): {
|
|
995
|
+
dtype: string;
|
|
996
|
+
subfolder: string;
|
|
997
|
+
model_file_name: string;
|
|
998
|
+
} | null;
|
|
961
999
|
declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
|
|
962
1000
|
cacheDir?: string;
|
|
963
1001
|
/**
|
|
@@ -968,6 +1006,8 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
|
|
|
968
1006
|
* what affects values, and that stays with the caller.
|
|
969
1007
|
*/
|
|
970
1008
|
sessionOptions?: Record<string, unknown>;
|
|
1009
|
+
/** Called as the model and tokenizer files download. */
|
|
1010
|
+
onDownloadProgress?: (progress: ModelDownloadProgress) => void;
|
|
971
1011
|
/** Test seam: a stand-in for `@huggingface/transformers`. */
|
|
972
1012
|
transformers?: TransformersModule;
|
|
973
1013
|
}): Promise<ProfileEmbedder>;
|
|
@@ -1222,4 +1262,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1222
1262
|
|
|
1223
1263
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1224
1264
|
|
|
1225
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
|
1265
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -250,16 +250,20 @@ function sanitizeFtsQuery(query) {
|
|
|
250
250
|
if (tokens.length === 0) return null;
|
|
251
251
|
return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
|
|
252
252
|
}
|
|
253
|
-
|
|
253
|
+
var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
|
|
254
|
+
var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
|
|
255
|
+
function documentFtsTopIds(db, match, limit, exactTitle) {
|
|
256
|
+
const title = exactTitle?.normalize("NFKC").trim() ?? "";
|
|
254
257
|
return db.prepare(
|
|
255
|
-
|
|
256
|
-
|
|
258
|
+
`SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
|
|
259
|
+
ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
|
|
260
|
+
).all(match, title, limit).map((row) => row.document_id);
|
|
257
261
|
}
|
|
258
262
|
var SMOKE_QUERY_TOP_N = 10;
|
|
259
263
|
function documentSmokeQueryMisses(db, smoke) {
|
|
260
264
|
const match = sanitizeFtsQuery(smoke.query);
|
|
261
265
|
if (!match) return [...smoke.expectedDocumentIds];
|
|
262
|
-
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N);
|
|
266
|
+
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
|
|
263
267
|
return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
|
|
264
268
|
}
|
|
265
269
|
|
|
@@ -1924,12 +1928,15 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1924
1928
|
return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
|
|
1925
1929
|
}
|
|
1926
1930
|
// ── search ────────────────────────────────────────────────────────────────
|
|
1927
|
-
/**
|
|
1931
|
+
/**
|
|
1932
|
+
* Catalog-wide title/summary/alias FTS — always runs regardless of routing.
|
|
1933
|
+
* Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
|
|
1934
|
+
*/
|
|
1928
1935
|
searchDocumentsFts(query, limit = 10) {
|
|
1929
1936
|
const match = ftsQuery(query);
|
|
1930
1937
|
if (!match) return [];
|
|
1931
1938
|
try {
|
|
1932
|
-
return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
|
|
1939
|
+
return documentFtsTopIds(this.router.db, match, limit, query).map((documentId, i) => ({
|
|
1933
1940
|
documentId,
|
|
1934
1941
|
rank: i
|
|
1935
1942
|
}));
|
|
@@ -2488,8 +2495,8 @@ import { createWriteStream as createWriteStream3 } from "fs";
|
|
|
2488
2495
|
import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
|
|
2489
2496
|
import { join as join4, resolve as resolve3 } from "path";
|
|
2490
2497
|
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
2491
|
-
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
|
|
2492
|
-
var PARQUET_EXPORT_VERSION =
|
|
2498
|
+
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
|
|
2499
|
+
var PARQUET_EXPORT_VERSION = 3;
|
|
2493
2500
|
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
2494
2501
|
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
2495
2502
|
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
@@ -2514,7 +2521,7 @@ async function exportCatalogParquet(opts) {
|
|
|
2514
2521
|
const staging = join4(work, "ndjson");
|
|
2515
2522
|
await mkdir3(staging, { recursive: true });
|
|
2516
2523
|
const files = [];
|
|
2517
|
-
const metadata =
|
|
2524
|
+
const metadata = parquetKvMetadata(manifest);
|
|
2518
2525
|
const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
|
|
2519
2526
|
const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
|
|
2520
2527
|
try {
|
|
@@ -2782,7 +2789,9 @@ function parseJsonStrings(raw) {
|
|
|
2782
2789
|
function sqlString(value) {
|
|
2783
2790
|
return `'${value.replace(/'/g, "''")}'`;
|
|
2784
2791
|
}
|
|
2785
|
-
function
|
|
2792
|
+
function parquetKvMetadata(manifest) {
|
|
2793
|
+
const binary = manifest.embedding.quantization.binary;
|
|
2794
|
+
const centered = embeddingProfileCenter2(manifest.embedding) !== null;
|
|
2786
2795
|
return {
|
|
2787
2796
|
gezk_catalog_id: manifest.id,
|
|
2788
2797
|
gezk_catalog_version: manifest.version,
|
|
@@ -2792,7 +2801,9 @@ function kvMetadata(manifest) {
|
|
|
2792
2801
|
gezk_embedding_model: manifest.embedding.model.repo,
|
|
2793
2802
|
gezk_embedding_dimensions: String(manifest.embedding.dimensions),
|
|
2794
2803
|
gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
|
|
2795
|
-
gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
|
|
2804
|
+
gezk_embedding_bit: centered ? "centered-sign bits, LSB-first, ceil(dimensions/8) bytes: bit = (x - c) > 0, x the float unit vector, c = gezk_embedding_bit_center" : "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0, x the float unit vector",
|
|
2805
|
+
gezk_embedding_bit_method: binary.method,
|
|
2806
|
+
...centered ? { gezk_embedding_bit_center: JSON.stringify(binary.center) } : {},
|
|
2796
2807
|
gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
|
|
2797
2808
|
gezk_license: manifest.license.name
|
|
2798
2809
|
};
|
|
@@ -3468,6 +3479,25 @@ var EmbedderUnavailableError = class extends Error {
|
|
|
3468
3479
|
this.name = "EmbedderUnavailableError";
|
|
3469
3480
|
}
|
|
3470
3481
|
};
|
|
3482
|
+
function aggregateDownloadProgress(onProgress) {
|
|
3483
|
+
const files = /* @__PURE__ */ new Map();
|
|
3484
|
+
return (info) => {
|
|
3485
|
+
if (info.status !== "progress") return;
|
|
3486
|
+
const loaded = typeof info.loaded === "number" ? info.loaded : 0;
|
|
3487
|
+
const total = typeof info.total === "number" ? info.total : 0;
|
|
3488
|
+
files.set(`${String(info.name)}/${String(info.file)}`, {
|
|
3489
|
+
loaded,
|
|
3490
|
+
total: Math.max(total, loaded)
|
|
3491
|
+
});
|
|
3492
|
+
let bytesDone = 0;
|
|
3493
|
+
let bytesTotal = 0;
|
|
3494
|
+
for (const file of files.values()) {
|
|
3495
|
+
bytesDone += file.loaded;
|
|
3496
|
+
bytesTotal += file.total;
|
|
3497
|
+
}
|
|
3498
|
+
onProgress({ bytesDone, bytesTotal });
|
|
3499
|
+
};
|
|
3500
|
+
}
|
|
3471
3501
|
var ONNX_DTYPE_BY_SUFFIX = [
|
|
3472
3502
|
["_q4f16", "q4f16"],
|
|
3473
3503
|
["_quantized", "q8"],
|
|
@@ -3488,21 +3518,24 @@ function resolveTransformersModelOptions(profile) {
|
|
|
3488
3518
|
`profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
|
|
3489
3519
|
);
|
|
3490
3520
|
}
|
|
3491
|
-
|
|
3521
|
+
const graph = transformersGraphOptions(onnxFile);
|
|
3522
|
+
if (!graph) {
|
|
3492
3523
|
throw new EmbedderUnavailableError(
|
|
3493
|
-
`profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
|
|
3524
|
+
`profile ${profile.id} pins ${onnxFile}; this runtime loads named ONNX graphs only`
|
|
3494
3525
|
);
|
|
3495
3526
|
}
|
|
3527
|
+
return { revision: profile.model.revision, ...graph };
|
|
3528
|
+
}
|
|
3529
|
+
function transformersGraphOptions(onnxFile) {
|
|
3530
|
+
if (!onnxFile.endsWith(".onnx")) return null;
|
|
3496
3531
|
const slash = onnxFile.lastIndexOf("/");
|
|
3497
3532
|
const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
|
|
3498
3533
|
const base = onnxFile.slice(slash + 1, -".onnx".length);
|
|
3499
3534
|
const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
|
|
3500
3535
|
const [suffix, dtype] = variant ?? ["", "fp32"];
|
|
3501
3536
|
const modelFileName = base.slice(0, base.length - suffix.length);
|
|
3502
|
-
if (modelFileName === "")
|
|
3503
|
-
|
|
3504
|
-
}
|
|
3505
|
-
return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
|
|
3537
|
+
if (modelFileName === "") return null;
|
|
3538
|
+
return { dtype, subfolder, model_file_name: modelFileName };
|
|
3506
3539
|
}
|
|
3507
3540
|
async function loadTransformers() {
|
|
3508
3541
|
try {
|
|
@@ -3533,13 +3566,16 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3533
3566
|
transformers.env.useFSCache = true;
|
|
3534
3567
|
transformers.env.allowRemoteModels = true;
|
|
3535
3568
|
}
|
|
3569
|
+
const progress = opts.onDownloadProgress ? { progress_callback: aggregateDownloadProgress(opts.onDownloadProgress) } : {};
|
|
3536
3570
|
const [pipe, tokenizer] = await Promise.all([
|
|
3537
3571
|
transformers.pipeline("feature-extraction", profile.model.repo, {
|
|
3538
3572
|
...modelOptions,
|
|
3539
|
-
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
|
|
3573
|
+
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
|
|
3574
|
+
...progress
|
|
3540
3575
|
}),
|
|
3541
3576
|
transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
|
|
3542
|
-
revision: modelOptions.revision
|
|
3577
|
+
revision: modelOptions.revision,
|
|
3578
|
+
...progress
|
|
3543
3579
|
})
|
|
3544
3580
|
]);
|
|
3545
3581
|
let verification;
|
|
@@ -4689,6 +4725,7 @@ export {
|
|
|
4689
4725
|
newOutlineTopic,
|
|
4690
4726
|
newerRegistryEntries,
|
|
4691
4727
|
openCatalogDatabase,
|
|
4728
|
+
parquetKvMetadata,
|
|
4692
4729
|
parseGitbookSummary,
|
|
4693
4730
|
parseJupyterBookToc,
|
|
4694
4731
|
parseMkdocsNav,
|
|
@@ -4706,6 +4743,7 @@ export {
|
|
|
4706
4743
|
signRegistryIndex,
|
|
4707
4744
|
titleFromFolderName,
|
|
4708
4745
|
transformersCachePath,
|
|
4746
|
+
transformersGraphOptions,
|
|
4709
4747
|
validateExtractedCatalog,
|
|
4710
4748
|
verifyManifestSignature,
|
|
4711
4749
|
verifyProfileArtifacts,
|
package/package.json
CHANGED