@bendyline/gezel-knowledge 1.1.2 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/index.d.ts +74 -4
- package/dist/index.js +273 -33
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -12,13 +12,15 @@ npm install @bendyline/gezel-knowledge
|
|
|
12
12
|
|
|
13
13
|
## What a `.gezk` is
|
|
14
14
|
|
|
15
|
-
An open format (
|
|
15
|
+
An open format (format version 0.6, preliminary until format 1.0; 0.5
|
|
16
|
+
archives still open): a ZIP whose first entry is
|
|
16
17
|
the stored `mimetype` magic (`application/vnd.gezk+zip`), then
|
|
17
18
|
`manifest.json`, `README.md`, `LICENSES/`, a `router.db` (topics, document
|
|
18
19
|
directory, brotli bodies, routing centroids) and shard SQLite databases with
|
|
19
20
|
FTS5 and quantized vectors in plain BLOB tables. Nothing beyond stock SQLite
|
|
20
21
|
is needed to read one. The format definitions are the separate
|
|
21
|
-
[`@bendyline/gezk`](https://www.npmjs.com/package/@bendyline/gezk) package
|
|
22
|
+
[`@bendyline/gezk`](https://www.npmjs.com/package/@bendyline/gezk) package,
|
|
23
|
+
whose npm version is independent of the format version it writes;
|
|
22
24
|
the specification, JSON Schemas, conformance fixtures and a Python reference
|
|
23
25
|
reader live in [bendyline/gezk](https://github.com/bendyline/gezk).
|
|
24
26
|
|
package/dist/index.d.ts
CHANGED
|
@@ -209,6 +209,26 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
209
209
|
* builds must not set it: the constant IS the format's scale design.
|
|
210
210
|
*/
|
|
211
211
|
shardTargetChunks?: number;
|
|
212
|
+
/**
|
|
213
|
+
* How a multi-shard catalog's documents are assigned to shards.
|
|
214
|
+
*
|
|
215
|
+
* - `'topic'` (default; the gezk spec §8.1): greedy fill in
|
|
216
|
+
* `(topicPathKey, documentId)` order, so shards are slices of the table
|
|
217
|
+
* of contents.
|
|
218
|
+
* - `'semantic'`: embed every chunk first, then fill shards by balanced
|
|
219
|
+
* k-means over each document's mean chunk vector, so documents that
|
|
220
|
+
* answer the same queries share a shard and routing (§8.4) reaches them
|
|
221
|
+
* within its budget. Topic slices put an artist, their albums and their
|
|
222
|
+
* songs in different shards. Within a shard documents keep
|
|
223
|
+
* `(topicPathKey, documentId)` order. Every chunk is still embedded
|
|
224
|
+
* exactly once: vectors are staged in `workDir` between the passes.
|
|
225
|
+
*
|
|
226
|
+
* Measured on a 1.85M-chunk Wikipedia music catalog (155 queries,
|
|
227
|
+
* routed recall@8 vs an exact scan): 10 shards, S=6 82.6% → 85.7% and
|
|
228
|
+
* S=3 52.1% → 63.9%; recut to 37 shards, S=6 36.0% → 55.5%. The gain
|
|
229
|
+
* grows with the shard count.
|
|
230
|
+
*/
|
|
231
|
+
shardFill?: 'topic' | 'semantic';
|
|
212
232
|
}
|
|
213
233
|
interface CompileReport {
|
|
214
234
|
manifest: KnowledgeCatalogManifest;
|
|
@@ -485,7 +505,10 @@ declare class CatalogHandle {
|
|
|
485
505
|
assets(): CatalogAssetInfo[];
|
|
486
506
|
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
487
507
|
readAsset(path: string): CatalogAssetRead | null;
|
|
488
|
-
/**
|
|
508
|
+
/**
|
|
509
|
+
* Catalog-wide title/summary/alias FTS — always runs regardless of routing.
|
|
510
|
+
* Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
|
|
511
|
+
*/
|
|
489
512
|
searchDocumentsFts(query: string, limit?: number): Array<{
|
|
490
513
|
documentId: string;
|
|
491
514
|
rank: number;
|
|
@@ -653,6 +676,13 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
|
653
676
|
* stores them (provably identical, one third of float32's size), with the
|
|
654
677
|
* dequantisation rule recorded in the file metadata (gezk spec §6).
|
|
655
678
|
*
|
|
679
|
+
* The bit rule in that metadata follows the catalog's profile: a `sign`
|
|
680
|
+
* profile stores sign(x), a `centered-sign` profile sign(x − c) with the
|
|
681
|
+
* profile's center c — which the metadata then carries too, so the Parquet
|
|
682
|
+
* files alone are enough to binarise a query the way the archive did. (Export
|
|
683
|
+
* version 2 wrote the plain-sign rule for every profile, which misdescribed
|
|
684
|
+
* centered-sign catalogs; version 3 fixed that.)
|
|
685
|
+
*
|
|
656
686
|
* Determinism: rows are read in primary-key order and written single-
|
|
657
687
|
* threaded with insertion order preserved, a fixed codec and level, a fixed
|
|
658
688
|
* row-group size, and no wall-clock metadata — so two exports of one archive
|
|
@@ -661,7 +691,8 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
|
661
691
|
* bytes.
|
|
662
692
|
*/
|
|
663
693
|
|
|
664
|
-
|
|
694
|
+
/** 3: the bit-rule metadata follows the profile's binary method (was always plain sign). */
|
|
695
|
+
declare const PARQUET_EXPORT_VERSION = 3;
|
|
665
696
|
type ParquetExportSource = {
|
|
666
697
|
archivePath: string;
|
|
667
698
|
} | {
|
|
@@ -712,6 +743,16 @@ interface ParquetExportReport {
|
|
|
712
743
|
}
|
|
713
744
|
declare const PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
714
745
|
declare function exportCatalogParquet(opts: ParquetExportOptions): Promise<ParquetExportReport>;
|
|
746
|
+
/**
|
|
747
|
+
* The key/value metadata every exported file carries. The bit rule is read
|
|
748
|
+
* from the profile — never assumed — because a `centered-sign` profile's bits
|
|
749
|
+
* are meaningless to a reader who binarises a query with plain sign(x). `x`
|
|
750
|
+
* is the float unit vector the bits were taken from, before int8 rounding, so
|
|
751
|
+
* recomputing them from `q / 127` can flip the few dimensions within rounding
|
|
752
|
+
* error of the threshold. Insertion order is the file's key order, so it is
|
|
753
|
+
* part of the deterministic bytes.
|
|
754
|
+
*/
|
|
755
|
+
declare function parquetKvMetadata(manifest: KnowledgeCatalogManifest): Record<string, string>;
|
|
715
756
|
/** Where a test or CLI may find a DuckDB CLI without configuration: env, then a gezel engine cache. */
|
|
716
757
|
declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | null>;
|
|
717
758
|
|
|
@@ -923,9 +964,19 @@ interface ProfileEmbedder {
|
|
|
923
964
|
countTokens(text: string): number;
|
|
924
965
|
dispose(): Promise<void>;
|
|
925
966
|
}
|
|
967
|
+
/**
|
|
968
|
+
* The `@huggingface/transformers` range this package supports, the same range
|
|
969
|
+
* as its optional peer dependency (a test holds the two together). Error
|
|
970
|
+
* messages quote it so the install command they print is exact.
|
|
971
|
+
*/
|
|
972
|
+
declare const TRANSFORMERS_PEER_RANGE = "^3.8.1";
|
|
926
973
|
declare class EmbedderUnavailableError extends Error {
|
|
927
974
|
readonly isActionable = true;
|
|
928
|
-
|
|
975
|
+
/** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
|
|
976
|
+
readonly runtimeMissing: boolean;
|
|
977
|
+
constructor(message: string, options?: {
|
|
978
|
+
runtimeMissing?: boolean;
|
|
979
|
+
});
|
|
929
980
|
}
|
|
930
981
|
/** The slice of `@huggingface/transformers` this module drives; injectable for tests. */
|
|
931
982
|
interface TransformersModule {
|
|
@@ -950,6 +1001,11 @@ type PipelineFn = ((texts: string[], options: {
|
|
|
950
1001
|
interface TokenizerFn {
|
|
951
1002
|
encode(text: string): number[];
|
|
952
1003
|
}
|
|
1004
|
+
/** Bytes fetched so far across every file a profile load pulls, summed. */
|
|
1005
|
+
interface ModelDownloadProgress {
|
|
1006
|
+
bytesDone: number;
|
|
1007
|
+
bytesTotal: number;
|
|
1008
|
+
}
|
|
953
1009
|
/** The load options that make transformers.js fetch exactly the profile's ONNX graph. */
|
|
954
1010
|
interface TransformersModelOptions {
|
|
955
1011
|
revision: string;
|
|
@@ -958,6 +1014,18 @@ interface TransformersModelOptions {
|
|
|
958
1014
|
model_file_name: string;
|
|
959
1015
|
}
|
|
960
1016
|
declare function resolveTransformersModelOptions(profile: KnowledgeEmbeddingProfile): TransformersModelOptions;
|
|
1017
|
+
/**
|
|
1018
|
+
* The transformers.js options that select exactly one pinned ONNX graph:
|
|
1019
|
+
* its folder, its base name, and the dtype whose suffix transformers.js
|
|
1020
|
+
* appends to that name. A file with no recognized suffix loads as fp32 under
|
|
1021
|
+
* its full name, which selects the same file. Null for a path that is not a
|
|
1022
|
+
* named `.onnx` graph.
|
|
1023
|
+
*/
|
|
1024
|
+
declare function transformersGraphOptions(onnxFile: string): {
|
|
1025
|
+
dtype: string;
|
|
1026
|
+
subfolder: string;
|
|
1027
|
+
model_file_name: string;
|
|
1028
|
+
} | null;
|
|
961
1029
|
declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
|
|
962
1030
|
cacheDir?: string;
|
|
963
1031
|
/**
|
|
@@ -968,6 +1036,8 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
|
|
|
968
1036
|
* what affects values, and that stays with the caller.
|
|
969
1037
|
*/
|
|
970
1038
|
sessionOptions?: Record<string, unknown>;
|
|
1039
|
+
/** Called as the model and tokenizer files download. */
|
|
1040
|
+
onDownloadProgress?: (progress: ModelDownloadProgress) => void;
|
|
971
1041
|
/** Test seam: a stand-in for `@huggingface/transformers`. */
|
|
972
1042
|
transformers?: TransformersModule;
|
|
973
1043
|
}): Promise<ProfileEmbedder>;
|
|
@@ -1222,4 +1292,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1222
1292
|
|
|
1223
1293
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1224
1294
|
|
|
1225
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
|
1295
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -181,7 +181,16 @@ function splitOversizeLine(line, opts) {
|
|
|
181
181
|
|
|
182
182
|
// src/compiler/compile.ts
|
|
183
183
|
import { createHash } from "crypto";
|
|
184
|
-
import {
|
|
184
|
+
import {
|
|
185
|
+
closeSync,
|
|
186
|
+
mkdirSync,
|
|
187
|
+
openSync,
|
|
188
|
+
readFileSync,
|
|
189
|
+
readSync,
|
|
190
|
+
rmSync,
|
|
191
|
+
statSync,
|
|
192
|
+
writeSync
|
|
193
|
+
} from "fs";
|
|
185
194
|
import { stat } from "fs/promises";
|
|
186
195
|
import { join } from "path";
|
|
187
196
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
@@ -250,16 +259,20 @@ function sanitizeFtsQuery(query) {
|
|
|
250
259
|
if (tokens.length === 0) return null;
|
|
251
260
|
return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
|
|
252
261
|
}
|
|
253
|
-
|
|
262
|
+
var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
|
|
263
|
+
var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
|
|
264
|
+
function documentFtsTopIds(db, match, limit, exactTitle) {
|
|
265
|
+
const title = exactTitle?.normalize("NFKC").trim() ?? "";
|
|
254
266
|
return db.prepare(
|
|
255
|
-
|
|
256
|
-
|
|
267
|
+
`SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
|
|
268
|
+
ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
|
|
269
|
+
).all(match, title, limit).map((row) => row.document_id);
|
|
257
270
|
}
|
|
258
271
|
var SMOKE_QUERY_TOP_N = 10;
|
|
259
272
|
function documentSmokeQueryMisses(db, smoke) {
|
|
260
273
|
const match = sanitizeFtsQuery(smoke.query);
|
|
261
274
|
if (!match) return [...smoke.expectedDocumentIds];
|
|
262
|
-
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N);
|
|
275
|
+
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
|
|
263
276
|
return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
|
|
264
277
|
}
|
|
265
278
|
|
|
@@ -306,10 +319,47 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
306
319
|
const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
|
|
307
320
|
const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
|
|
308
321
|
const embedded = totalChunks <= shardTarget;
|
|
322
|
+
const semanticFill = opts.shardFill === "semantic" && !embedded;
|
|
323
|
+
const embedInput = (p, chunk) => `${profile.passageInstruction}${buildContextHeader(
|
|
324
|
+
p.doc.title,
|
|
325
|
+
chunk.headingPath,
|
|
326
|
+
opts.chunkingProfile.contextHeader.max,
|
|
327
|
+
opts.countTokens
|
|
328
|
+
)}${chunk.text}`;
|
|
329
|
+
rmSync(opts.workDir, { recursive: true, force: true });
|
|
330
|
+
mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
|
|
331
|
+
const stagedPath = join(opts.workDir, "staged-vectors.f32");
|
|
332
|
+
let staged = null;
|
|
333
|
+
let docVectors = [];
|
|
334
|
+
if (semanticFill) {
|
|
335
|
+
docVectors = await stageVectors({
|
|
336
|
+
prepared,
|
|
337
|
+
stagedPath,
|
|
338
|
+
dimensions: profile.dimensions,
|
|
339
|
+
batchSize: opts.embedBatchSize ?? 32,
|
|
340
|
+
embed: opts.embed,
|
|
341
|
+
embedInput,
|
|
342
|
+
onProgress: (done) => opts.onProgress?.({ phase: "embed", done, total: totalChunks })
|
|
343
|
+
});
|
|
344
|
+
staged = new StagedVectorReader(stagedPath, profile.dimensions);
|
|
345
|
+
}
|
|
309
346
|
const shardOf = /* @__PURE__ */ new Map();
|
|
310
347
|
let shardCount = 1;
|
|
311
348
|
if (embedded) {
|
|
312
349
|
for (const p of prepared) shardOf.set(p.doc.id, 0);
|
|
350
|
+
} else if (semanticFill) {
|
|
351
|
+
shardCount = Math.ceil(totalChunks / shardTarget);
|
|
352
|
+
const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalChunks / shardCount * 1.02));
|
|
353
|
+
const assignment = semanticShards(
|
|
354
|
+
docVectors,
|
|
355
|
+
prepared.map((p) => p.chunks.length),
|
|
356
|
+
shardCount,
|
|
357
|
+
capacity,
|
|
358
|
+
seedFor(`${opts.catalog.id}#shard-fill`, 0)
|
|
359
|
+
);
|
|
360
|
+
prepared.forEach((p, i) => shardOf.set(p.doc.id, assignment[i]));
|
|
361
|
+
shardCount = new Set(assignment).size;
|
|
362
|
+
docVectors = [];
|
|
313
363
|
} else {
|
|
314
364
|
let current = 0;
|
|
315
365
|
let filled = 0;
|
|
@@ -323,8 +373,6 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
323
373
|
}
|
|
324
374
|
shardCount = current + 1;
|
|
325
375
|
}
|
|
326
|
-
rmSync(opts.workDir, { recursive: true, force: true });
|
|
327
|
-
mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
|
|
328
376
|
const routerStaged = join(opts.workDir, ROUTER_DB_PATH);
|
|
329
377
|
const shardPath = (id) => embedded ? ROUTER_DB_PATH : `index/shards/${String(id).padStart(3, "0")}.db`;
|
|
330
378
|
const shardDbs = /* @__PURE__ */ new Map();
|
|
@@ -463,7 +511,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
463
511
|
let processedChunks = 0;
|
|
464
512
|
const flushEmbeds = async () => {
|
|
465
513
|
if (pendingTexts.length === 0) return;
|
|
466
|
-
const vectors = await opts.embed(pendingTexts);
|
|
514
|
+
const vectors = staged ? staged.next(pendingTexts.length) : await opts.embed(pendingTexts);
|
|
467
515
|
if (vectors.length !== pendingTexts.length) {
|
|
468
516
|
throw new Error(
|
|
469
517
|
`embedder returned ${vectors.length} vectors for ${pendingTexts.length} texts`
|
|
@@ -488,7 +536,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
488
536
|
if ((row.chunkId - 1) % stride === 0) samples.push(unit);
|
|
489
537
|
}
|
|
490
538
|
processedChunks += pendingTexts.length;
|
|
491
|
-
opts.onProgress?.({
|
|
539
|
+
opts.onProgress?.({
|
|
540
|
+
phase: staged ? "write" : "embed",
|
|
541
|
+
done: processedChunks,
|
|
542
|
+
total: totalChunks
|
|
543
|
+
});
|
|
492
544
|
pendingTexts = [];
|
|
493
545
|
pendingRows = [];
|
|
494
546
|
};
|
|
@@ -526,18 +578,14 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
526
578
|
const chunkId = Number(row.id);
|
|
527
579
|
insertChunkFts.run(BigInt(chunkId), p.doc.title, headingText, chunk.text);
|
|
528
580
|
shardChunkCounts.set(shardId, (shardChunkCounts.get(shardId) ?? 0) + 1);
|
|
529
|
-
|
|
530
|
-
p.doc.title,
|
|
531
|
-
chunk.headingPath,
|
|
532
|
-
opts.chunkingProfile.contextHeader.max,
|
|
533
|
-
opts.countTokens
|
|
534
|
-
);
|
|
535
|
-
pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
|
|
581
|
+
pendingTexts.push(embedInput(p, chunk));
|
|
536
582
|
pendingRows.push({ shardId, chunkId });
|
|
537
583
|
if (pendingTexts.length >= embedBatchSize) await flushEmbeds();
|
|
538
584
|
}
|
|
539
585
|
}
|
|
540
586
|
await flushEmbeds();
|
|
587
|
+
staged?.close();
|
|
588
|
+
rmSync(stagedPath, { force: true });
|
|
541
589
|
for (let id = 0; id < shardCount; id++) {
|
|
542
590
|
if (!embedded)
|
|
543
591
|
putMeta(shardDb(id), {
|
|
@@ -909,6 +957,158 @@ ${path.join(" > ")}
|
|
|
909
957
|
}
|
|
910
958
|
return header;
|
|
911
959
|
}
|
|
960
|
+
async function stageVectors(args) {
|
|
961
|
+
const { prepared, dimensions } = args;
|
|
962
|
+
const docVectors = prepared.map(() => new Float32Array(dimensions));
|
|
963
|
+
const fd = openSync(args.stagedPath, "w");
|
|
964
|
+
let texts = [];
|
|
965
|
+
let owners = [];
|
|
966
|
+
let done = 0;
|
|
967
|
+
const flush = async () => {
|
|
968
|
+
if (texts.length === 0) return;
|
|
969
|
+
const vectors = await args.embed(texts);
|
|
970
|
+
if (vectors.length !== texts.length) {
|
|
971
|
+
throw new Error(`embedder returned ${vectors.length} vectors for ${texts.length} texts`);
|
|
972
|
+
}
|
|
973
|
+
const block = new Float32Array(texts.length * dimensions);
|
|
974
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
975
|
+
const unit = l2Normalize(vectors[i]);
|
|
976
|
+
if (unit.length !== dimensions) {
|
|
977
|
+
throw new Error(`embedder returned dim ${unit.length}, profile expects ${dimensions}`);
|
|
978
|
+
}
|
|
979
|
+
block.set(unit, i * dimensions);
|
|
980
|
+
const sum = docVectors[owners[i]];
|
|
981
|
+
for (let d = 0; d < dimensions; d++) sum[d] = sum[d] + unit[d];
|
|
982
|
+
}
|
|
983
|
+
writeSync(fd, new Uint8Array(block.buffer));
|
|
984
|
+
done += texts.length;
|
|
985
|
+
args.onProgress(done);
|
|
986
|
+
texts = [];
|
|
987
|
+
owners = [];
|
|
988
|
+
};
|
|
989
|
+
try {
|
|
990
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
991
|
+
const p = prepared[i];
|
|
992
|
+
for (const chunk of p.chunks) {
|
|
993
|
+
texts.push(args.embedInput(p, chunk));
|
|
994
|
+
owners.push(i);
|
|
995
|
+
if (texts.length >= args.batchSize) await flush();
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
await flush();
|
|
999
|
+
} finally {
|
|
1000
|
+
closeSync(fd);
|
|
1001
|
+
}
|
|
1002
|
+
return docVectors.map((v) => Float32Array.from(l2Normalize(Array.from(v))));
|
|
1003
|
+
}
|
|
1004
|
+
var StagedVectorReader = class {
|
|
1005
|
+
constructor(path, dimensions) {
|
|
1006
|
+
this.dimensions = dimensions;
|
|
1007
|
+
this.fd = openSync(path, "r");
|
|
1008
|
+
}
|
|
1009
|
+
dimensions;
|
|
1010
|
+
fd;
|
|
1011
|
+
position = 0;
|
|
1012
|
+
next(count) {
|
|
1013
|
+
const bytes = count * this.dimensions * 4;
|
|
1014
|
+
const buffer = Buffer.alloc(bytes);
|
|
1015
|
+
let read = 0;
|
|
1016
|
+
while (read < bytes) {
|
|
1017
|
+
const n = readSync(this.fd, buffer, read, bytes - read, this.position + read);
|
|
1018
|
+
if (n === 0)
|
|
1019
|
+
throw new Error("staged vectors ended early: the write pass and pass 1 disagree");
|
|
1020
|
+
read += n;
|
|
1021
|
+
}
|
|
1022
|
+
this.position += bytes;
|
|
1023
|
+
const floats = new Float32Array(buffer.buffer, buffer.byteOffset, count * this.dimensions);
|
|
1024
|
+
const out = [];
|
|
1025
|
+
for (let i = 0; i < count; i++) {
|
|
1026
|
+
out.push(Array.from(floats.subarray(i * this.dimensions, (i + 1) * this.dimensions)));
|
|
1027
|
+
}
|
|
1028
|
+
return out;
|
|
1029
|
+
}
|
|
1030
|
+
close() {
|
|
1031
|
+
closeSync(this.fd);
|
|
1032
|
+
}
|
|
1033
|
+
};
|
|
1034
|
+
var SEMANTIC_FILL_ROUNDS = 6;
|
|
1035
|
+
function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
|
|
1036
|
+
const n = docVectors.length;
|
|
1037
|
+
const dim = docVectors[0].length;
|
|
1038
|
+
const dot = (a, b) => {
|
|
1039
|
+
let sum = 0;
|
|
1040
|
+
for (let d = 0; d < dim; d++) sum += a[d] * b[d];
|
|
1041
|
+
return sum;
|
|
1042
|
+
};
|
|
1043
|
+
const stride = Math.max(1, Math.ceil(n / 2e4));
|
|
1044
|
+
const sample = docVectors.filter((_, i) => i % stride === 0);
|
|
1045
|
+
const centers = kMeans(sample, shardCount, seed).map((c) => c.centroid);
|
|
1046
|
+
for (let i = 0; centers.length < shardCount; i++) {
|
|
1047
|
+
centers.push(
|
|
1048
|
+
Float32Array.from(docVectors[Math.floor(i * n / shardCount) % n])
|
|
1049
|
+
);
|
|
1050
|
+
}
|
|
1051
|
+
let assignment = new Array(n).fill(-1);
|
|
1052
|
+
for (let round = 0; round < SEMANTIC_FILL_ROUNDS; round++) {
|
|
1053
|
+
const best = new Int32Array(n);
|
|
1054
|
+
const margin = new Float64Array(n);
|
|
1055
|
+
for (let i = 0; i < n; i++) {
|
|
1056
|
+
let first = Number.NEGATIVE_INFINITY;
|
|
1057
|
+
let second = Number.NEGATIVE_INFINITY;
|
|
1058
|
+
let arg = 0;
|
|
1059
|
+
for (let c = 0; c < shardCount; c++) {
|
|
1060
|
+
const s = dot(docVectors[i], centers[c]);
|
|
1061
|
+
if (s > first) {
|
|
1062
|
+
second = first;
|
|
1063
|
+
first = s;
|
|
1064
|
+
arg = c;
|
|
1065
|
+
} else if (s > second) {
|
|
1066
|
+
second = s;
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
best[i] = arg;
|
|
1070
|
+
margin[i] = first - (Number.isFinite(second) ? second : first);
|
|
1071
|
+
}
|
|
1072
|
+
const order = Array.from({ length: n }, (_, i) => i).sort(
|
|
1073
|
+
(a, b) => margin[b] - margin[a] || a - b
|
|
1074
|
+
);
|
|
1075
|
+
const load = new Array(shardCount).fill(0);
|
|
1076
|
+
const next = new Array(n).fill(-1);
|
|
1077
|
+
for (const i of order) {
|
|
1078
|
+
const size = docChunks[i];
|
|
1079
|
+
let shard = best[i];
|
|
1080
|
+
if (load[shard] + size > capacity) {
|
|
1081
|
+
const ranked = centers.map((c, id) => ({ id, s: dot(docVectors[i], c) })).sort((a, b) => b.s - a.s || a.id - b.id);
|
|
1082
|
+
const fit = ranked.find((r) => load[r.id] + size <= capacity);
|
|
1083
|
+
shard = fit ? fit.id : load.reduce((min, l, id) => l < load[min] ? id : min, 0);
|
|
1084
|
+
}
|
|
1085
|
+
next[i] = shard;
|
|
1086
|
+
load[shard] = load[shard] + size;
|
|
1087
|
+
}
|
|
1088
|
+
const moved = next.some((s, i) => s !== assignment[i]);
|
|
1089
|
+
assignment = next;
|
|
1090
|
+
if (!moved) break;
|
|
1091
|
+
const sums = centers.map(() => new Float64Array(dim));
|
|
1092
|
+
for (let i = 0; i < n; i++) {
|
|
1093
|
+
const sum = sums[assignment[i]];
|
|
1094
|
+
const v = docVectors[i];
|
|
1095
|
+
const w = docChunks[i];
|
|
1096
|
+
for (let d = 0; d < dim; d++) sum[d] = sum[d] + v[d] * w;
|
|
1097
|
+
}
|
|
1098
|
+
for (let c = 0; c < shardCount; c++) {
|
|
1099
|
+
const sum = sums[c];
|
|
1100
|
+
let norm = 0;
|
|
1101
|
+
for (let d = 0; d < dim; d++) norm += sum[d] ** 2;
|
|
1102
|
+
if (norm === 0) continue;
|
|
1103
|
+
norm = Math.sqrt(norm);
|
|
1104
|
+
const center = centers[c];
|
|
1105
|
+
for (let d = 0; d < dim; d++) center[d] = sum[d] / norm;
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
const renumber = /* @__PURE__ */ new Map();
|
|
1109
|
+
for (const s of assignment) if (!renumber.has(s)) renumber.set(s, renumber.size);
|
|
1110
|
+
return assignment.map((s) => renumber.get(s));
|
|
1111
|
+
}
|
|
912
1112
|
function seedFor(catalogId, shardId) {
|
|
913
1113
|
const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
|
|
914
1114
|
return digest.readBigUInt64LE(24);
|
|
@@ -1924,12 +2124,15 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1924
2124
|
return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
|
|
1925
2125
|
}
|
|
1926
2126
|
// ── search ────────────────────────────────────────────────────────────────
|
|
1927
|
-
/**
|
|
2127
|
+
/**
|
|
2128
|
+
* Catalog-wide title/summary/alias FTS — always runs regardless of routing.
|
|
2129
|
+
* Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
|
|
2130
|
+
*/
|
|
1928
2131
|
searchDocumentsFts(query, limit = 10) {
|
|
1929
2132
|
const match = ftsQuery(query);
|
|
1930
2133
|
if (!match) return [];
|
|
1931
2134
|
try {
|
|
1932
|
-
return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
|
|
2135
|
+
return documentFtsTopIds(this.router.db, match, limit, query).map((documentId, i) => ({
|
|
1933
2136
|
documentId,
|
|
1934
2137
|
rank: i
|
|
1935
2138
|
}));
|
|
@@ -2488,8 +2691,8 @@ import { createWriteStream as createWriteStream3 } from "fs";
|
|
|
2488
2691
|
import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
|
|
2489
2692
|
import { join as join4, resolve as resolve3 } from "path";
|
|
2490
2693
|
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
2491
|
-
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
|
|
2492
|
-
var PARQUET_EXPORT_VERSION =
|
|
2694
|
+
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
|
|
2695
|
+
var PARQUET_EXPORT_VERSION = 3;
|
|
2493
2696
|
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
2494
2697
|
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
2495
2698
|
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
@@ -2514,7 +2717,7 @@ async function exportCatalogParquet(opts) {
|
|
|
2514
2717
|
const staging = join4(work, "ndjson");
|
|
2515
2718
|
await mkdir3(staging, { recursive: true });
|
|
2516
2719
|
const files = [];
|
|
2517
|
-
const metadata =
|
|
2720
|
+
const metadata = parquetKvMetadata(manifest);
|
|
2518
2721
|
const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
|
|
2519
2722
|
const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
|
|
2520
2723
|
try {
|
|
@@ -2782,7 +2985,9 @@ function parseJsonStrings(raw) {
|
|
|
2782
2985
|
function sqlString(value) {
|
|
2783
2986
|
return `'${value.replace(/'/g, "''")}'`;
|
|
2784
2987
|
}
|
|
2785
|
-
function
|
|
2988
|
+
function parquetKvMetadata(manifest) {
|
|
2989
|
+
const binary = manifest.embedding.quantization.binary;
|
|
2990
|
+
const centered = embeddingProfileCenter2(manifest.embedding) !== null;
|
|
2786
2991
|
return {
|
|
2787
2992
|
gezk_catalog_id: manifest.id,
|
|
2788
2993
|
gezk_catalog_version: manifest.version,
|
|
@@ -2792,7 +2997,9 @@ function kvMetadata(manifest) {
|
|
|
2792
2997
|
gezk_embedding_model: manifest.embedding.model.repo,
|
|
2793
2998
|
gezk_embedding_dimensions: String(manifest.embedding.dimensions),
|
|
2794
2999
|
gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
|
|
2795
|
-
gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
|
|
3000
|
+
gezk_embedding_bit: centered ? "centered-sign bits, LSB-first, ceil(dimensions/8) bytes: bit = (x - c) > 0, x the float unit vector, c = gezk_embedding_bit_center" : "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0, x the float unit vector",
|
|
3001
|
+
gezk_embedding_bit_method: binary.method,
|
|
3002
|
+
...centered ? { gezk_embedding_bit_center: JSON.stringify(binary.center) } : {},
|
|
2796
3003
|
gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
|
|
2797
3004
|
gezk_license: manifest.license.name
|
|
2798
3005
|
};
|
|
@@ -3461,13 +3668,36 @@ async function verifyProfileArtifacts(profile, opts) {
|
|
|
3461
3668
|
// src/embedding/profile-embedder.ts
|
|
3462
3669
|
var MAX_BATCH = 8;
|
|
3463
3670
|
var MAX_CHARS = 8e3;
|
|
3671
|
+
var TRANSFORMERS_PEER_RANGE = "^3.8.1";
|
|
3464
3672
|
var EmbedderUnavailableError = class extends Error {
|
|
3465
3673
|
isActionable = true;
|
|
3466
|
-
|
|
3674
|
+
/** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
|
|
3675
|
+
runtimeMissing;
|
|
3676
|
+
constructor(message, options = {}) {
|
|
3467
3677
|
super(message);
|
|
3468
3678
|
this.name = "EmbedderUnavailableError";
|
|
3679
|
+
this.runtimeMissing = options.runtimeMissing ?? false;
|
|
3469
3680
|
}
|
|
3470
3681
|
};
|
|
3682
|
+
function aggregateDownloadProgress(onProgress) {
|
|
3683
|
+
const files = /* @__PURE__ */ new Map();
|
|
3684
|
+
return (info) => {
|
|
3685
|
+
if (info.status !== "progress") return;
|
|
3686
|
+
const loaded = typeof info.loaded === "number" ? info.loaded : 0;
|
|
3687
|
+
const total = typeof info.total === "number" ? info.total : 0;
|
|
3688
|
+
files.set(`${String(info.name)}/${String(info.file)}`, {
|
|
3689
|
+
loaded,
|
|
3690
|
+
total: Math.max(total, loaded)
|
|
3691
|
+
});
|
|
3692
|
+
let bytesDone = 0;
|
|
3693
|
+
let bytesTotal = 0;
|
|
3694
|
+
for (const file of files.values()) {
|
|
3695
|
+
bytesDone += file.loaded;
|
|
3696
|
+
bytesTotal += file.total;
|
|
3697
|
+
}
|
|
3698
|
+
onProgress({ bytesDone, bytesTotal });
|
|
3699
|
+
};
|
|
3700
|
+
}
|
|
3471
3701
|
var ONNX_DTYPE_BY_SUFFIX = [
|
|
3472
3702
|
["_q4f16", "q4f16"],
|
|
3473
3703
|
["_quantized", "q8"],
|
|
@@ -3488,21 +3718,24 @@ function resolveTransformersModelOptions(profile) {
|
|
|
3488
3718
|
`profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
|
|
3489
3719
|
);
|
|
3490
3720
|
}
|
|
3491
|
-
|
|
3721
|
+
const graph = transformersGraphOptions(onnxFile);
|
|
3722
|
+
if (!graph) {
|
|
3492
3723
|
throw new EmbedderUnavailableError(
|
|
3493
|
-
`profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
|
|
3724
|
+
`profile ${profile.id} pins ${onnxFile}; this runtime loads named ONNX graphs only`
|
|
3494
3725
|
);
|
|
3495
3726
|
}
|
|
3727
|
+
return { revision: profile.model.revision, ...graph };
|
|
3728
|
+
}
|
|
3729
|
+
function transformersGraphOptions(onnxFile) {
|
|
3730
|
+
if (!onnxFile.endsWith(".onnx")) return null;
|
|
3496
3731
|
const slash = onnxFile.lastIndexOf("/");
|
|
3497
3732
|
const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
|
|
3498
3733
|
const base = onnxFile.slice(slash + 1, -".onnx".length);
|
|
3499
3734
|
const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
|
|
3500
3735
|
const [suffix, dtype] = variant ?? ["", "fp32"];
|
|
3501
3736
|
const modelFileName = base.slice(0, base.length - suffix.length);
|
|
3502
|
-
if (modelFileName === "")
|
|
3503
|
-
|
|
3504
|
-
}
|
|
3505
|
-
return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
|
|
3737
|
+
if (modelFileName === "") return null;
|
|
3738
|
+
return { dtype, subfolder, model_file_name: modelFileName };
|
|
3506
3739
|
}
|
|
3507
3740
|
async function loadTransformers() {
|
|
3508
3741
|
try {
|
|
@@ -3512,7 +3745,8 @@ async function loadTransformers() {
|
|
|
3512
3745
|
const message = err instanceof Error ? err.message : String(err);
|
|
3513
3746
|
if (/Cannot find (module|package)/i.test(message)) {
|
|
3514
3747
|
throw new EmbedderUnavailableError(
|
|
3515
|
-
|
|
3748
|
+
`the embedding runtime is not installed \u2014 knowledge builds and semantic search need the optional peer @huggingface/transformers@${TRANSFORMERS_PEER_RANGE}; install it alongside @bendyline/gezel-knowledge`,
|
|
3749
|
+
{ runtimeMissing: true }
|
|
3516
3750
|
);
|
|
3517
3751
|
}
|
|
3518
3752
|
throw err;
|
|
@@ -3533,13 +3767,16 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3533
3767
|
transformers.env.useFSCache = true;
|
|
3534
3768
|
transformers.env.allowRemoteModels = true;
|
|
3535
3769
|
}
|
|
3770
|
+
const progress = opts.onDownloadProgress ? { progress_callback: aggregateDownloadProgress(opts.onDownloadProgress) } : {};
|
|
3536
3771
|
const [pipe, tokenizer] = await Promise.all([
|
|
3537
3772
|
transformers.pipeline("feature-extraction", profile.model.repo, {
|
|
3538
3773
|
...modelOptions,
|
|
3539
|
-
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
|
|
3774
|
+
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
|
|
3775
|
+
...progress
|
|
3540
3776
|
}),
|
|
3541
3777
|
transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
|
|
3542
|
-
revision: modelOptions.revision
|
|
3778
|
+
revision: modelOptions.revision,
|
|
3779
|
+
...progress
|
|
3543
3780
|
})
|
|
3544
3781
|
]);
|
|
3545
3782
|
let verification;
|
|
@@ -4655,6 +4892,7 @@ export {
|
|
|
4655
4892
|
SHARD_MAX_CHUNKS,
|
|
4656
4893
|
SHARD_TARGET_CHUNKS,
|
|
4657
4894
|
SOURCE_NOTICES_PATH,
|
|
4895
|
+
TRANSFORMERS_PEER_RANGE,
|
|
4658
4896
|
ZIP_FIXED_MTIME,
|
|
4659
4897
|
assertDuckdbCli,
|
|
4660
4898
|
assetReferences,
|
|
@@ -4689,6 +4927,7 @@ export {
|
|
|
4689
4927
|
newOutlineTopic,
|
|
4690
4928
|
newerRegistryEntries,
|
|
4691
4929
|
openCatalogDatabase,
|
|
4930
|
+
parquetKvMetadata,
|
|
4692
4931
|
parseGitbookSummary,
|
|
4693
4932
|
parseJupyterBookToc,
|
|
4694
4933
|
parseMkdocsNav,
|
|
@@ -4706,6 +4945,7 @@ export {
|
|
|
4706
4945
|
signRegistryIndex,
|
|
4707
4946
|
titleFromFolderName,
|
|
4708
4947
|
transformersCachePath,
|
|
4948
|
+
transformersGraphOptions,
|
|
4709
4949
|
validateExtractedCatalog,
|
|
4710
4950
|
verifyManifestSignature,
|
|
4711
4951
|
verifyProfileArtifacts,
|
package/package.json
CHANGED