@bendyline/gezel-knowledge 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +32 -2
- package/dist/index.js +216 -14
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -209,6 +209,26 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
209
209
|
* builds must not set it: the constant IS the format's scale design.
|
|
210
210
|
*/
|
|
211
211
|
shardTargetChunks?: number;
|
|
212
|
+
/**
|
|
213
|
+
* How a multi-shard catalog's documents are assigned to shards.
|
|
214
|
+
*
|
|
215
|
+
* - `'topic'` (default; the gezk spec §8.1): greedy fill in
|
|
216
|
+
* `(topicPathKey, documentId)` order, so shards are slices of the table
|
|
217
|
+
* of contents.
|
|
218
|
+
* - `'semantic'`: embed every chunk first, then fill shards by balanced
|
|
219
|
+
* k-means over each document's mean chunk vector, so documents that
|
|
220
|
+
* answer the same queries share a shard and routing (§8.4) reaches them
|
|
221
|
+
* within its budget. Topic slices put an artist, their albums and their
|
|
222
|
+
* songs in different shards. Within a shard documents keep
|
|
223
|
+
* `(topicPathKey, documentId)` order. Every chunk is still embedded
|
|
224
|
+
* exactly once: vectors are staged in `workDir` between the passes.
|
|
225
|
+
*
|
|
226
|
+
* Measured on a 1.85M-chunk Wikipedia music catalog (155 queries,
|
|
227
|
+
* routed recall@8 vs an exact scan): 10 shards, S=6 82.6% → 85.7% and
|
|
228
|
+
* S=3 52.1% → 63.9%; recut to 37 shards, S=6 36.0% → 55.5%. The gain
|
|
229
|
+
* grows with the shard count.
|
|
230
|
+
*/
|
|
231
|
+
shardFill?: 'topic' | 'semantic';
|
|
212
232
|
}
|
|
213
233
|
interface CompileReport {
|
|
214
234
|
manifest: KnowledgeCatalogManifest;
|
|
@@ -944,9 +964,19 @@ interface ProfileEmbedder {
|
|
|
944
964
|
countTokens(text: string): number;
|
|
945
965
|
dispose(): Promise<void>;
|
|
946
966
|
}
|
|
967
|
+
/**
|
|
968
|
+
* The `@huggingface/transformers` range this package supports, the same range
|
|
969
|
+
* as its optional peer dependency (a test holds the two together). Error
|
|
970
|
+
* messages quote it so the install command they print is exact.
|
|
971
|
+
*/
|
|
972
|
+
declare const TRANSFORMERS_PEER_RANGE = "^3.8.1";
|
|
947
973
|
declare class EmbedderUnavailableError extends Error {
|
|
948
974
|
readonly isActionable = true;
|
|
949
|
-
|
|
975
|
+
/** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
|
|
976
|
+
readonly runtimeMissing: boolean;
|
|
977
|
+
constructor(message: string, options?: {
|
|
978
|
+
runtimeMissing?: boolean;
|
|
979
|
+
});
|
|
950
980
|
}
|
|
951
981
|
/** The slice of `@huggingface/transformers` this module drives; injectable for tests. */
|
|
952
982
|
interface TransformersModule {
|
|
@@ -1262,4 +1292,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1262
1292
|
|
|
1263
1293
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1264
1294
|
|
|
1265
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
|
1295
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -181,7 +181,16 @@ function splitOversizeLine(line, opts) {
|
|
|
181
181
|
|
|
182
182
|
// src/compiler/compile.ts
|
|
183
183
|
import { createHash } from "crypto";
|
|
184
|
-
import {
|
|
184
|
+
import {
|
|
185
|
+
closeSync,
|
|
186
|
+
mkdirSync,
|
|
187
|
+
openSync,
|
|
188
|
+
readFileSync,
|
|
189
|
+
readSync,
|
|
190
|
+
rmSync,
|
|
191
|
+
statSync,
|
|
192
|
+
writeSync
|
|
193
|
+
} from "fs";
|
|
185
194
|
import { stat } from "fs/promises";
|
|
186
195
|
import { join } from "path";
|
|
187
196
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
@@ -310,10 +319,47 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
310
319
|
const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
|
|
311
320
|
const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
|
|
312
321
|
const embedded = totalChunks <= shardTarget;
|
|
322
|
+
const semanticFill = opts.shardFill === "semantic" && !embedded;
|
|
323
|
+
const embedInput = (p, chunk) => `${profile.passageInstruction}${buildContextHeader(
|
|
324
|
+
p.doc.title,
|
|
325
|
+
chunk.headingPath,
|
|
326
|
+
opts.chunkingProfile.contextHeader.max,
|
|
327
|
+
opts.countTokens
|
|
328
|
+
)}${chunk.text}`;
|
|
329
|
+
rmSync(opts.workDir, { recursive: true, force: true });
|
|
330
|
+
mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
|
|
331
|
+
const stagedPath = join(opts.workDir, "staged-vectors.f32");
|
|
332
|
+
let staged = null;
|
|
333
|
+
let docVectors = [];
|
|
334
|
+
if (semanticFill) {
|
|
335
|
+
docVectors = await stageVectors({
|
|
336
|
+
prepared,
|
|
337
|
+
stagedPath,
|
|
338
|
+
dimensions: profile.dimensions,
|
|
339
|
+
batchSize: opts.embedBatchSize ?? 32,
|
|
340
|
+
embed: opts.embed,
|
|
341
|
+
embedInput,
|
|
342
|
+
onProgress: (done) => opts.onProgress?.({ phase: "embed", done, total: totalChunks })
|
|
343
|
+
});
|
|
344
|
+
staged = new StagedVectorReader(stagedPath, profile.dimensions);
|
|
345
|
+
}
|
|
313
346
|
const shardOf = /* @__PURE__ */ new Map();
|
|
314
347
|
let shardCount = 1;
|
|
315
348
|
if (embedded) {
|
|
316
349
|
for (const p of prepared) shardOf.set(p.doc.id, 0);
|
|
350
|
+
} else if (semanticFill) {
|
|
351
|
+
shardCount = Math.ceil(totalChunks / shardTarget);
|
|
352
|
+
const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalChunks / shardCount * 1.02));
|
|
353
|
+
const assignment = semanticShards(
|
|
354
|
+
docVectors,
|
|
355
|
+
prepared.map((p) => p.chunks.length),
|
|
356
|
+
shardCount,
|
|
357
|
+
capacity,
|
|
358
|
+
seedFor(`${opts.catalog.id}#shard-fill`, 0)
|
|
359
|
+
);
|
|
360
|
+
prepared.forEach((p, i) => shardOf.set(p.doc.id, assignment[i]));
|
|
361
|
+
shardCount = new Set(assignment).size;
|
|
362
|
+
docVectors = [];
|
|
317
363
|
} else {
|
|
318
364
|
let current = 0;
|
|
319
365
|
let filled = 0;
|
|
@@ -327,8 +373,6 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
327
373
|
}
|
|
328
374
|
shardCount = current + 1;
|
|
329
375
|
}
|
|
330
|
-
rmSync(opts.workDir, { recursive: true, force: true });
|
|
331
|
-
mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
|
|
332
376
|
const routerStaged = join(opts.workDir, ROUTER_DB_PATH);
|
|
333
377
|
const shardPath = (id) => embedded ? ROUTER_DB_PATH : `index/shards/${String(id).padStart(3, "0")}.db`;
|
|
334
378
|
const shardDbs = /* @__PURE__ */ new Map();
|
|
@@ -467,7 +511,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
467
511
|
let processedChunks = 0;
|
|
468
512
|
const flushEmbeds = async () => {
|
|
469
513
|
if (pendingTexts.length === 0) return;
|
|
470
|
-
const vectors = await opts.embed(pendingTexts);
|
|
514
|
+
const vectors = staged ? staged.next(pendingTexts.length) : await opts.embed(pendingTexts);
|
|
471
515
|
if (vectors.length !== pendingTexts.length) {
|
|
472
516
|
throw new Error(
|
|
473
517
|
`embedder returned ${vectors.length} vectors for ${pendingTexts.length} texts`
|
|
@@ -492,7 +536,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
492
536
|
if ((row.chunkId - 1) % stride === 0) samples.push(unit);
|
|
493
537
|
}
|
|
494
538
|
processedChunks += pendingTexts.length;
|
|
495
|
-
opts.onProgress?.({
|
|
539
|
+
opts.onProgress?.({
|
|
540
|
+
phase: staged ? "write" : "embed",
|
|
541
|
+
done: processedChunks,
|
|
542
|
+
total: totalChunks
|
|
543
|
+
});
|
|
496
544
|
pendingTexts = [];
|
|
497
545
|
pendingRows = [];
|
|
498
546
|
};
|
|
@@ -530,18 +578,14 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
530
578
|
const chunkId = Number(row.id);
|
|
531
579
|
insertChunkFts.run(BigInt(chunkId), p.doc.title, headingText, chunk.text);
|
|
532
580
|
shardChunkCounts.set(shardId, (shardChunkCounts.get(shardId) ?? 0) + 1);
|
|
533
|
-
|
|
534
|
-
p.doc.title,
|
|
535
|
-
chunk.headingPath,
|
|
536
|
-
opts.chunkingProfile.contextHeader.max,
|
|
537
|
-
opts.countTokens
|
|
538
|
-
);
|
|
539
|
-
pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
|
|
581
|
+
pendingTexts.push(embedInput(p, chunk));
|
|
540
582
|
pendingRows.push({ shardId, chunkId });
|
|
541
583
|
if (pendingTexts.length >= embedBatchSize) await flushEmbeds();
|
|
542
584
|
}
|
|
543
585
|
}
|
|
544
586
|
await flushEmbeds();
|
|
587
|
+
staged?.close();
|
|
588
|
+
rmSync(stagedPath, { force: true });
|
|
545
589
|
for (let id = 0; id < shardCount; id++) {
|
|
546
590
|
if (!embedded)
|
|
547
591
|
putMeta(shardDb(id), {
|
|
@@ -913,6 +957,158 @@ ${path.join(" > ")}
|
|
|
913
957
|
}
|
|
914
958
|
return header;
|
|
915
959
|
}
|
|
960
|
+
async function stageVectors(args) {
|
|
961
|
+
const { prepared, dimensions } = args;
|
|
962
|
+
const docVectors = prepared.map(() => new Float32Array(dimensions));
|
|
963
|
+
const fd = openSync(args.stagedPath, "w");
|
|
964
|
+
let texts = [];
|
|
965
|
+
let owners = [];
|
|
966
|
+
let done = 0;
|
|
967
|
+
const flush = async () => {
|
|
968
|
+
if (texts.length === 0) return;
|
|
969
|
+
const vectors = await args.embed(texts);
|
|
970
|
+
if (vectors.length !== texts.length) {
|
|
971
|
+
throw new Error(`embedder returned ${vectors.length} vectors for ${texts.length} texts`);
|
|
972
|
+
}
|
|
973
|
+
const block = new Float32Array(texts.length * dimensions);
|
|
974
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
975
|
+
const unit = l2Normalize(vectors[i]);
|
|
976
|
+
if (unit.length !== dimensions) {
|
|
977
|
+
throw new Error(`embedder returned dim ${unit.length}, profile expects ${dimensions}`);
|
|
978
|
+
}
|
|
979
|
+
block.set(unit, i * dimensions);
|
|
980
|
+
const sum = docVectors[owners[i]];
|
|
981
|
+
for (let d = 0; d < dimensions; d++) sum[d] = sum[d] + unit[d];
|
|
982
|
+
}
|
|
983
|
+
writeSync(fd, new Uint8Array(block.buffer));
|
|
984
|
+
done += texts.length;
|
|
985
|
+
args.onProgress(done);
|
|
986
|
+
texts = [];
|
|
987
|
+
owners = [];
|
|
988
|
+
};
|
|
989
|
+
try {
|
|
990
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
991
|
+
const p = prepared[i];
|
|
992
|
+
for (const chunk of p.chunks) {
|
|
993
|
+
texts.push(args.embedInput(p, chunk));
|
|
994
|
+
owners.push(i);
|
|
995
|
+
if (texts.length >= args.batchSize) await flush();
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
await flush();
|
|
999
|
+
} finally {
|
|
1000
|
+
closeSync(fd);
|
|
1001
|
+
}
|
|
1002
|
+
return docVectors.map((v) => Float32Array.from(l2Normalize(Array.from(v))));
|
|
1003
|
+
}
|
|
1004
|
+
var StagedVectorReader = class {
|
|
1005
|
+
constructor(path, dimensions) {
|
|
1006
|
+
this.dimensions = dimensions;
|
|
1007
|
+
this.fd = openSync(path, "r");
|
|
1008
|
+
}
|
|
1009
|
+
dimensions;
|
|
1010
|
+
fd;
|
|
1011
|
+
position = 0;
|
|
1012
|
+
next(count) {
|
|
1013
|
+
const bytes = count * this.dimensions * 4;
|
|
1014
|
+
const buffer = Buffer.alloc(bytes);
|
|
1015
|
+
let read = 0;
|
|
1016
|
+
while (read < bytes) {
|
|
1017
|
+
const n = readSync(this.fd, buffer, read, bytes - read, this.position + read);
|
|
1018
|
+
if (n === 0)
|
|
1019
|
+
throw new Error("staged vectors ended early: the write pass and pass 1 disagree");
|
|
1020
|
+
read += n;
|
|
1021
|
+
}
|
|
1022
|
+
this.position += bytes;
|
|
1023
|
+
const floats = new Float32Array(buffer.buffer, buffer.byteOffset, count * this.dimensions);
|
|
1024
|
+
const out = [];
|
|
1025
|
+
for (let i = 0; i < count; i++) {
|
|
1026
|
+
out.push(Array.from(floats.subarray(i * this.dimensions, (i + 1) * this.dimensions)));
|
|
1027
|
+
}
|
|
1028
|
+
return out;
|
|
1029
|
+
}
|
|
1030
|
+
close() {
|
|
1031
|
+
closeSync(this.fd);
|
|
1032
|
+
}
|
|
1033
|
+
};
|
|
1034
|
+
var SEMANTIC_FILL_ROUNDS = 6;
|
|
1035
|
+
function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
|
|
1036
|
+
const n = docVectors.length;
|
|
1037
|
+
const dim = docVectors[0].length;
|
|
1038
|
+
const dot = (a, b) => {
|
|
1039
|
+
let sum = 0;
|
|
1040
|
+
for (let d = 0; d < dim; d++) sum += a[d] * b[d];
|
|
1041
|
+
return sum;
|
|
1042
|
+
};
|
|
1043
|
+
const stride = Math.max(1, Math.ceil(n / 2e4));
|
|
1044
|
+
const sample = docVectors.filter((_, i) => i % stride === 0);
|
|
1045
|
+
const centers = kMeans(sample, shardCount, seed).map((c) => c.centroid);
|
|
1046
|
+
for (let i = 0; centers.length < shardCount; i++) {
|
|
1047
|
+
centers.push(
|
|
1048
|
+
Float32Array.from(docVectors[Math.floor(i * n / shardCount) % n])
|
|
1049
|
+
);
|
|
1050
|
+
}
|
|
1051
|
+
let assignment = new Array(n).fill(-1);
|
|
1052
|
+
for (let round = 0; round < SEMANTIC_FILL_ROUNDS; round++) {
|
|
1053
|
+
const best = new Int32Array(n);
|
|
1054
|
+
const margin = new Float64Array(n);
|
|
1055
|
+
for (let i = 0; i < n; i++) {
|
|
1056
|
+
let first = Number.NEGATIVE_INFINITY;
|
|
1057
|
+
let second = Number.NEGATIVE_INFINITY;
|
|
1058
|
+
let arg = 0;
|
|
1059
|
+
for (let c = 0; c < shardCount; c++) {
|
|
1060
|
+
const s = dot(docVectors[i], centers[c]);
|
|
1061
|
+
if (s > first) {
|
|
1062
|
+
second = first;
|
|
1063
|
+
first = s;
|
|
1064
|
+
arg = c;
|
|
1065
|
+
} else if (s > second) {
|
|
1066
|
+
second = s;
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
best[i] = arg;
|
|
1070
|
+
margin[i] = first - (Number.isFinite(second) ? second : first);
|
|
1071
|
+
}
|
|
1072
|
+
const order = Array.from({ length: n }, (_, i) => i).sort(
|
|
1073
|
+
(a, b) => margin[b] - margin[a] || a - b
|
|
1074
|
+
);
|
|
1075
|
+
const load = new Array(shardCount).fill(0);
|
|
1076
|
+
const next = new Array(n).fill(-1);
|
|
1077
|
+
for (const i of order) {
|
|
1078
|
+
const size = docChunks[i];
|
|
1079
|
+
let shard = best[i];
|
|
1080
|
+
if (load[shard] + size > capacity) {
|
|
1081
|
+
const ranked = centers.map((c, id) => ({ id, s: dot(docVectors[i], c) })).sort((a, b) => b.s - a.s || a.id - b.id);
|
|
1082
|
+
const fit = ranked.find((r) => load[r.id] + size <= capacity);
|
|
1083
|
+
shard = fit ? fit.id : load.reduce((min, l, id) => l < load[min] ? id : min, 0);
|
|
1084
|
+
}
|
|
1085
|
+
next[i] = shard;
|
|
1086
|
+
load[shard] = load[shard] + size;
|
|
1087
|
+
}
|
|
1088
|
+
const moved = next.some((s, i) => s !== assignment[i]);
|
|
1089
|
+
assignment = next;
|
|
1090
|
+
if (!moved) break;
|
|
1091
|
+
const sums = centers.map(() => new Float64Array(dim));
|
|
1092
|
+
for (let i = 0; i < n; i++) {
|
|
1093
|
+
const sum = sums[assignment[i]];
|
|
1094
|
+
const v = docVectors[i];
|
|
1095
|
+
const w = docChunks[i];
|
|
1096
|
+
for (let d = 0; d < dim; d++) sum[d] = sum[d] + v[d] * w;
|
|
1097
|
+
}
|
|
1098
|
+
for (let c = 0; c < shardCount; c++) {
|
|
1099
|
+
const sum = sums[c];
|
|
1100
|
+
let norm = 0;
|
|
1101
|
+
for (let d = 0; d < dim; d++) norm += sum[d] ** 2;
|
|
1102
|
+
if (norm === 0) continue;
|
|
1103
|
+
norm = Math.sqrt(norm);
|
|
1104
|
+
const center = centers[c];
|
|
1105
|
+
for (let d = 0; d < dim; d++) center[d] = sum[d] / norm;
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
const renumber = /* @__PURE__ */ new Map();
|
|
1109
|
+
for (const s of assignment) if (!renumber.has(s)) renumber.set(s, renumber.size);
|
|
1110
|
+
return assignment.map((s) => renumber.get(s));
|
|
1111
|
+
}
|
|
916
1112
|
function seedFor(catalogId, shardId) {
|
|
917
1113
|
const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
|
|
918
1114
|
return digest.readBigUInt64LE(24);
|
|
@@ -3472,11 +3668,15 @@ async function verifyProfileArtifacts(profile, opts) {
|
|
|
3472
3668
|
// src/embedding/profile-embedder.ts
|
|
3473
3669
|
var MAX_BATCH = 8;
|
|
3474
3670
|
var MAX_CHARS = 8e3;
|
|
3671
|
+
var TRANSFORMERS_PEER_RANGE = "^3.8.1";
|
|
3475
3672
|
var EmbedderUnavailableError = class extends Error {
|
|
3476
3673
|
isActionable = true;
|
|
3477
|
-
|
|
3674
|
+
/** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
|
|
3675
|
+
runtimeMissing;
|
|
3676
|
+
constructor(message, options = {}) {
|
|
3478
3677
|
super(message);
|
|
3479
3678
|
this.name = "EmbedderUnavailableError";
|
|
3679
|
+
this.runtimeMissing = options.runtimeMissing ?? false;
|
|
3480
3680
|
}
|
|
3481
3681
|
};
|
|
3482
3682
|
function aggregateDownloadProgress(onProgress) {
|
|
@@ -3545,7 +3745,8 @@ async function loadTransformers() {
|
|
|
3545
3745
|
const message = err instanceof Error ? err.message : String(err);
|
|
3546
3746
|
if (/Cannot find (module|package)/i.test(message)) {
|
|
3547
3747
|
throw new EmbedderUnavailableError(
|
|
3548
|
-
|
|
3748
|
+
`the embedding runtime is not installed \u2014 knowledge builds and semantic search need the optional peer @huggingface/transformers@${TRANSFORMERS_PEER_RANGE}; install it alongside @bendyline/gezel-knowledge`,
|
|
3749
|
+
{ runtimeMissing: true }
|
|
3549
3750
|
);
|
|
3550
3751
|
}
|
|
3551
3752
|
throw err;
|
|
@@ -4691,6 +4892,7 @@ export {
|
|
|
4691
4892
|
SHARD_MAX_CHUNKS,
|
|
4692
4893
|
SHARD_TARGET_CHUNKS,
|
|
4693
4894
|
SOURCE_NOTICES_PATH,
|
|
4895
|
+
TRANSFORMERS_PEER_RANGE,
|
|
4694
4896
|
ZIP_FIXED_MTIME,
|
|
4695
4897
|
assertDuckdbCli,
|
|
4696
4898
|
assetReferences,
|
package/package.json
CHANGED