@delali/narsil 0.2.4-next.45 → 0.2.4-next.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-OBXS7OZM.mjs → chunk-4COAUBLJ.mjs} +1 -1
- package/dist/{chunk-WDL632CN.mjs → chunk-DFZDV5FV.mjs} +244 -239
- package/dist/{chunk-XNCJFU2E.mjs → chunk-EV45L35L.mjs} +225 -81
- package/dist/{chunk-WASNWIEK.mjs → chunk-UZTT2BKQ.mjs} +1 -1
- package/dist/{direct-executor-IMXNRS7O.mjs → direct-executor-EOTFGXTO.mjs} +3 -3
- package/dist/distribution.mjs +4 -4
- package/dist/index.browser.mjs +467 -318
- package/dist/index.mjs +4 -4
- package/dist/persistence/durability/checkpoint-worker.mjs +4 -4
- package/dist/vector/hnsw-build-worker.mjs +1 -1
- package/dist/vector/search-worker.mjs +4 -3
- package/dist/workers/entry.mjs +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { english } from './chunk-TGP5M5R7.mjs';
|
|
2
|
-
import { createScalarQuantizer } from './chunk-
|
|
3
|
-
import { MAX_M, createVectorStore, createHNSWIndex, createOrdinalFilter, addToOrdinalFilter, removeFromOrdinalFilter, ordinalFilterValues, magnitude,
|
|
2
|
+
import { createScalarQuantizer } from './chunk-4COAUBLJ.mjs';
|
|
3
|
+
import { MAX_M, createVectorStore, createHNSWIndex, createOrdinalFilter, addToOrdinalFilter, removeFromOrdinalFilter, ordinalFilterValues, magnitude, toScore, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, ABSENT_DOCUMENT_RANK } from './chunk-DFZDV5FV.mjs';
|
|
4
4
|
import { compareCodePoints, compareComparableKeys, toComparableSortValue, readSortField, compareSortValues, compareComparableValues, SORT_VALUE_MAX_CODE_POINTS } from './chunk-BFIH7OV5.mjs';
|
|
5
5
|
import { generateId, createRequestId } from './chunk-WTONLDU3.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes, createNarsilError } from './chunk-P4ELXVJQ.mjs';
|
|
@@ -9475,9 +9475,54 @@ var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
|
|
|
9475
9475
|
var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
|
|
9476
9476
|
var WORKER_BUILD_SIZE_THRESHOLD = 5e3;
|
|
9477
9477
|
var WORKER_COPY_MIN_VECTORS = 1024;
|
|
9478
|
+
var BUILD_CHUNK_SIZE = 100;
|
|
9479
|
+
var REBUILD_REMOVED_RATIO = 0.2;
|
|
9478
9480
|
function liveSize(state) {
|
|
9479
9481
|
return state.store.size - state.tombstones.size;
|
|
9480
9482
|
}
|
|
9483
|
+
function adoptGraph(state, graph) {
|
|
9484
|
+
if (graph === null && state.hnsw !== null) {
|
|
9485
|
+
state.hnsw.clear();
|
|
9486
|
+
}
|
|
9487
|
+
state.hnsw = graph;
|
|
9488
|
+
state.compactedNodeCount = 0;
|
|
9489
|
+
if (graph === null) return;
|
|
9490
|
+
for (const docId of state.tombstones) {
|
|
9491
|
+
if (graph.has(docId)) {
|
|
9492
|
+
graph.markTombstone(docId);
|
|
9493
|
+
}
|
|
9494
|
+
}
|
|
9495
|
+
}
|
|
9496
|
+
async function insertIntoGraph(state, graph, docIds, admit, inserted) {
|
|
9497
|
+
let count = 0;
|
|
9498
|
+
for (const docId of docIds) {
|
|
9499
|
+
if (state.disposed) return false;
|
|
9500
|
+
if (!admit(docId)) continue;
|
|
9501
|
+
graph.insertNode(docId);
|
|
9502
|
+
state.buffer.delete(docId);
|
|
9503
|
+
inserted?.(docId);
|
|
9504
|
+
count += 1;
|
|
9505
|
+
if (count % BUILD_CHUNK_SIZE === 0) {
|
|
9506
|
+
await yieldToEventLoop();
|
|
9507
|
+
}
|
|
9508
|
+
}
|
|
9509
|
+
return !state.disposed;
|
|
9510
|
+
}
|
|
9511
|
+
async function buildGraphFromStore(state) {
|
|
9512
|
+
const graph = createHNSWIndex(state.dimension, state.store, state.hnswConfig, state.sq8 ?? void 0);
|
|
9513
|
+
const completed = await insertIntoGraph(state, graph, allLiveDocIds(state), () => true);
|
|
9514
|
+
if (completed) {
|
|
9515
|
+
adoptGraph(state, graph);
|
|
9516
|
+
}
|
|
9517
|
+
}
|
|
9518
|
+
function graphNeedsRebuild(state) {
|
|
9519
|
+
const graph = state.hnsw;
|
|
9520
|
+
if (graph === null) return false;
|
|
9521
|
+
const removed = graph.tombstoneCount + state.compactedNodeCount;
|
|
9522
|
+
const held = graph.size + removed;
|
|
9523
|
+
if (held === 0) return false;
|
|
9524
|
+
return removed / held > REBUILD_REMOVED_RATIO;
|
|
9525
|
+
}
|
|
9481
9526
|
function yieldToEventLoop() {
|
|
9482
9527
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
9483
9528
|
}
|
|
@@ -10103,7 +10148,7 @@ function captureCloneSnapshot(state) {
|
|
|
10103
10148
|
};
|
|
10104
10149
|
}
|
|
10105
10150
|
function scheduleWorkerCopyLoad(state) {
|
|
10106
|
-
if (state.disposed || state.workerCopyLoading) return;
|
|
10151
|
+
if (state.disposed || state.workerCopyLoading || state.building) return;
|
|
10107
10152
|
if (state.workerCopyHandle !== null) return;
|
|
10108
10153
|
if (!state.hnsw || state.buffer.size > 0) return;
|
|
10109
10154
|
if (liveSize(state) < WORKER_COPY_MIN_VECTORS) return;
|
|
@@ -10184,7 +10229,7 @@ async function searchViaWorkerCopies(state, query, k, metric, minSimilarity, efS
|
|
|
10184
10229
|
}
|
|
10185
10230
|
|
|
10186
10231
|
// src/vector/vector-index/build.ts
|
|
10187
|
-
async function tryWorkerBuild(state, liveDocIds
|
|
10232
|
+
async function tryWorkerBuild(state, liveDocIds) {
|
|
10188
10233
|
const vectorData = new Float32Array(liveDocIds.length * state.dimension);
|
|
10189
10234
|
const validDocIds = [];
|
|
10190
10235
|
let offset = 0;
|
|
@@ -10193,6 +10238,7 @@ async function tryWorkerBuild(state, liveDocIds, bufferSnapshot) {
|
|
|
10193
10238
|
if (!entry || state.tombstones.has(docId)) continue;
|
|
10194
10239
|
vectorData.set(entry.vector, offset);
|
|
10195
10240
|
validDocIds.push(docId);
|
|
10241
|
+
state.buffer.delete(docId);
|
|
10196
10242
|
offset += state.dimension;
|
|
10197
10243
|
}
|
|
10198
10244
|
if (validDocIds.length === 0) return false;
|
|
@@ -10204,64 +10250,77 @@ async function tryWorkerBuild(state, liveDocIds, bufferSnapshot) {
|
|
|
10204
10250
|
};
|
|
10205
10251
|
const timeoutMs = Math.max(1e4, liveDocIds.length * 2);
|
|
10206
10252
|
const outcome = await dispatchWorkerBuild(validDocIds, packedData, state.dimension, resolvedConfig, timeoutMs, true);
|
|
10207
|
-
if (!outcome.ok) return false;
|
|
10253
|
+
if (!outcome.ok || state.disposed) return false;
|
|
10208
10254
|
const newHnsw = createHNSWIndex(state.dimension, state.store, state.hnswConfig, state.sq8 ?? void 0);
|
|
10209
10255
|
newHnsw.deserialize(outcome.graph);
|
|
10210
|
-
state
|
|
10211
|
-
|
|
10212
|
-
|
|
10213
|
-
|
|
10214
|
-
|
|
10256
|
+
adoptGraph(state, newHnsw);
|
|
10257
|
+
return true;
|
|
10258
|
+
}
|
|
10259
|
+
async function promoteToGraph(state) {
|
|
10260
|
+
const liveDocIds = Array.from(allLiveDocIds(state));
|
|
10261
|
+
if (liveDocIds.length === 0) return;
|
|
10262
|
+
if (state.sq8) {
|
|
10263
|
+
calibrateAndQuantizeAll(state);
|
|
10215
10264
|
}
|
|
10216
|
-
|
|
10217
|
-
state
|
|
10265
|
+
if (liveDocIds.length > WORKER_BUILD_SIZE_THRESHOLD) {
|
|
10266
|
+
const built = await tryWorkerBuild(state, liveDocIds);
|
|
10267
|
+
if (built) return;
|
|
10268
|
+
}
|
|
10269
|
+
await buildGraphFromStore(state);
|
|
10270
|
+
}
|
|
10271
|
+
function quantizeIncoming(state, docId) {
|
|
10272
|
+
const sq8 = state.sq8;
|
|
10273
|
+
if (sq8 === null) return false;
|
|
10274
|
+
if (!sq8.isCalibrated()) {
|
|
10275
|
+
calibrateAndQuantizeAll(state);
|
|
10276
|
+
return false;
|
|
10277
|
+
}
|
|
10278
|
+
const entry = state.store.get(docId);
|
|
10279
|
+
if (entry === void 0) return false;
|
|
10280
|
+
const outsideBounds = sq8.needsRecalibration(entry.vector);
|
|
10281
|
+
sq8.quantize(docId, entry.vector);
|
|
10282
|
+
return outsideBounds;
|
|
10283
|
+
}
|
|
10284
|
+
async function growGraph(state, graph, bufferSnapshot) {
|
|
10285
|
+
let outsideCalibration = false;
|
|
10286
|
+
const admit = (docId) => {
|
|
10287
|
+
if (state.hnsw !== graph) return false;
|
|
10288
|
+
if (state.tombstones.has(docId) || !state.store.has(docId)) {
|
|
10289
|
+
state.buffer.delete(docId);
|
|
10290
|
+
return false;
|
|
10291
|
+
}
|
|
10292
|
+
return true;
|
|
10293
|
+
};
|
|
10294
|
+
const inserted = (docId) => {
|
|
10295
|
+
if (quantizeIncoming(state, docId)) {
|
|
10296
|
+
outsideCalibration = true;
|
|
10297
|
+
}
|
|
10298
|
+
};
|
|
10299
|
+
await insertIntoGraph(state, graph, bufferSnapshot, admit, inserted);
|
|
10300
|
+
if (outsideCalibration) {
|
|
10301
|
+
recalibrateFromStore(state);
|
|
10218
10302
|
}
|
|
10219
|
-
return true;
|
|
10220
10303
|
}
|
|
10221
10304
|
function triggerBuild(state) {
|
|
10222
10305
|
if (state.building) return;
|
|
10223
10306
|
invalidateWorkerCopies(state);
|
|
10224
10307
|
state.building = true;
|
|
10225
|
-
const liveDocIds = [];
|
|
10226
|
-
for (const [docId] of state.store.entries()) {
|
|
10227
|
-
if (state.tombstones.has(docId)) continue;
|
|
10228
|
-
liveDocIds.push(docId);
|
|
10229
|
-
}
|
|
10230
10308
|
const bufferSnapshot = new Set(state.buffer);
|
|
10309
|
+
const existingGraph = state.hnsw;
|
|
10231
10310
|
const buildPromise = (async () => {
|
|
10232
10311
|
try {
|
|
10233
|
-
if (liveDocIds.length === 0 || state.disposed) return;
|
|
10234
|
-
if (state.sq8) {
|
|
10235
|
-
calibrateAndQuantizeAll(state);
|
|
10236
|
-
}
|
|
10237
|
-
if (liveDocIds.length > WORKER_BUILD_SIZE_THRESHOLD) {
|
|
10238
|
-
const workerResult = await tryWorkerBuild(state, liveDocIds, bufferSnapshot);
|
|
10239
|
-
if (workerResult) return;
|
|
10240
|
-
}
|
|
10241
|
-
const newHnsw = createHNSWIndex(state.dimension, state.store, state.hnswConfig, state.sq8 ?? void 0);
|
|
10242
|
-
const CHUNK_SIZE = 100;
|
|
10243
|
-
for (let i = 0; i < liveDocIds.length; i++) {
|
|
10244
|
-
const docId = liveDocIds[i];
|
|
10245
|
-
if (!state.store.has(docId) || state.tombstones.has(docId)) continue;
|
|
10246
|
-
newHnsw.insertNode(docId);
|
|
10247
|
-
if ((i + 1) % CHUNK_SIZE === 0) {
|
|
10248
|
-
if (state.disposed) return;
|
|
10249
|
-
await yieldToEventLoop();
|
|
10250
|
-
}
|
|
10251
|
-
}
|
|
10252
10312
|
if (state.disposed) return;
|
|
10253
|
-
|
|
10254
|
-
|
|
10255
|
-
|
|
10256
|
-
newHnsw.markTombstone(docId);
|
|
10257
|
-
}
|
|
10258
|
-
}
|
|
10259
|
-
for (const docId of bufferSnapshot) {
|
|
10260
|
-
state.buffer.delete(docId);
|
|
10313
|
+
if (existingGraph !== null) {
|
|
10314
|
+
await growGraph(state, existingGraph, bufferSnapshot);
|
|
10315
|
+
return;
|
|
10261
10316
|
}
|
|
10317
|
+
await promoteToGraph(state);
|
|
10262
10318
|
} finally {
|
|
10263
10319
|
state.building = false;
|
|
10264
10320
|
state.pendingBuild = null;
|
|
10321
|
+
if (state.buffer.size > 0) {
|
|
10322
|
+
scheduleBuild(state);
|
|
10323
|
+
}
|
|
10265
10324
|
}
|
|
10266
10325
|
})();
|
|
10267
10326
|
state.pendingBuild = buildPromise;
|
|
@@ -10281,6 +10340,7 @@ function scheduleBuild(state) {
|
|
|
10281
10340
|
function compact(state) {
|
|
10282
10341
|
if (state.tombstones.size === 0) return;
|
|
10283
10342
|
if (state.hnsw) {
|
|
10343
|
+
state.compactedNodeCount += state.hnsw.tombstoneCount;
|
|
10284
10344
|
state.hnsw.compactTombstones();
|
|
10285
10345
|
}
|
|
10286
10346
|
for (const docId of state.tombstones) {
|
|
@@ -10295,43 +10355,48 @@ function compact(state) {
|
|
|
10295
10355
|
recalibrateFromStore(state);
|
|
10296
10356
|
}
|
|
10297
10357
|
}
|
|
10298
|
-
async function
|
|
10299
|
-
|
|
10300
|
-
|
|
10301
|
-
|
|
10358
|
+
async function insertMissing(state, graph) {
|
|
10359
|
+
const missingOrReplaced = (docId) => !graph.has(docId) || state.buffer.has(docId);
|
|
10360
|
+
await insertIntoGraph(state, graph, allLiveDocIds(state), missingOrReplaced);
|
|
10361
|
+
}
|
|
10362
|
+
async function foldIntoGraph(state) {
|
|
10363
|
+
const rebuildNeeded = graphNeedsRebuild(state);
|
|
10364
|
+
const compactRecalibrates = state.tombstones.size > 0 && state.sq8?.isCalibrated() === true;
|
|
10302
10365
|
compact(state);
|
|
10303
|
-
|
|
10304
|
-
|
|
10305
|
-
if (state.hnsw) {
|
|
10306
|
-
state.hnsw.clear();
|
|
10307
|
-
state.hnsw = null;
|
|
10308
|
-
}
|
|
10366
|
+
if (liveSize(state) === 0) {
|
|
10367
|
+
adoptGraph(state, null);
|
|
10309
10368
|
state.buffer.clear();
|
|
10310
10369
|
if (state.sq8) {
|
|
10311
10370
|
state.sq8.clear();
|
|
10312
10371
|
}
|
|
10313
10372
|
return;
|
|
10314
10373
|
}
|
|
10315
|
-
|
|
10316
|
-
|
|
10374
|
+
const graph = state.hnsw;
|
|
10375
|
+
if (graph === null || rebuildNeeded) {
|
|
10376
|
+
await buildGraphFromStore(state);
|
|
10377
|
+
} else {
|
|
10378
|
+
await insertMissing(state, graph);
|
|
10317
10379
|
}
|
|
10318
|
-
|
|
10319
|
-
|
|
10320
|
-
for (const [docId] of state.store.entries()) {
|
|
10321
|
-
if (state.tombstones.has(docId)) continue;
|
|
10322
|
-
liveDocIds.push(docId);
|
|
10380
|
+
if (state.sq8 && state.store.size > 0 && !compactRecalibrates) {
|
|
10381
|
+
recalibrateFromStore(state);
|
|
10323
10382
|
}
|
|
10324
|
-
|
|
10325
|
-
|
|
10326
|
-
|
|
10327
|
-
|
|
10328
|
-
await yieldToEventLoop();
|
|
10329
|
-
}
|
|
10383
|
+
}
|
|
10384
|
+
async function optimize(state) {
|
|
10385
|
+
while (state.pendingBuild) {
|
|
10386
|
+
await state.pendingBuild;
|
|
10330
10387
|
}
|
|
10331
|
-
state.
|
|
10332
|
-
state.
|
|
10333
|
-
|
|
10334
|
-
|
|
10388
|
+
if (state.disposed) return;
|
|
10389
|
+
state.building = true;
|
|
10390
|
+
const work = foldIntoGraph(state);
|
|
10391
|
+
state.pendingBuild = work;
|
|
10392
|
+
try {
|
|
10393
|
+
await work;
|
|
10394
|
+
} finally {
|
|
10395
|
+
state.building = false;
|
|
10396
|
+
state.pendingBuild = null;
|
|
10397
|
+
if (state.buffer.size > 0) {
|
|
10398
|
+
scheduleBuild(state);
|
|
10399
|
+
}
|
|
10335
10400
|
}
|
|
10336
10401
|
}
|
|
10337
10402
|
function maintenanceStatus(state) {
|
|
@@ -10339,12 +10404,13 @@ function maintenanceStatus(state) {
|
|
|
10339
10404
|
const tombstoneRatio = storeSize > 0 ? state.tombstones.size / storeSize : 0;
|
|
10340
10405
|
const graphCount = state.hnsw ? 1 : 0;
|
|
10341
10406
|
const estimatedCompactMs = Math.round(state.tombstones.size * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
|
|
10342
|
-
const
|
|
10407
|
+
const optimizeVectorCount = state.hnsw === null || graphNeedsRebuild(state) ? storeSize : state.buffer.size;
|
|
10408
|
+
const estimatedOptimizeMs = Math.round(optimizeVectorCount * ESTIMATED_MS_PER_VECTOR_REBUILD * state.dimensionScale);
|
|
10343
10409
|
return {
|
|
10344
10410
|
tombstoneRatio,
|
|
10345
10411
|
graphCount,
|
|
10346
10412
|
bufferSize: state.buffer.size,
|
|
10347
|
-
building: state.building,
|
|
10413
|
+
building: state.building || state.buildScheduled,
|
|
10348
10414
|
estimatedCompactMs,
|
|
10349
10415
|
estimatedOptimizeMs
|
|
10350
10416
|
};
|
|
@@ -10435,10 +10501,7 @@ function deserialize(state, payload) {
|
|
|
10435
10501
|
state.store.clear();
|
|
10436
10502
|
state.tombstones.clear();
|
|
10437
10503
|
state.buffer.clear();
|
|
10438
|
-
|
|
10439
|
-
state.hnsw.clear();
|
|
10440
|
-
state.hnsw = null;
|
|
10441
|
-
}
|
|
10504
|
+
adoptGraph(state, null);
|
|
10442
10505
|
if (state.sq8) {
|
|
10443
10506
|
state.sq8.clear();
|
|
10444
10507
|
}
|
|
@@ -10463,7 +10526,7 @@ function deserialize(state, payload) {
|
|
|
10463
10526
|
state.sq8 ?? void 0
|
|
10464
10527
|
);
|
|
10465
10528
|
restoredHnsw.deserialize(graphData);
|
|
10466
|
-
state
|
|
10529
|
+
adoptGraph(state, restoredHnsw);
|
|
10467
10530
|
for (let i = 1; i < payload.graphs.length; i++) {
|
|
10468
10531
|
const additionalGraph = payload.graphs[i];
|
|
10469
10532
|
for (const [nodeDocId] of additionalGraph.nodes) {
|
|
@@ -10479,8 +10542,7 @@ function deserialize(state, payload) {
|
|
|
10479
10542
|
}
|
|
10480
10543
|
}
|
|
10481
10544
|
if (state.buffer.size > restoredHnsw.size) {
|
|
10482
|
-
state
|
|
10483
|
-
state.hnsw = null;
|
|
10545
|
+
adoptGraph(state, null);
|
|
10484
10546
|
state.buffer.clear();
|
|
10485
10547
|
for (const [docId] of state.store.entries()) {
|
|
10486
10548
|
if (state.tombstones.has(docId)) continue;
|
|
@@ -10495,6 +10557,87 @@ function deserialize(state, payload) {
|
|
|
10495
10557
|
}
|
|
10496
10558
|
}
|
|
10497
10559
|
|
|
10560
|
+
// src/core/heap.ts
|
|
10561
|
+
function createMaxHeap(compare) {
|
|
10562
|
+
return createHeap((a, b) => compare(b, a));
|
|
10563
|
+
}
|
|
10564
|
+
function createBoundedMaxHeap(compare, capacity) {
|
|
10565
|
+
const heap = createMaxHeap(compare);
|
|
10566
|
+
const innerPush = heap.push;
|
|
10567
|
+
return {
|
|
10568
|
+
push(item) {
|
|
10569
|
+
if (heap.size < capacity) {
|
|
10570
|
+
innerPush(item);
|
|
10571
|
+
return;
|
|
10572
|
+
}
|
|
10573
|
+
const top = heap.peek();
|
|
10574
|
+
if (top !== void 0 && compare(item, top) < 0) {
|
|
10575
|
+
heap.pop();
|
|
10576
|
+
innerPush(item);
|
|
10577
|
+
}
|
|
10578
|
+
},
|
|
10579
|
+
pop: heap.pop,
|
|
10580
|
+
peek: heap.peek,
|
|
10581
|
+
get size() {
|
|
10582
|
+
return heap.size;
|
|
10583
|
+
},
|
|
10584
|
+
toSortedArray: heap.toSortedArray
|
|
10585
|
+
};
|
|
10586
|
+
}
|
|
10587
|
+
function createHeap(compare) {
|
|
10588
|
+
const items = [];
|
|
10589
|
+
function siftUp(idx) {
|
|
10590
|
+
while (idx > 0) {
|
|
10591
|
+
const parent = idx - 1 >> 1;
|
|
10592
|
+
if (compare(items[idx], items[parent]) >= 0) break;
|
|
10593
|
+
const tmp = items[idx];
|
|
10594
|
+
items[idx] = items[parent];
|
|
10595
|
+
items[parent] = tmp;
|
|
10596
|
+
idx = parent;
|
|
10597
|
+
}
|
|
10598
|
+
}
|
|
10599
|
+
function siftDown2(idx) {
|
|
10600
|
+
const len = items.length;
|
|
10601
|
+
while (true) {
|
|
10602
|
+
let smallest = idx;
|
|
10603
|
+
const left = 2 * idx + 1;
|
|
10604
|
+
const right = 2 * idx + 2;
|
|
10605
|
+
if (left < len && compare(items[left], items[smallest]) < 0) smallest = left;
|
|
10606
|
+
if (right < len && compare(items[right], items[smallest]) < 0) smallest = right;
|
|
10607
|
+
if (smallest === idx) break;
|
|
10608
|
+
const tmp = items[idx];
|
|
10609
|
+
items[idx] = items[smallest];
|
|
10610
|
+
items[smallest] = tmp;
|
|
10611
|
+
idx = smallest;
|
|
10612
|
+
}
|
|
10613
|
+
}
|
|
10614
|
+
return {
|
|
10615
|
+
push(item) {
|
|
10616
|
+
items.push(item);
|
|
10617
|
+
siftUp(items.length - 1);
|
|
10618
|
+
},
|
|
10619
|
+
pop() {
|
|
10620
|
+
if (items.length === 0) return void 0;
|
|
10621
|
+
const top = items[0];
|
|
10622
|
+
const last = items.pop();
|
|
10623
|
+
if (items.length > 0) {
|
|
10624
|
+
items[0] = last;
|
|
10625
|
+
siftDown2(0);
|
|
10626
|
+
}
|
|
10627
|
+
return top;
|
|
10628
|
+
},
|
|
10629
|
+
peek() {
|
|
10630
|
+
return items[0];
|
|
10631
|
+
},
|
|
10632
|
+
get size() {
|
|
10633
|
+
return items.length;
|
|
10634
|
+
},
|
|
10635
|
+
toSortedArray() {
|
|
10636
|
+
return [...items].sort(compare);
|
|
10637
|
+
}
|
|
10638
|
+
};
|
|
10639
|
+
}
|
|
10640
|
+
|
|
10498
10641
|
// src/vector/vector-index/search.ts
|
|
10499
10642
|
function* bufferCandidates(state, filter) {
|
|
10500
10643
|
for (const docId of state.buffer) {
|
|
@@ -10645,6 +10788,7 @@ function createVectorIndex(fieldName, dimension, config) {
|
|
|
10645
10788
|
buffer: /* @__PURE__ */ new Set(),
|
|
10646
10789
|
sq8: quantizationMode === "sq8" ? createScalarQuantizer(dimension, store) : null,
|
|
10647
10790
|
hnsw: null,
|
|
10791
|
+
compactedNodeCount: 0,
|
|
10648
10792
|
building: false,
|
|
10649
10793
|
buildScheduled: false,
|
|
10650
10794
|
pendingBuild: null,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ReplicationMessageTypes } from './chunk-JQUL2T6I.mjs';
|
|
2
2
|
import { MAX_INDEX_NAME_LENGTH } from './chunk-XWICBGYT.mjs';
|
|
3
|
-
import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, clampRowCount, DEFAULT_PAGE_SIZE, validateSchema, validateVectorPromotion, collectQueryTermStats, createPartitionManager, createVectorIndex, flattenSchema, isTextFieldType, fnv1aBytes, requireWithinResultWindow, normalizeSort, sortSignatureOf, requireSortableFields, fanOutQuery, applySorting, applyPagination, resolveProjection, countsWithoutScores, fanOutMatchCount, tokenize, decodePageCursor, requireMatchingCursor, encodePageCursor, resolveIndexAnalysis, deserializePayloadV2, deserializePayloadV1, mergePartitionStats, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, createPartitionIndex, fnv1a, createWorkerPool, partitionsIn, detectRuntime, isNodeMainThread, spawnNodeWorker, createSharedFrozenSegment, buildCompactedSegmentPayload, freezeSegmentShared, createFrozenSegment, deserializeMetadata, isCompositePartition, serializeMetadata, kWayMerge, validateRequiredFields, validateDocument, validateDocumentStrict } from './chunk-
|
|
3
|
+
import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, clampRowCount, DEFAULT_PAGE_SIZE, validateSchema, validateVectorPromotion, collectQueryTermStats, createPartitionManager, createVectorIndex, flattenSchema, isTextFieldType, fnv1aBytes, requireWithinResultWindow, normalizeSort, sortSignatureOf, requireSortableFields, fanOutQuery, applySorting, applyPagination, resolveProjection, countsWithoutScores, fanOutMatchCount, tokenize, decodePageCursor, requireMatchingCursor, encodePageCursor, resolveIndexAnalysis, deserializePayloadV2, deserializePayloadV1, mergePartitionStats, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, createPartitionIndex, fnv1a, createWorkerPool, partitionsIn, detectRuntime, isNodeMainThread, spawnNodeWorker, createSharedFrozenSegment, buildCompactedSegmentPayload, freezeSegmentShared, createFrozenSegment, deserializeMetadata, isCompositePartition, serializeMetadata, kWayMerge, validateRequiredFields, validateDocument, validateDocumentStrict } from './chunk-EV45L35L.mjs';
|
|
4
4
|
import { crc32, crc32Init, crc32Update, crc32Final } from './chunk-KMMJMDUA.mjs';
|
|
5
5
|
import { compareCodePoints, compareComparableKeys } from './chunk-BFIH7OV5.mjs';
|
|
6
6
|
import { generateId, createRequestId, isValidWorkerResponse } from './chunk-WTONLDU3.mjs';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export { createDirectExecutor } from './chunk-
|
|
1
|
+
export { createDirectExecutor } from './chunk-EV45L35L.mjs';
|
|
2
2
|
import './chunk-TGP5M5R7.mjs';
|
|
3
|
-
import './chunk-
|
|
4
|
-
import './chunk-
|
|
3
|
+
import './chunk-4COAUBLJ.mjs';
|
|
4
|
+
import './chunk-DFZDV5FV.mjs';
|
|
5
5
|
import './chunk-U2BKYEC7.mjs';
|
|
6
6
|
import './chunk-BFIH7OV5.mjs';
|
|
7
7
|
import './chunk-WTONLDU3.mjs';
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, listBindingOf, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, queryBindingOf, foldGroupReducer, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, computeEntryChecksum, dedupePinnedEntries, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-
|
|
1
|
+
import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, listBindingOf, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, queryBindingOf, foldGroupReducer, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, computeEntryChecksum, dedupePinnedEntries, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-UZTT2BKQ.mjs';
|
|
2
2
|
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-JQUL2T6I.mjs';
|
|
3
3
|
import { validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH } from './chunk-XWICBGYT.mjs';
|
|
4
|
-
import { deserializePayloadV2, createPartitionIndex, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, getLanguage, DEFAULT_PAGE_SIZE, requireWithinResultWindow, fnv1a, FILTER_EXPRESSION_KEYS, MAX_CURSOR_LENGTH, mergePartitionStats, RESULT_WINDOW, FIELD_FILTER_OPERATORS, MAX_NESTING_DEPTH } from './chunk-
|
|
4
|
+
import { deserializePayloadV2, createPartitionIndex, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, getLanguage, DEFAULT_PAGE_SIZE, requireWithinResultWindow, fnv1a, FILTER_EXPRESSION_KEYS, MAX_CURSOR_LENGTH, mergePartitionStats, RESULT_WINDOW, FIELD_FILTER_OPERATORS, MAX_NESTING_DEPTH } from './chunk-EV45L35L.mjs';
|
|
5
5
|
import './chunk-TGP5M5R7.mjs';
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-
|
|
6
|
+
import './chunk-4COAUBLJ.mjs';
|
|
7
|
+
import './chunk-DFZDV5FV.mjs';
|
|
8
8
|
import { crc32 } from './chunk-KMMJMDUA.mjs';
|
|
9
9
|
import './chunk-U2BKYEC7.mjs';
|
|
10
10
|
import { compareCodePoints, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-BFIH7OV5.mjs';
|