@delali/narsil 0.3.1-next.82 → 0.3.1-next.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-IH3GKNDQ.mjs → chunk-7JZZ3HNV.mjs} +20 -10
- package/dist/{chunk-4CWHWOBI.mjs → chunk-CUP2E4OH.mjs} +166 -90
- package/dist/{chunk-IN5CH4P2.mjs → chunk-RZXNWQSM.mjs} +181 -92
- package/dist/{direct-executor-JVXDR5Y5.mjs → direct-executor-6HASZ36Y.mjs} +2 -2
- package/dist/distribution.mjs +3 -3
- package/dist/index.browser.mjs +401 -226
- package/dist/index.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +3 -3
- package/dist/vector/search-worker.d.ts +2 -0
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +1 -1
- package/package.json +7 -7
|
@@ -1664,7 +1664,7 @@ function readNeighborsLocked(state, ord, layer) {
|
|
|
1664
1664
|
if (nodeReadHeld(state.locks, ord, version)) return count;
|
|
1665
1665
|
}
|
|
1666
1666
|
}
|
|
1667
|
-
function searchLayer(state, qVec, qMag, ef, layer, metric,
|
|
1667
|
+
function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, distFn, results) {
|
|
1668
1668
|
const getDistance = distFn ?? ((ord) => queryDistanceByOrd(state, qVec, qMag, ord, metric));
|
|
1669
1669
|
const workspace = state.workspace;
|
|
1670
1670
|
const frontier = workspace.frontier;
|
|
@@ -1680,10 +1680,10 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distF
|
|
|
1680
1680
|
if (state.visited[epOrd] === stamp) continue;
|
|
1681
1681
|
state.visited[epOrd] = stamp;
|
|
1682
1682
|
if (!nodeExists(state, epOrd)) continue;
|
|
1683
|
-
if (skipTombstones && isTombstoned(state, epOrd)) continue;
|
|
1684
1683
|
const dist = getDistance(epOrd);
|
|
1685
1684
|
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
1686
1685
|
pushHeap(frontier, epOrd, dist);
|
|
1686
|
+
if (excludeTombstones && isTombstoned(state, epOrd)) continue;
|
|
1687
1687
|
pushHeap(found, epOrd, dist);
|
|
1688
1688
|
if (found.size > ef) {
|
|
1689
1689
|
popHeap(found);
|
|
@@ -1691,19 +1691,19 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distF
|
|
|
1691
1691
|
}
|
|
1692
1692
|
if (found.size > 0) furthestDist = found.distances[0];
|
|
1693
1693
|
while (popHeap(frontier)) {
|
|
1694
|
-
if (frontier.topDistance > furthestDist) break;
|
|
1694
|
+
if (frontier.topDistance > furthestDist && found.size >= ef) break;
|
|
1695
1695
|
const count = readNeighborsLocked(state, frontier.topOrd, layer);
|
|
1696
1696
|
for (let i = 0; i < count; i++) {
|
|
1697
1697
|
const neighborOrd = scratch[i];
|
|
1698
1698
|
ensureVisited(state, neighborOrd + 1);
|
|
1699
1699
|
if (state.visited[neighborOrd] === stamp) continue;
|
|
1700
1700
|
state.visited[neighborOrd] = stamp;
|
|
1701
|
-
if (skipTombstones && isTombstoned(state, neighborOrd)) continue;
|
|
1702
1701
|
if (!nodeExists(state, neighborOrd)) continue;
|
|
1703
1702
|
const dist = getDistance(neighborOrd);
|
|
1704
1703
|
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
1705
1704
|
if (dist < furthestDist || found.size < ef) {
|
|
1706
1705
|
pushHeap(frontier, neighborOrd, dist);
|
|
1706
|
+
if (excludeTombstones && isTombstoned(state, neighborOrd)) continue;
|
|
1707
1707
|
pushHeap(found, neighborOrd, dist);
|
|
1708
1708
|
if (found.size > ef) {
|
|
1709
1709
|
popHeap(found);
|
|
@@ -1981,13 +1981,6 @@ function markTombstone(state, ord) {
|
|
|
1981
1981
|
if (Atomics.compareExchange(state.tombstones, ord, 0, 1) === 0) {
|
|
1982
1982
|
Atomics.add(state.header, GRAPH_TOMBSTONE_COUNT, 1);
|
|
1983
1983
|
}
|
|
1984
|
-
if (entryPointOf(state) !== ord) return;
|
|
1985
|
-
lockEntry(state.locks);
|
|
1986
|
-
try {
|
|
1987
|
-
if (entryPointOf(state) === ord) setEntryPoint(state, highestNode(state, false));
|
|
1988
|
-
} finally {
|
|
1989
|
-
unlockEntry(state.locks);
|
|
1990
|
-
}
|
|
1991
1984
|
}
|
|
1992
1985
|
function compactTombstones(state) {
|
|
1993
1986
|
if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
|
|
@@ -2166,7 +2159,7 @@ function traverse(state, query, k, searchMetric, options) {
|
|
|
2166
2159
|
if (entryPoint === -1) return { candidates, depth: useQuantized ? depth : 0, arenaQuery, qMag, native: null };
|
|
2167
2160
|
setSingleEntryPoint(workspace, entryPoint);
|
|
2168
2161
|
for (let layer = topLayerOf(state); layer >= 1; layer--) {
|
|
2169
|
-
searchLayer(state, query, qMag, 1, layer, searchMetric,
|
|
2162
|
+
searchLayer(state, query, qMag, 1, layer, searchMetric, false, distFn, candidates);
|
|
2170
2163
|
if (candidates.size > 0) {
|
|
2171
2164
|
setSingleEntryPoint(workspace, candidates.ords[0]);
|
|
2172
2165
|
}
|
|
@@ -3278,9 +3271,17 @@ function createOsqQuantizer(dimension, bits, metric, store) {
|
|
|
3278
3271
|
if (view === null || ordinal === void 0 || !view.holdsOrdinal(ordinal)) return void 0;
|
|
3279
3272
|
return view.recordAt(ordinal);
|
|
3280
3273
|
},
|
|
3274
|
+
recordAtOrdinal(ordinal) {
|
|
3275
|
+
const view = shared();
|
|
3276
|
+
if (view === null || !view.holdsOrdinal(ordinal)) return void 0;
|
|
3277
|
+
return view.recordAt(ordinal);
|
|
3278
|
+
},
|
|
3281
3279
|
restoreRecord(docId, record) {
|
|
3282
3280
|
requireShared().restoreRecord(ordinalOf(docId), record);
|
|
3283
3281
|
},
|
|
3282
|
+
restoreRecordAtOrdinal(ordinal, record) {
|
|
3283
|
+
requireShared().restoreRecord(ordinal, record);
|
|
3284
|
+
},
|
|
3284
3285
|
restoreCentroid(centroid) {
|
|
3285
3286
|
requireShared().writeCentroid(centroid);
|
|
3286
3287
|
},
|
|
@@ -3332,6 +3333,78 @@ function resetDocIds(handles) {
|
|
|
3332
3333
|
Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
|
|
3333
3334
|
}
|
|
3334
3335
|
|
|
3336
|
+
// src/vector/vector-store/ordinal-reads.ts
|
|
3337
|
+
function entryAt(store, ordinal) {
|
|
3338
|
+
return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
|
|
3339
|
+
}
|
|
3340
|
+
function createOrdinalReads(openStore, readable) {
|
|
3341
|
+
function pairDistance(metric) {
|
|
3342
|
+
const open = openStore();
|
|
3343
|
+
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
3344
|
+
const inner = open.view.pairDistance(metric);
|
|
3345
|
+
return (ordA, ordB) => readable(ordA) && readable(ordB) ? inner(ordA, ordB) : Number.POSITIVE_INFINITY;
|
|
3346
|
+
}
|
|
3347
|
+
return {
|
|
3348
|
+
entryForOrdinal(ordinal) {
|
|
3349
|
+
const open = openStore();
|
|
3350
|
+
if (open === null || !readable(ordinal)) return void 0;
|
|
3351
|
+
return entryAt(open, ordinal);
|
|
3352
|
+
},
|
|
3353
|
+
distanceByOrdinal(ordA, ordB, metric) {
|
|
3354
|
+
const open = openStore();
|
|
3355
|
+
if (open === null || !readable(ordA) || !readable(ordB)) return Number.POSITIVE_INFINITY;
|
|
3356
|
+
return open.view.distanceByOrdinal(ordA, ordB, metric);
|
|
3357
|
+
},
|
|
3358
|
+
distanceFromArena(prepared, ordinal, metric) {
|
|
3359
|
+
const open = openStore();
|
|
3360
|
+
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
3361
|
+
return open.view.distanceFromArena(prepared, ordinal, metric);
|
|
3362
|
+
},
|
|
3363
|
+
queryDistance(prepared, metric) {
|
|
3364
|
+
const open = openStore();
|
|
3365
|
+
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
3366
|
+
const inner = open.view.queryDistance(prepared, metric);
|
|
3367
|
+
return (ordinal) => readable(ordinal) ? inner(ordinal) : Number.POSITIVE_INFINITY;
|
|
3368
|
+
},
|
|
3369
|
+
ordinalDistance(from, metric) {
|
|
3370
|
+
const pair = pairDistance(metric);
|
|
3371
|
+
return (ordinal) => pair(from, ordinal);
|
|
3372
|
+
},
|
|
3373
|
+
pairDistance
|
|
3374
|
+
};
|
|
3375
|
+
}
|
|
3376
|
+
|
|
3377
|
+
// src/vector/vector-store/listing.ts
|
|
3378
|
+
function* entriesOf(open, documents) {
|
|
3379
|
+
if (open === null) return;
|
|
3380
|
+
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
3381
|
+
const docId = documents[ordinal];
|
|
3382
|
+
if (docId === void 0) continue;
|
|
3383
|
+
yield [docId, entryAt(open, ordinal)];
|
|
3384
|
+
}
|
|
3385
|
+
}
|
|
3386
|
+
function* docIdsOf(open, documents) {
|
|
3387
|
+
if (open === null) return;
|
|
3388
|
+
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
3389
|
+
const docId = documents[ordinal];
|
|
3390
|
+
if (docId !== void 0) yield docId;
|
|
3391
|
+
}
|
|
3392
|
+
}
|
|
3393
|
+
function snapshotOf(open, documents, replacedOrdinals) {
|
|
3394
|
+
const slots = documents.length;
|
|
3395
|
+
const dimension = open === null ? 0 : open.handles.dimension;
|
|
3396
|
+
const vectors = new Float32Array(slots * dimension);
|
|
3397
|
+
const magnitudes = new Float64Array(slots);
|
|
3398
|
+
const docIds = new Array(slots);
|
|
3399
|
+
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
3400
|
+
docIds[ordinal] = documents[ordinal] ?? null;
|
|
3401
|
+
if (open === null || documents[ordinal] === void 0 && !replacedOrdinals.has(ordinal)) continue;
|
|
3402
|
+
vectors.set(open.view.vectorAt(ordinal), ordinal * dimension);
|
|
3403
|
+
magnitudes[ordinal] = open.magnitudes[ordinal];
|
|
3404
|
+
}
|
|
3405
|
+
return { dimension, slots, vectors, magnitudes, docIds, replacedOrdinals: [...replacedOrdinals] };
|
|
3406
|
+
}
|
|
3407
|
+
|
|
3335
3408
|
// src/vector/vector-store/open-store.ts
|
|
3336
3409
|
function openHandles(dimension, codeBits, blockBytes2) {
|
|
3337
3410
|
const handles = createSharedVectorStoreHandles(dimension, codeBits, blockBytes2);
|
|
@@ -3407,13 +3480,14 @@ function writeVector(store, ordinal, vector) {
|
|
|
3407
3480
|
block.float32(base + vector.length).set(vector, base);
|
|
3408
3481
|
store.magnitudes[ordinal] = magnitude(vector);
|
|
3409
3482
|
}
|
|
3410
|
-
function blockFullyCold(store, blockIndex, documents) {
|
|
3483
|
+
function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
|
|
3411
3484
|
const capacity = store.handles.layout.capacity;
|
|
3412
3485
|
const slots = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
3413
3486
|
const end = Math.min(slots, (blockIndex + 1) * capacity);
|
|
3414
3487
|
if (end < (blockIndex + 1) * capacity) return false;
|
|
3415
3488
|
for (let ordinal = blockIndex * capacity; ordinal < end; ordinal++) {
|
|
3416
|
-
|
|
3489
|
+
const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
|
|
3490
|
+
if (held && store.diskFile[ordinal] === IN_MEMORY) return false;
|
|
3417
3491
|
}
|
|
3418
3492
|
return true;
|
|
3419
3493
|
}
|
|
@@ -3426,10 +3500,11 @@ function fileHoldsSameBytes(store, ordinal, location) {
|
|
|
3426
3500
|
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
|
3427
3501
|
return true;
|
|
3428
3502
|
}
|
|
3429
|
-
function forgetUnreferencedFiles(store, documents) {
|
|
3503
|
+
function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
|
|
3430
3504
|
const referenced = /* @__PURE__ */ new Set();
|
|
3431
3505
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
3432
|
-
|
|
3506
|
+
const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
|
|
3507
|
+
if (held && store.diskFile[ordinal] !== IN_MEMORY) {
|
|
3433
3508
|
referenced.add(store.diskFile[ordinal]);
|
|
3434
3509
|
}
|
|
3435
3510
|
}
|
|
@@ -3470,15 +3545,20 @@ function createVectorStore(options) {
|
|
|
3470
3545
|
const blockBytes2 = options?.blockBytes ?? BLOCK_MAX_BYTES;
|
|
3471
3546
|
const docToOrd = /* @__PURE__ */ new Map();
|
|
3472
3547
|
const ordToDoc = [];
|
|
3548
|
+
const replacedOrdinals = /* @__PURE__ */ new Set();
|
|
3473
3549
|
let unknownPartitions = 0;
|
|
3474
3550
|
let liveCount = 0;
|
|
3475
3551
|
let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes2);
|
|
3476
3552
|
function forgetDocuments() {
|
|
3477
3553
|
docToOrd.clear();
|
|
3478
3554
|
ordToDoc.length = 0;
|
|
3555
|
+
replacedOrdinals.clear();
|
|
3479
3556
|
unknownPartitions = 0;
|
|
3480
3557
|
liveCount = 0;
|
|
3481
3558
|
}
|
|
3559
|
+
function readable(ordinal) {
|
|
3560
|
+
return ordToDoc[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
|
|
3561
|
+
}
|
|
3482
3562
|
function requireOpen() {
|
|
3483
3563
|
if (open === null) {
|
|
3484
3564
|
throw new NarsilError(ErrorCodes.VECTOR_DIMENSION_MISMATCH, "The vector store holds no vector yet");
|
|
@@ -3495,11 +3575,27 @@ function createVectorStore(options) {
|
|
|
3495
3575
|
Atomics.store(store.present, ordinal, 0);
|
|
3496
3576
|
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
3497
3577
|
}
|
|
3498
|
-
function
|
|
3578
|
+
function keepReplacedOrdinal(store, ordinal) {
|
|
3579
|
+
ordToDoc[ordinal] = void 0;
|
|
3580
|
+
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
3581
|
+
store.partitions[ordinal] = NO_PARTITION;
|
|
3582
|
+
replacedOrdinals.add(ordinal);
|
|
3583
|
+
}
|
|
3584
|
+
function restoreReplacedVector(store, ordinal, vector) {
|
|
3585
|
+
store.diskFile[ordinal] = IN_MEMORY;
|
|
3586
|
+
writeVector(store, ordinal, vector);
|
|
3587
|
+
Atomics.store(store.present, ordinal, 1);
|
|
3588
|
+
replacedOrdinals.add(ordinal);
|
|
3589
|
+
}
|
|
3590
|
+
function claimSlot(store, docId, cold) {
|
|
3499
3591
|
const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
3500
3592
|
ensureOrdinal(store, ordinal, cold);
|
|
3501
3593
|
store.diskFile[ordinal] = IN_MEMORY;
|
|
3502
3594
|
appendDocId(store.handles, ordinal, docId);
|
|
3595
|
+
return ordinal;
|
|
3596
|
+
}
|
|
3597
|
+
function claimOrdinal(store, docId, cold, partitionId) {
|
|
3598
|
+
const ordinal = claimSlot(store, docId, cold);
|
|
3503
3599
|
recordPartition(store, ordinal, partitionId);
|
|
3504
3600
|
ordToDoc[ordinal] = docId;
|
|
3505
3601
|
docToOrd.set(docId, ordinal);
|
|
@@ -3509,25 +3605,30 @@ function createVectorStore(options) {
|
|
|
3509
3605
|
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
3510
3606
|
Atomics.store(store.present, ordinal, 1);
|
|
3511
3607
|
}
|
|
3608
|
+
function publishOnDisk(store, ordinal, vectorMagnitude, location) {
|
|
3609
|
+
store.magnitudes[ordinal] = vectorMagnitude;
|
|
3610
|
+
store.diskOffset[ordinal] = location.offset;
|
|
3611
|
+
store.diskFile[ordinal] = location.fileIndex;
|
|
3612
|
+
publishOrdinal(store, ordinal);
|
|
3613
|
+
}
|
|
3512
3614
|
function appendOrdinal(store, docId, vector, partitionId) {
|
|
3513
3615
|
const ordinal = claimOrdinal(store, docId, false, partitionId);
|
|
3514
3616
|
writeVector(store, ordinal, vector);
|
|
3515
3617
|
publishOrdinal(store, ordinal);
|
|
3516
3618
|
return ordinal;
|
|
3517
3619
|
}
|
|
3518
|
-
function settleReplacement(store, ordinal, previous) {
|
|
3620
|
+
function settleReplacement(store, ordinal, previous, keepsReplacedVector) {
|
|
3519
3621
|
if (previous !== void 0) {
|
|
3520
|
-
|
|
3622
|
+
if (keepsReplacedVector) keepReplacedOrdinal(store, previous);
|
|
3623
|
+
else retireOrdinal(store, previous);
|
|
3521
3624
|
return ordinal;
|
|
3522
3625
|
}
|
|
3523
3626
|
liveCount++;
|
|
3524
3627
|
Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
3525
3628
|
return ordinal;
|
|
3526
3629
|
}
|
|
3527
|
-
function entryAt(store, ordinal) {
|
|
3528
|
-
return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
|
|
3529
|
-
}
|
|
3530
3630
|
return {
|
|
3631
|
+
...createOrdinalReads(() => open, readable),
|
|
3531
3632
|
get size() {
|
|
3532
3633
|
return liveCount;
|
|
3533
3634
|
},
|
|
@@ -3546,7 +3647,10 @@ function createVectorStore(options) {
|
|
|
3546
3647
|
get view() {
|
|
3547
3648
|
return requireOpen().view;
|
|
3548
3649
|
},
|
|
3549
|
-
|
|
3650
|
+
get replacedVectorCount() {
|
|
3651
|
+
return replacedOrdinals.size;
|
|
3652
|
+
},
|
|
3653
|
+
insert(docId, vector, partitionId, keepsReplacedVector = false) {
|
|
3550
3654
|
if (open === null) open = openHandles(vector.length, codeBits, blockBytes2);
|
|
3551
3655
|
const store = open;
|
|
3552
3656
|
if (vector.length !== store.handles.dimension) {
|
|
@@ -3558,18 +3662,37 @@ function createVectorStore(options) {
|
|
|
3558
3662
|
}
|
|
3559
3663
|
const previous = docToOrd.get(docId);
|
|
3560
3664
|
const ordinal = appendOrdinal(store, docId, vector, partitionId);
|
|
3561
|
-
return settleReplacement(store, ordinal, previous);
|
|
3665
|
+
return settleReplacement(store, ordinal, previous, keepsReplacedVector);
|
|
3666
|
+
},
|
|
3667
|
+
releaseReplacedVectors() {
|
|
3668
|
+
const released = [...replacedOrdinals];
|
|
3669
|
+
replacedOrdinals.clear();
|
|
3670
|
+
if (open !== null) for (const ordinal of released) Atomics.store(open.present, ordinal, 0);
|
|
3671
|
+
return released;
|
|
3562
3672
|
},
|
|
3563
3673
|
insertCold(docId, vectorMagnitude, location, partitionId) {
|
|
3564
3674
|
const store = requireOpen();
|
|
3565
3675
|
const previous = docToOrd.get(docId);
|
|
3566
3676
|
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
3567
3677
|
const ordinal = claimOrdinal(store, docId, true, partitionId);
|
|
3568
|
-
store
|
|
3569
|
-
store
|
|
3570
|
-
|
|
3678
|
+
publishOnDisk(store, ordinal, vectorMagnitude, location);
|
|
3679
|
+
return settleReplacement(store, ordinal, previous, false);
|
|
3680
|
+
},
|
|
3681
|
+
insertReplaced(docId, vector, location) {
|
|
3682
|
+
if (open === null) open = openHandles(vector.length, codeBits, blockBytes2);
|
|
3683
|
+
const store = open;
|
|
3684
|
+
if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
3685
|
+
const ordinal = claimSlot(store, docId, location !== void 0);
|
|
3686
|
+
store.partitions[ordinal] = NO_PARTITION;
|
|
3687
|
+
ordToDoc[ordinal] = void 0;
|
|
3688
|
+
replacedOrdinals.add(ordinal);
|
|
3689
|
+
if (location !== void 0) {
|
|
3690
|
+
publishOnDisk(store, ordinal, magnitude(vector), location);
|
|
3691
|
+
return ordinal;
|
|
3692
|
+
}
|
|
3693
|
+
writeVector(store, ordinal, vector);
|
|
3571
3694
|
publishOrdinal(store, ordinal);
|
|
3572
|
-
return
|
|
3695
|
+
return ordinal;
|
|
3573
3696
|
},
|
|
3574
3697
|
addVectorFile(path) {
|
|
3575
3698
|
const store = requireOpen();
|
|
@@ -3596,11 +3719,11 @@ function createVectorStore(options) {
|
|
|
3596
3719
|
const store = requireOpen();
|
|
3597
3720
|
let released = 0;
|
|
3598
3721
|
for (let index = 0; index < store.handles.blocks.length; index++) {
|
|
3599
|
-
if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc)) continue;
|
|
3722
|
+
if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, replacedOrdinals)) continue;
|
|
3600
3723
|
store.handles.blocks[index] = null;
|
|
3601
3724
|
released += 1;
|
|
3602
3725
|
}
|
|
3603
|
-
if (released + forgetUnreferencedFiles(store, ordToDoc) > 0) relayout(store);
|
|
3726
|
+
if (released + forgetUnreferencedFiles(store, ordToDoc, replacedOrdinals) > 0) relayout(store);
|
|
3604
3727
|
return released;
|
|
3605
3728
|
},
|
|
3606
3729
|
setPartition(docId, partitionId) {
|
|
@@ -3644,21 +3767,8 @@ function createVectorStore(options) {
|
|
|
3644
3767
|
holdsOrdinal(ordinal) {
|
|
3645
3768
|
return ordinal >= 0 && ordinal < ordToDoc.length && ordToDoc[ordinal] !== void 0;
|
|
3646
3769
|
},
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
|
|
3650
|
-
const docId = ordToDoc[ordinal];
|
|
3651
|
-
if (docId === void 0) continue;
|
|
3652
|
-
yield [docId, entryAt(open, ordinal)];
|
|
3653
|
-
}
|
|
3654
|
-
},
|
|
3655
|
-
*docIds() {
|
|
3656
|
-
if (open === null) return;
|
|
3657
|
-
for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
|
|
3658
|
-
const docId = ordToDoc[ordinal];
|
|
3659
|
-
if (docId !== void 0) yield docId;
|
|
3660
|
-
}
|
|
3661
|
-
},
|
|
3770
|
+
entries: () => entriesOf(open, ordToDoc),
|
|
3771
|
+
docIds: () => docIdsOf(open, ordToDoc),
|
|
3662
3772
|
clear() {
|
|
3663
3773
|
forgetDocuments();
|
|
3664
3774
|
if (open === null) return;
|
|
@@ -3690,59 +3800,16 @@ function createVectorStore(options) {
|
|
|
3690
3800
|
docIdForOrdinal(ordinal) {
|
|
3691
3801
|
return ordToDoc[ordinal];
|
|
3692
3802
|
},
|
|
3693
|
-
entryForOrdinal(ordinal) {
|
|
3694
|
-
if (open === null || ordinal < 0 || ordinal >= ordToDoc.length || ordToDoc[ordinal] === void 0) {
|
|
3695
|
-
return void 0;
|
|
3696
|
-
}
|
|
3697
|
-
return entryAt(open, ordinal);
|
|
3698
|
-
},
|
|
3699
|
-
distanceByOrdinal(ordA, ordB, metric) {
|
|
3700
|
-
if (open === null || ordToDoc[ordA] === void 0 || ordToDoc[ordB] === void 0) {
|
|
3701
|
-
return Number.POSITIVE_INFINITY;
|
|
3702
|
-
}
|
|
3703
|
-
return open.view.distanceByOrdinal(ordA, ordB, metric);
|
|
3704
|
-
},
|
|
3705
3803
|
prepareQueryArena(query) {
|
|
3706
3804
|
return open === null ? null : open.view.prepareQueryArena(query);
|
|
3707
3805
|
},
|
|
3708
|
-
|
|
3709
|
-
if (open === null || ordToDoc[ordinal] === void 0) return Number.POSITIVE_INFINITY;
|
|
3710
|
-
return open.view.distanceFromArena(prepared, ordinal, metric);
|
|
3711
|
-
},
|
|
3712
|
-
queryDistance(prepared, metric) {
|
|
3713
|
-
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
3714
|
-
const inner = open.view.queryDistance(prepared, metric);
|
|
3715
|
-
return (ordinal) => ordToDoc[ordinal] === void 0 ? Number.POSITIVE_INFINITY : inner(ordinal);
|
|
3716
|
-
},
|
|
3717
|
-
ordinalDistance(from, metric) {
|
|
3718
|
-
const pair = this.pairDistance(metric);
|
|
3719
|
-
return (ordinal) => pair(from, ordinal);
|
|
3720
|
-
},
|
|
3721
|
-
pairDistance(metric) {
|
|
3722
|
-
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
3723
|
-
const inner = open.view.pairDistance(metric);
|
|
3724
|
-
return (ordA, ordB) => ordToDoc[ordA] === void 0 || ordToDoc[ordB] === void 0 ? Number.POSITIVE_INFINITY : inner(ordA, ordB);
|
|
3725
|
-
},
|
|
3726
|
-
exportSnapshot() {
|
|
3727
|
-
const slots = ordToDoc.length;
|
|
3728
|
-
const dimension = open === null ? 0 : open.handles.dimension;
|
|
3729
|
-
const vectors = new Float32Array(slots * dimension);
|
|
3730
|
-
const magnitudes = new Float64Array(slots);
|
|
3731
|
-
const docIds = new Array(slots);
|
|
3732
|
-
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
3733
|
-
const docId = ordToDoc[ordinal];
|
|
3734
|
-
docIds[ordinal] = docId ?? null;
|
|
3735
|
-
if (docId === void 0 || open === null) continue;
|
|
3736
|
-
vectors.set(open.view.vectorAt(ordinal), ordinal * dimension);
|
|
3737
|
-
magnitudes[ordinal] = open.magnitudes[ordinal];
|
|
3738
|
-
}
|
|
3739
|
-
return { dimension, slots, vectors, magnitudes, docIds };
|
|
3740
|
-
},
|
|
3806
|
+
exportSnapshot: () => snapshotOf(open, ordToDoc, replacedOrdinals),
|
|
3741
3807
|
restoreSnapshot(snapshot) {
|
|
3742
3808
|
this.clear();
|
|
3743
3809
|
if (snapshot.dimension === 0 || snapshot.slots === 0) return;
|
|
3744
3810
|
if (open === null) open = openHandles(snapshot.dimension, codeBits, blockBytes2);
|
|
3745
3811
|
const store = open;
|
|
3812
|
+
const replaced = new Set(snapshot.replacedOrdinals);
|
|
3746
3813
|
for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
|
|
3747
3814
|
const docId = snapshot.docIds[ordinal];
|
|
3748
3815
|
const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
|
|
@@ -3750,6 +3817,7 @@ function createVectorStore(options) {
|
|
|
3750
3817
|
ensureOrdinal(store, ordinal, false);
|
|
3751
3818
|
ordToDoc[ordinal] = void 0;
|
|
3752
3819
|
store.partitions[ordinal] = NO_PARTITION;
|
|
3820
|
+
if (replaced.has(ordinal)) restoreReplacedVector(store, ordinal, vector);
|
|
3753
3821
|
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
3754
3822
|
continue;
|
|
3755
3823
|
}
|
|
@@ -3794,7 +3862,7 @@ function valuesOf(bytes) {
|
|
|
3794
3862
|
return new Uint32Array(copy.buffer, 0, copy.byteLength / BYTES_PER_VALUE);
|
|
3795
3863
|
}
|
|
3796
3864
|
function savedTopLayer(state, ordinal) {
|
|
3797
|
-
if (ordinal === NO_ORDINAL
|
|
3865
|
+
if (ordinal === NO_ORDINAL) return NO_LAYER;
|
|
3798
3866
|
return Math.min(nodeMaxLayer(state, ordinal), MAX_LAYER_CAP);
|
|
3799
3867
|
}
|
|
3800
3868
|
function neighbourValueCount(state, topLayers, start, end) {
|
|
@@ -3860,6 +3928,25 @@ function serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, pa
|
|
|
3860
3928
|
}
|
|
3861
3929
|
return parts;
|
|
3862
3930
|
}
|
|
3931
|
+
function graphWithoutNodes(part, firstNumber, numbers) {
|
|
3932
|
+
const values = valuesOf(part.neighbours);
|
|
3933
|
+
const levels = part.levels.slice();
|
|
3934
|
+
const kept = new Uint32Array(values.length);
|
|
3935
|
+
let cursor = 0;
|
|
3936
|
+
let length = 0;
|
|
3937
|
+
for (let at = 0; at < part.levels.length; at++) {
|
|
3938
|
+
const start = cursor;
|
|
3939
|
+
for (let layer = 0; layer < part.levels[at]; layer++) cursor += values[cursor] + 1;
|
|
3940
|
+
if (numbers.has(firstNumber + at)) {
|
|
3941
|
+
levels[at] = 0;
|
|
3942
|
+
continue;
|
|
3943
|
+
}
|
|
3944
|
+
kept.set(values.subarray(start, cursor), length);
|
|
3945
|
+
length += cursor - start;
|
|
3946
|
+
}
|
|
3947
|
+
const entryPoint = part.entryPoint !== null && numbers.has(part.entryPoint) ? null : part.entryPoint;
|
|
3948
|
+
return { ...part, entryPoint, levels, neighbours: littleEndianBytesOf(kept.subarray(0, length)) };
|
|
3949
|
+
}
|
|
3863
3950
|
function endsInsideAList() {
|
|
3864
3951
|
throw new NarsilError(
|
|
3865
3952
|
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
@@ -4171,6 +4258,8 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
4171
4258
|
if (ord !== void 0) markTombstone(state, ord);
|
|
4172
4259
|
},
|
|
4173
4260
|
markTombstoneOrdinal: (ordinal) => markTombstone(state, ordinal),
|
|
4261
|
+
holdsNode: (ordinal) => hasNode(state.adjacency, ordinal),
|
|
4262
|
+
holdsTombstone: (ordinal) => hasNode(state.adjacency, ordinal) && isTombstoned(state, ordinal),
|
|
4174
4263
|
has: (docId) => {
|
|
4175
4264
|
const ord = store.getOrdinal(docId);
|
|
4176
4265
|
return ord !== void 0 && hasNode(state.adjacency, ord) && !isTombstoned(state, ord);
|
|
@@ -4194,4 +4283,4 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
4194
4283
|
};
|
|
4195
4284
|
}
|
|
4196
4285
|
|
|
4197
|
-
export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore, topLayerInParts };
|
|
4286
|
+
export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, graphWithoutNodes, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore, topLayerInParts };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export { createDirectExecutor } from './chunk-
|
|
1
|
+
export { createDirectExecutor } from './chunk-CUP2E4OH.mjs';
|
|
2
2
|
import './chunk-XYDP7CCJ.mjs';
|
|
3
3
|
import './chunk-OLXM65YH.mjs';
|
|
4
4
|
import './chunk-ZCKPRCYI.mjs';
|
|
5
5
|
import './chunk-TGP5M5R7.mjs';
|
|
6
|
-
import './chunk-
|
|
6
|
+
import './chunk-RZXNWQSM.mjs';
|
|
7
7
|
import './chunk-ZJ4CSAB2.mjs';
|
|
8
8
|
import './chunk-B3UOQJD3.mjs';
|
|
9
9
|
import './chunk-SREPGC6O.mjs';
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
2
|
import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-GNYXCLJS.mjs';
|
|
3
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf, encodeIndexConfiguration, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-
|
|
3
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf, encodeIndexConfiguration, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-7JZZ3HNV.mjs';
|
|
4
4
|
import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-4D6OH4EF.mjs';
|
|
5
5
|
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
|
|
6
6
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, providedDocId } from './chunk-2453RJYK.mjs';
|
|
7
|
-
import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-
|
|
7
|
+
import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-CUP2E4OH.mjs';
|
|
8
8
|
import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-XYDP7CCJ.mjs';
|
|
9
9
|
import { getLanguage } from './chunk-OLXM65YH.mjs';
|
|
10
10
|
import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-ZCKPRCYI.mjs';
|
|
11
11
|
import './chunk-TGP5M5R7.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-RZXNWQSM.mjs';
|
|
13
13
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
14
14
|
import './chunk-B3UOQJD3.mjs';
|
|
15
15
|
import { isPatternTolerance } from './chunk-SREPGC6O.mjs';
|