@delali/narsil 0.3.1-next.82 → 0.3.1-next.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-IH3GKNDQ.mjs → chunk-7JZZ3HNV.mjs} +20 -10
- package/dist/{chunk-4CWHWOBI.mjs → chunk-CUP2E4OH.mjs} +166 -90
- package/dist/{chunk-IN5CH4P2.mjs → chunk-RZXNWQSM.mjs} +181 -92
- package/dist/{direct-executor-JVXDR5Y5.mjs → direct-executor-6HASZ36Y.mjs} +2 -2
- package/dist/distribution.mjs +3 -3
- package/dist/index.browser.mjs +401 -226
- package/dist/index.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +3 -3
- package/dist/vector/search-worker.d.ts +2 -0
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +1 -1
- package/package.json +7 -7
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
|
|
4
|
-
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION,
|
|
4
|
+
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-CUP2E4OH.mjs';
|
|
5
5
|
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
|
|
6
6
|
import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
|
|
7
7
|
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
|
|
8
|
-
import { releaseLocksHeldBy } from './chunk-
|
|
8
|
+
import { releaseLocksHeldBy } from './chunk-RZXNWQSM.mjs';
|
|
9
9
|
import { requireSearchBackendSettings } from './chunk-B3UOQJD3.mjs';
|
|
10
10
|
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-SREPGC6O.mjs';
|
|
11
11
|
import { getNestedValue, parseVectorDimension } from './chunk-4VBG6PEH.mjs';
|
|
@@ -3756,7 +3756,8 @@ async function readVectorFile(store, file, fieldPath) {
|
|
|
3756
3756
|
async function loadVectorField(store, ref, vectorIndex) {
|
|
3757
3757
|
const restore = vectorIndex.restoreCheckpoint({
|
|
3758
3758
|
liveVectors: liveVectorsOf(ref),
|
|
3759
|
-
holdsGraph: ref.graphParts.length > 0
|
|
3759
|
+
holdsGraph: ref.graphParts.length > 0,
|
|
3760
|
+
graph: await readGraphParts(store, ref)
|
|
3760
3761
|
});
|
|
3761
3762
|
for (const file of ref.files) {
|
|
3762
3763
|
const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
|
|
@@ -3771,7 +3772,7 @@ async function loadVectorField(store, ref, vectorIndex) {
|
|
|
3771
3772
|
}
|
|
3772
3773
|
});
|
|
3773
3774
|
}
|
|
3774
|
-
restore.finish(
|
|
3775
|
+
restore.finish();
|
|
3775
3776
|
}
|
|
3776
3777
|
var EMPTY_BIN = new Uint8Array(0);
|
|
3777
3778
|
var EMPTY_BIN_BYTES = 2;
|
|
@@ -3835,7 +3836,7 @@ async function writeField(input, fieldPath, plan, outcome) {
|
|
|
3835
3836
|
const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
|
|
3836
3837
|
const written = [];
|
|
3837
3838
|
for (let index = 0; index < plan.newFiles; index++) {
|
|
3838
|
-
const { payload, ordinals } = plan.readNewFile(index);
|
|
3839
|
+
const { payload, ordinals, dead } = plan.readNewFile(index);
|
|
3839
3840
|
const id = nextFileId;
|
|
3840
3841
|
nextFileId += 1;
|
|
3841
3842
|
const key = vectorFileKey(input.indexName, fieldPath, id);
|
|
@@ -3845,13 +3846,13 @@ async function writeField(input, fieldPath, plan, outcome) {
|
|
|
3845
3846
|
`vector file "${key}"`
|
|
3846
3847
|
);
|
|
3847
3848
|
await input.store.write(key, packed.chunks);
|
|
3848
|
-
const dead = deadBitsWhere(ordinals.length, (position) => ordinals[position] === NO_LIVE_VECTOR_AT_POSITION);
|
|
3849
3849
|
files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
|
|
3850
3850
|
written.push({ id, key, ordinals });
|
|
3851
3851
|
outcome.layouts.push({
|
|
3852
3852
|
fieldPath,
|
|
3853
3853
|
key,
|
|
3854
3854
|
docIds: payload.docIds,
|
|
3855
|
+
dead,
|
|
3855
3856
|
vectorsOffset: packed.byteLength - payload.vectors.byteLength
|
|
3856
3857
|
});
|
|
3857
3858
|
}
|
|
@@ -5722,7 +5723,12 @@ async function adoptVectorLayouts(store, vectorIndexes, layouts) {
|
|
|
5722
5723
|
const vecIndex = vectorIndexes.get(layout.fieldPath);
|
|
5723
5724
|
if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
|
|
5724
5725
|
const path = await pathOf.call(store, layout.key);
|
|
5725
|
-
await vecIndex.adoptDiskLayout({
|
|
5726
|
+
await vecIndex.adoptDiskLayout({
|
|
5727
|
+
path,
|
|
5728
|
+
docIds: layout.docIds,
|
|
5729
|
+
dead: layout.dead,
|
|
5730
|
+
vectorsOffset: layout.vectorsOffset
|
|
5731
|
+
});
|
|
5726
5732
|
}
|
|
5727
5733
|
}
|
|
5728
5734
|
async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
|
|
@@ -6677,7 +6683,11 @@ async function readGraph(store, ref) {
|
|
|
6677
6683
|
async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
|
|
6678
6684
|
let liveVectors = 0;
|
|
6679
6685
|
for (const file of ref.files) liveVectors += file.count - countDead(file.dead, file.count);
|
|
6680
|
-
const restore = vectorIndex.restoreCheckpoint({
|
|
6686
|
+
const restore = vectorIndex.restoreCheckpoint({
|
|
6687
|
+
liveVectors,
|
|
6688
|
+
holdsGraph: ref.graphKey !== null,
|
|
6689
|
+
graph: await readGraph(store, ref)
|
|
6690
|
+
});
|
|
6681
6691
|
for (const file of ref.files) {
|
|
6682
6692
|
const bytes = await store.read(file.key);
|
|
6683
6693
|
if (bytes === null) {
|
|
@@ -6710,7 +6720,7 @@ async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
|
|
|
6710
6720
|
}
|
|
6711
6721
|
});
|
|
6712
6722
|
}
|
|
6713
|
-
restore.finish(
|
|
6723
|
+
restore.finish();
|
|
6714
6724
|
}
|
|
6715
6725
|
async function loadEarlierLayout(store, indexName, manifest, deps) {
|
|
6716
6726
|
let highestPartitionId = manifest.partitions.length - 1;
|
|
@@ -13690,7 +13700,7 @@ function partitionVectorPlan(vectorIndex, docIds) {
|
|
|
13690
13700
|
codes: null,
|
|
13691
13701
|
vectors: vectorsToBytes(packed)
|
|
13692
13702
|
};
|
|
13693
|
-
files.push({ payload, ordinals: new Int32Array(slice.length) });
|
|
13703
|
+
files.push({ payload, ordinals: new Int32Array(slice.length), dead: null });
|
|
13694
13704
|
}
|
|
13695
13705
|
return {
|
|
13696
13706
|
unchanged: false,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-XYDP7CCJ.mjs';
|
|
2
2
|
import { getLanguage, detectRuntime } from './chunk-OLXM65YH.mjs';
|
|
3
3
|
import { MAX_DOC_ID_LENGTH } from './chunk-ZCKPRCYI.mjs';
|
|
4
|
-
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes,
|
|
4
|
+
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, graphWithoutNodes, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, ordinalFilterHas, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, topLayerInParts, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-RZXNWQSM.mjs';
|
|
5
5
|
import { noteFallback } from './chunk-B3UOQJD3.mjs';
|
|
6
6
|
import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-SREPGC6O.mjs';
|
|
7
7
|
import { GEO_UNIT_METRES } from './chunk-WQBCBKY3.mjs';
|
|
@@ -6066,6 +6066,9 @@ function emptyFieldBeforeRestore(state) {
|
|
|
6066
6066
|
state.savedFiles = [];
|
|
6067
6067
|
state.savedSignature = null;
|
|
6068
6068
|
}
|
|
6069
|
+
function releaseReplacedVectors(state) {
|
|
6070
|
+
for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
|
|
6071
|
+
}
|
|
6069
6072
|
function adoptGraph(state, graph) {
|
|
6070
6073
|
if (graph === null && state.hnsw !== null) {
|
|
6071
6074
|
state.hnsw.clear();
|
|
@@ -6601,9 +6604,7 @@ function captureCloneCodes(state) {
|
|
|
6601
6604
|
const records = new Uint8Array(slots * layout.slotStride);
|
|
6602
6605
|
const present = new Uint8Array(slots);
|
|
6603
6606
|
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
6604
|
-
const
|
|
6605
|
-
if (docId === void 0) continue;
|
|
6606
|
-
const record = quantizer.recordOf(docId);
|
|
6607
|
+
const record = quantizer.recordAtOrdinal(ordinal);
|
|
6607
6608
|
if (record === void 0) continue;
|
|
6608
6609
|
records.set(record, ordinal * layout.slotStride);
|
|
6609
6610
|
present[ordinal] = 1;
|
|
@@ -6740,7 +6741,7 @@ async function dispatcherFor(state, graph) {
|
|
|
6740
6741
|
workerCount: Math.max(1, host.workerCount)
|
|
6741
6742
|
};
|
|
6742
6743
|
}
|
|
6743
|
-
if (liveSize(state) < WORKER_COPY_MIN_VECTORS) return null;
|
|
6744
|
+
if (liveSize(state) < WORKER_COPY_MIN_VECTORS || !sharedMemoryAvailable()) return null;
|
|
6744
6745
|
const handle = await shareGraph(state, graph, graph === state.hnsw);
|
|
6745
6746
|
const pool = state.workerCopyPool;
|
|
6746
6747
|
if (handle === null || pool === null || state.sharedHandles.get(graph)?.mode !== "shared") return null;
|
|
@@ -6830,12 +6831,40 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
6830
6831
|
state.buffer.delete(docId);
|
|
6831
6832
|
count += 1;
|
|
6832
6833
|
if (count % BUILD_CHUNK_SIZE === 0) {
|
|
6834
|
+
noteWrite(state);
|
|
6833
6835
|
await yieldToEventLoop();
|
|
6834
6836
|
}
|
|
6835
6837
|
}
|
|
6838
|
+
if (count > 0) noteWrite(state);
|
|
6836
6839
|
return !state.disposed;
|
|
6837
6840
|
}
|
|
6838
6841
|
|
|
6842
|
+
// src/vector/vector-index/dead-bits.ts
|
|
6843
|
+
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
6844
|
+
function deadBytesFor(count) {
|
|
6845
|
+
return Math.ceil(count / 8);
|
|
6846
|
+
}
|
|
6847
|
+
function isDead(dead, position) {
|
|
6848
|
+
return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
|
|
6849
|
+
}
|
|
6850
|
+
function countDead(dead, count) {
|
|
6851
|
+
if (dead === null) return 0;
|
|
6852
|
+
let found = 0;
|
|
6853
|
+
for (let position = 0; position < count; position++) {
|
|
6854
|
+
if (isDead(dead, position)) found += 1;
|
|
6855
|
+
}
|
|
6856
|
+
return found;
|
|
6857
|
+
}
|
|
6858
|
+
function deadBitsWhere(count, isDeadAt) {
|
|
6859
|
+
let dead = null;
|
|
6860
|
+
for (let position = 0; position < count; position++) {
|
|
6861
|
+
if (!isDeadAt(position)) continue;
|
|
6862
|
+
dead ??= new Uint8Array(deadBytesFor(count));
|
|
6863
|
+
dead[position >> 3] |= 1 << (position & 7);
|
|
6864
|
+
}
|
|
6865
|
+
return dead;
|
|
6866
|
+
}
|
|
6867
|
+
|
|
6839
6868
|
// src/vector/vector-index/disk.ts
|
|
6840
6869
|
function readsFromDisk(state) {
|
|
6841
6870
|
return state.storage === "disk";
|
|
@@ -6846,6 +6875,7 @@ function holdsVectorsOnDisk(state) {
|
|
|
6846
6875
|
function* locationsOf(state, layout) {
|
|
6847
6876
|
const stride = state.dimension * 4;
|
|
6848
6877
|
for (let i = 0; i < layout.docIds.length; i++) {
|
|
6878
|
+
if (isDead(layout.dead, i)) continue;
|
|
6849
6879
|
yield [layout.docIds[i], { path: layout.path, offset: layout.vectorsOffset + i * stride }];
|
|
6850
6880
|
}
|
|
6851
6881
|
}
|
|
@@ -7213,32 +7243,6 @@ function graphPartsOfEarlierPayload(payload) {
|
|
|
7213
7243
|
};
|
|
7214
7244
|
}
|
|
7215
7245
|
|
|
7216
|
-
// src/vector/vector-index/dead-bits.ts
|
|
7217
|
-
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
7218
|
-
function deadBytesFor(count) {
|
|
7219
|
-
return Math.ceil(count / 8);
|
|
7220
|
-
}
|
|
7221
|
-
function isDead(dead, position) {
|
|
7222
|
-
return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
|
|
7223
|
-
}
|
|
7224
|
-
function countDead(dead, count) {
|
|
7225
|
-
if (dead === null) return 0;
|
|
7226
|
-
let found = 0;
|
|
7227
|
-
for (let position = 0; position < count; position++) {
|
|
7228
|
-
if (isDead(dead, position)) found += 1;
|
|
7229
|
-
}
|
|
7230
|
-
return found;
|
|
7231
|
-
}
|
|
7232
|
-
function deadBitsWhere(count, isDeadAt) {
|
|
7233
|
-
let dead = null;
|
|
7234
|
-
for (let position = 0; position < count; position++) {
|
|
7235
|
-
if (!isDeadAt(position)) continue;
|
|
7236
|
-
dead ??= new Uint8Array(deadBytesFor(count));
|
|
7237
|
-
dead[position >> 3] |= 1 << (position & 7);
|
|
7238
|
-
}
|
|
7239
|
-
return dead;
|
|
7240
|
-
}
|
|
7241
|
-
|
|
7242
7246
|
// src/vector/vector-index/persistence.ts
|
|
7243
7247
|
function recordFor(state, quantizer, docId) {
|
|
7244
7248
|
const record = quantizer.recordOf(docId);
|
|
@@ -7260,7 +7264,7 @@ function sameCentroid(state, planned) {
|
|
|
7260
7264
|
}
|
|
7261
7265
|
return true;
|
|
7262
7266
|
}
|
|
7263
|
-
function partCodes(state, docIds, centroid) {
|
|
7267
|
+
function partCodes(state, docIds, centroid, tombstonedAt) {
|
|
7264
7268
|
const quantizer = state.osq;
|
|
7265
7269
|
const layout = state.store.handles.codeLayout;
|
|
7266
7270
|
if (centroid === null || quantizer === null || layout === null) return null;
|
|
@@ -7273,7 +7277,8 @@ function partCodes(state, docIds, centroid) {
|
|
|
7273
7277
|
}
|
|
7274
7278
|
const records = new Uint8Array(docIds.length * layout.slotStride);
|
|
7275
7279
|
for (let i = 0; i < docIds.length; i++) {
|
|
7276
|
-
const
|
|
7280
|
+
const tombstoned = tombstonedAt.get(i);
|
|
7281
|
+
const record = tombstoned === void 0 ? recordFor(state, quantizer, docIds[i]) : quantizer.recordAtOrdinal(tombstoned);
|
|
7277
7282
|
if (record !== void 0) records.set(record, i * layout.slotStride);
|
|
7278
7283
|
}
|
|
7279
7284
|
return { bits: quantizer.bits, centroid, records };
|
|
@@ -7430,33 +7435,39 @@ function isLive(state, ordinal) {
|
|
|
7430
7435
|
const docId = state.store.docIdForOrdinal(ordinal);
|
|
7431
7436
|
return docId !== void 0 && !state.tombstones.has(docId);
|
|
7432
7437
|
}
|
|
7433
|
-
function
|
|
7434
|
-
|
|
7435
|
-
|
|
7436
|
-
|
|
7438
|
+
function isGraphTombstone(state, ordinal) {
|
|
7439
|
+
return ordinal !== NO_LIVE_VECTOR_AT_POSITION && state.hnsw !== null && state.hnsw.holdsTombstone(ordinal);
|
|
7440
|
+
}
|
|
7441
|
+
function isCarried(state, ordinal) {
|
|
7442
|
+
return isLive(state, ordinal) || isGraphTombstone(state, ordinal);
|
|
7443
|
+
}
|
|
7444
|
+
function carriedCountOf(state, file) {
|
|
7445
|
+
let carried = 0;
|
|
7446
|
+
for (let i = 0; i < file.ordinals.length; i++) if (isCarried(state, file.ordinals[i])) carried += 1;
|
|
7447
|
+
return carried;
|
|
7437
7448
|
}
|
|
7438
7449
|
function deadBitsOf(state, file) {
|
|
7439
7450
|
return deadBitsWhere(file.ordinals.length, (position) => !isLive(state, file.ordinals[position]));
|
|
7440
7451
|
}
|
|
7441
|
-
function filesWorthKeeping(state, listed, calibration,
|
|
7452
|
+
function filesWorthKeeping(state, listed, calibration, carriedCounts) {
|
|
7442
7453
|
const kept = [];
|
|
7443
7454
|
for (const file of listed) {
|
|
7444
|
-
const
|
|
7445
|
-
const
|
|
7446
|
-
if (file.calibration !== calibration ||
|
|
7455
|
+
const carried = carriedCountOf(state, file);
|
|
7456
|
+
const deadWithoutNodeShare = file.ordinals.length === 0 ? 1 : 1 - carried / file.ordinals.length;
|
|
7457
|
+
if (file.calibration !== calibration || deadWithoutNodeShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
|
|
7447
7458
|
kept.push(file);
|
|
7448
|
-
|
|
7459
|
+
carriedCounts.push(carried);
|
|
7449
7460
|
}
|
|
7450
7461
|
return kept;
|
|
7451
7462
|
}
|
|
7452
|
-
function absorbSmallerPartialFiles(kept,
|
|
7463
|
+
function absorbSmallerPartialFiles(kept, carriedCounts, vectorsToWrite, capacity) {
|
|
7453
7464
|
let pending = vectorsToWrite;
|
|
7454
7465
|
for (let index = kept.length - 1; index >= 0 && pending > 0; index--) {
|
|
7455
7466
|
if (kept[index].ordinals.length >= capacity) continue;
|
|
7456
|
-
if (
|
|
7457
|
-
pending +=
|
|
7467
|
+
if (carriedCounts[index] > pending) return;
|
|
7468
|
+
pending += carriedCounts[index];
|
|
7458
7469
|
kept.splice(index, 1);
|
|
7459
|
-
|
|
7470
|
+
carriedCounts.splice(index, 1);
|
|
7460
7471
|
}
|
|
7461
7472
|
}
|
|
7462
7473
|
function ordinalsOutsideOf(state, kept) {
|
|
@@ -7470,14 +7481,15 @@ function ordinalsOutsideOf(state, kept) {
|
|
|
7470
7481
|
}
|
|
7471
7482
|
const outside = [];
|
|
7472
7483
|
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
7473
|
-
if (saved[ordinal] === 0 &&
|
|
7484
|
+
if (saved[ordinal] === 0 && isCarried(state, ordinal)) outside.push(ordinal);
|
|
7474
7485
|
}
|
|
7475
7486
|
return outside;
|
|
7476
7487
|
}
|
|
7477
|
-
function countOutside(state,
|
|
7488
|
+
function countOutside(state, carriedCounts) {
|
|
7478
7489
|
let held = 0;
|
|
7479
|
-
for (const
|
|
7480
|
-
|
|
7490
|
+
for (const carried of carriedCounts) held += carried;
|
|
7491
|
+
const tombstonedNodes = state.hnsw?.tombstoneCount ?? 0;
|
|
7492
|
+
return Math.max(0, state.store.size - state.tombstones.size + tombstonedNodes - held);
|
|
7481
7493
|
}
|
|
7482
7494
|
function numberEveryVector(state, kept, written) {
|
|
7483
7495
|
let total = written.length;
|
|
@@ -7488,7 +7500,7 @@ function numberEveryVector(state, kept, written) {
|
|
|
7488
7500
|
for (const file of kept) {
|
|
7489
7501
|
for (let i = 0; i < file.ordinals.length; i++, number++) {
|
|
7490
7502
|
const ordinal = file.ordinals[i];
|
|
7491
|
-
if (!
|
|
7503
|
+
if (!isCarried(state, ordinal)) continue;
|
|
7492
7504
|
numberOfOrdinal[ordinal] = number;
|
|
7493
7505
|
ordinalOfNumber[number] = ordinal;
|
|
7494
7506
|
}
|
|
@@ -7509,16 +7521,30 @@ function graphPartsOf(state, numbers) {
|
|
|
7509
7521
|
);
|
|
7510
7522
|
return parts.map((graph) => ({ v: VECTOR_GRAPH_PART_PAYLOAD_VERSION, fieldName: state.fieldName, graphs: [graph] }));
|
|
7511
7523
|
}
|
|
7512
|
-
function
|
|
7524
|
+
function graphPartsWithoutNodes(parts, numbers) {
|
|
7525
|
+
return parts.map((part, index) => ({
|
|
7526
|
+
...part,
|
|
7527
|
+
graphs: part.graphs.map((graph) => graphWithoutNodes(graph, index * VECTOR_GRAPH_PART_VECTORS, numbers))
|
|
7528
|
+
}));
|
|
7529
|
+
}
|
|
7530
|
+
function plannedVectorOf(state, ordinal) {
|
|
7531
|
+
const docId = state.store.docIdForOrdinal(ordinal) ?? state.store.view.docIdAt(ordinal) ?? "";
|
|
7532
|
+
return { ordinal, docId, tombstoned: !isLive(state, ordinal) };
|
|
7533
|
+
}
|
|
7534
|
+
function readFile(state, planned, centroid) {
|
|
7513
7535
|
const dimension = state.dimension;
|
|
7514
7536
|
const vectors = new Float32Array(planned.length * dimension);
|
|
7515
7537
|
const ordinals = new Int32Array(planned.length).fill(NO_LIVE_VECTOR_AT_POSITION);
|
|
7538
|
+
const tombstonedAt = /* @__PURE__ */ new Map();
|
|
7539
|
+
const docIds = planned.map((vector) => vector.docId);
|
|
7516
7540
|
for (let i = 0; i < planned.length; i++) {
|
|
7517
|
-
const
|
|
7541
|
+
const { ordinal, docId, tombstoned } = planned[i];
|
|
7542
|
+
const held = tombstoned || state.store.holdsOrdinal(ordinal) ? ordinal : state.store.getOrdinal(docId);
|
|
7518
7543
|
const entry = held === void 0 ? void 0 : state.store.entryForOrdinal(held);
|
|
7519
7544
|
if (held === void 0 || entry === void 0) continue;
|
|
7520
7545
|
vectors.set(entry.vector, i * dimension);
|
|
7521
7546
|
ordinals[i] = held;
|
|
7547
|
+
if (tombstoned) tombstonedAt.set(i, held);
|
|
7522
7548
|
}
|
|
7523
7549
|
return {
|
|
7524
7550
|
payload: {
|
|
@@ -7526,10 +7552,11 @@ function readFile(state, planned, docIds, centroid) {
|
|
|
7526
7552
|
fieldName: state.fieldName,
|
|
7527
7553
|
dimension,
|
|
7528
7554
|
docIds,
|
|
7529
|
-
codes: partCodes(state, docIds, centroid),
|
|
7555
|
+
codes: partCodes(state, docIds, centroid, tombstonedAt),
|
|
7530
7556
|
vectors: vectorsToBytes(vectors)
|
|
7531
7557
|
},
|
|
7532
|
-
ordinals
|
|
7558
|
+
ordinals,
|
|
7559
|
+
dead: deadBitsWhere(planned.length, (i) => planned[i].tombstoned || ordinals[i] === NO_LIVE_VECTOR_AT_POSITION)
|
|
7533
7560
|
};
|
|
7534
7561
|
}
|
|
7535
7562
|
function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
@@ -7537,14 +7564,20 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
|
7537
7564
|
const calibration = signature.calibration;
|
|
7538
7565
|
const trusted = registryMatches(state, listedKeys);
|
|
7539
7566
|
const capacity = fileCapacityOf(state);
|
|
7540
|
-
const
|
|
7541
|
-
const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration,
|
|
7542
|
-
absorbSmallerPartialFiles(keptFiles,
|
|
7567
|
+
const carriedCounts = [];
|
|
7568
|
+
const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, carriedCounts);
|
|
7569
|
+
absorbSmallerPartialFiles(keptFiles, carriedCounts, countOutside(state, carriedCounts), capacity);
|
|
7543
7570
|
const written = ordinalsOutsideOf(state, keptFiles);
|
|
7544
|
-
const
|
|
7571
|
+
const planned = written.map((ordinal) => plannedVectorOf(state, ordinal));
|
|
7545
7572
|
const centroid = codedCentroid(state);
|
|
7546
7573
|
const listedBefore = trusted && listedKeys !== null;
|
|
7547
7574
|
const unchanged = !rewritesGraph && listedBefore && sameSignature(state.savedSignature, signature) && written.length === 0;
|
|
7575
|
+
let firstWrittenNumber = 0;
|
|
7576
|
+
for (const file of keptFiles) firstWrittenNumber += file.ordinals.length;
|
|
7577
|
+
const graphParts = unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written));
|
|
7578
|
+
const vanished = /* @__PURE__ */ new Set();
|
|
7579
|
+
let vanishedWhenStripped = 0;
|
|
7580
|
+
let graphPartsToWrite = graphParts;
|
|
7548
7581
|
return {
|
|
7549
7582
|
unchanged,
|
|
7550
7583
|
keptFiles,
|
|
@@ -7555,12 +7588,22 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
|
7555
7588
|
dead: deadBitsOf(state, file)
|
|
7556
7589
|
})),
|
|
7557
7590
|
newFiles: Math.ceil(written.length / capacity),
|
|
7558
|
-
graphParts
|
|
7591
|
+
get graphParts() {
|
|
7592
|
+
if (graphParts !== null && vanished.size !== vanishedWhenStripped) {
|
|
7593
|
+
graphPartsToWrite = graphPartsWithoutNodes(graphParts, vanished);
|
|
7594
|
+
vanishedWhenStripped = vanished.size;
|
|
7595
|
+
}
|
|
7596
|
+
return graphPartsToWrite;
|
|
7597
|
+
},
|
|
7559
7598
|
signature,
|
|
7560
7599
|
readNewFile(index) {
|
|
7561
7600
|
const start = index * capacity;
|
|
7562
7601
|
const end = start + capacity;
|
|
7563
|
-
|
|
7602
|
+
const file = readFile(state, planned.slice(start, end), centroid);
|
|
7603
|
+
for (let i = 0; i < file.ordinals.length; i++) {
|
|
7604
|
+
if (file.ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) vanished.add(firstWrittenNumber + start + i);
|
|
7605
|
+
}
|
|
7606
|
+
return file;
|
|
7564
7607
|
}
|
|
7565
7608
|
};
|
|
7566
7609
|
}
|
|
@@ -7593,9 +7636,10 @@ function insertNodesOfLaterGraph(state, graph, later, ordinals) {
|
|
|
7593
7636
|
if (docId !== void 0 && !graph.has(docId)) graph.insertOrdinal(ordinal);
|
|
7594
7637
|
}
|
|
7595
7638
|
}
|
|
7596
|
-
function restoreGraph(state, payload, ordinalOfNumber) {
|
|
7639
|
+
function restoreGraph(state, payload, ordinalOfNumber, tombstoned) {
|
|
7597
7640
|
const first = payload?.graphs[0]?.parts[0];
|
|
7598
7641
|
if (payload === null || first === void 0) {
|
|
7642
|
+
releaseReplacedVectors(state);
|
|
7599
7643
|
bufferEveryVectorOutsideTheGraph(state);
|
|
7600
7644
|
return;
|
|
7601
7645
|
}
|
|
@@ -7606,11 +7650,13 @@ function restoreGraph(state, payload, ordinalOfNumber) {
|
|
|
7606
7650
|
state.osq ?? void 0
|
|
7607
7651
|
);
|
|
7608
7652
|
graph.deserializeNumbered(payload.graphs[0], ordinalOfNumber);
|
|
7653
|
+
for (const ordinal of tombstoned) graph.markTombstoneOrdinal(ordinal);
|
|
7609
7654
|
adoptGraph(state, graph);
|
|
7610
7655
|
for (const later of payload.graphs.slice(1)) insertNodesOfLaterGraph(state, graph, later, ordinalOfNumber);
|
|
7611
7656
|
bufferEveryVectorOutsideTheGraph(state);
|
|
7612
7657
|
if (state.buffer.size <= graph.size) return;
|
|
7613
7658
|
adoptGraph(state, null);
|
|
7659
|
+
releaseReplacedVectors(state);
|
|
7614
7660
|
state.buffer.clear();
|
|
7615
7661
|
bufferEveryVectorOutsideTheGraph(state);
|
|
7616
7662
|
}
|
|
@@ -7619,6 +7665,9 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7619
7665
|
const cold = readsFromDisk(state) && (shape.holdsGraph || shape.liveVectors >= state.promotionThreshold);
|
|
7620
7666
|
const restoresCodes = state.osq !== null && osqBitsOf(state.quantizationMode) !== null && shape.holdsGraph;
|
|
7621
7667
|
const files = [];
|
|
7668
|
+
const graphNodes = shape.graph?.graphs[0] ?? null;
|
|
7669
|
+
const tombstoned = /* @__PURE__ */ new Set();
|
|
7670
|
+
let nextNumber = 0;
|
|
7622
7671
|
let sharedCodes = null;
|
|
7623
7672
|
let codesAgree = true;
|
|
7624
7673
|
function restoreRecords(payload, ordinals) {
|
|
@@ -7640,9 +7689,14 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7640
7689
|
const stride = layout.slotStride;
|
|
7641
7690
|
for (let i = 0; i < ordinals.length; i++) {
|
|
7642
7691
|
if (ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) continue;
|
|
7643
|
-
|
|
7692
|
+
const record = codes.records.subarray(i * stride, (i + 1) * stride);
|
|
7693
|
+
if (tombstoned.has(ordinals[i])) quantizer.restoreRecordAtOrdinal(ordinals[i], record);
|
|
7694
|
+
else quantizer.restoreRecord(payload.docIds[i], record);
|
|
7644
7695
|
}
|
|
7645
7696
|
}
|
|
7697
|
+
function graphHoldsNodeFor(number) {
|
|
7698
|
+
return graphNodes !== null && topLayerInParts(graphNodes, number) >= 0;
|
|
7699
|
+
}
|
|
7646
7700
|
function addFile(file) {
|
|
7647
7701
|
const { payload, location } = file;
|
|
7648
7702
|
if (payload.dimension !== state.dimension) {
|
|
@@ -7657,13 +7711,21 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7657
7711
|
const vectors = bytesToVectors(payload.vectors);
|
|
7658
7712
|
const fileIndex = onDisk && cold ? state.store.addVectorFile(location.path) : null;
|
|
7659
7713
|
const ordinals = new Int32Array(payload.docIds.length).fill(NO_LIVE_VECTOR_AT_POSITION);
|
|
7714
|
+
const firstNumber = nextNumber;
|
|
7715
|
+
nextNumber += payload.docIds.length;
|
|
7660
7716
|
for (let i = 0; i < payload.docIds.length; i++) {
|
|
7661
|
-
|
|
7717
|
+
const dead = isDead(file.dead, i);
|
|
7718
|
+
if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
|
|
7662
7719
|
const docId = payload.docIds[i];
|
|
7663
|
-
const replaced = state.store.getOrdinal(docId);
|
|
7664
|
-
if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
|
|
7665
7720
|
const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
|
|
7666
7721
|
const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
|
|
7722
|
+
if (dead) {
|
|
7723
|
+
ordinals[i] = state.store.insertReplaced(docId, vector, fileIndex === null ? void 0 : { fileIndex, offset });
|
|
7724
|
+
tombstoned.add(ordinals[i]);
|
|
7725
|
+
continue;
|
|
7726
|
+
}
|
|
7727
|
+
const replaced = state.store.getOrdinal(docId);
|
|
7728
|
+
if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
|
|
7667
7729
|
if (fileIndex !== null) {
|
|
7668
7730
|
ordinals[i] = state.store.insertCold(docId, magnitude(vector), { fileIndex, offset });
|
|
7669
7731
|
continue;
|
|
@@ -7674,21 +7736,27 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7674
7736
|
restoreRecords(payload, ordinals);
|
|
7675
7737
|
files.push({ id: file.id, key: file.key, ordinals, coded: payload.codes !== null });
|
|
7676
7738
|
}
|
|
7677
|
-
function finish(
|
|
7739
|
+
function finish() {
|
|
7678
7740
|
const codesRestored = restoresCodes && codesAgree && sharedCodes !== null;
|
|
7679
|
-
if (restoresCodes && !codesRestored)
|
|
7741
|
+
if (restoresCodes && !codesRestored) {
|
|
7742
|
+
recalibrateFromStore(state);
|
|
7743
|
+
for (const ordinal of tombstoned) state.osq?.writeCodes(ordinal);
|
|
7744
|
+
}
|
|
7680
7745
|
let total = 0;
|
|
7681
7746
|
for (const file of files) total += file.ordinals.length;
|
|
7682
7747
|
const ordinalOfNumber = new Int32Array(total);
|
|
7683
7748
|
let number = 0;
|
|
7684
7749
|
for (const file of files) {
|
|
7685
7750
|
for (let i = 0; i < file.ordinals.length; i++) {
|
|
7686
|
-
|
|
7751
|
+
const ordinal = file.ordinals[i];
|
|
7752
|
+
if (!state.store.holdsOrdinal(ordinal) && !tombstoned.has(ordinal)) {
|
|
7753
|
+
file.ordinals[i] = NO_LIVE_VECTOR_AT_POSITION;
|
|
7754
|
+
}
|
|
7687
7755
|
}
|
|
7688
7756
|
ordinalOfNumber.set(file.ordinals, number);
|
|
7689
7757
|
number += file.ordinals.length;
|
|
7690
7758
|
}
|
|
7691
|
-
restoreGraph(state, graph, ordinalOfNumber);
|
|
7759
|
+
restoreGraph(state, shape.graph, ordinalOfNumber, tombstoned);
|
|
7692
7760
|
const current = signatureOf(state).calibration;
|
|
7693
7761
|
const codedUnder = codesRestored && current !== null ? current : CALIBRATION_NEVER_CURRENT;
|
|
7694
7762
|
const saved = files.map((file) => ({
|
|
@@ -7705,11 +7773,12 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7705
7773
|
|
|
7706
7774
|
// src/vector/vector-index/maintenance.ts
|
|
7707
7775
|
function compact(state) {
|
|
7708
|
-
if (state.tombstones.size === 0) return;
|
|
7776
|
+
if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
|
|
7709
7777
|
if (state.hnsw) {
|
|
7710
7778
|
state.compactedNodeCount += state.hnsw.tombstoneCount;
|
|
7711
7779
|
state.hnsw.compactTombstones();
|
|
7712
7780
|
}
|
|
7781
|
+
state.freshGraph?.compactTombstones();
|
|
7713
7782
|
for (const docId of state.tombstones) {
|
|
7714
7783
|
state.store.remove(docId);
|
|
7715
7784
|
state.buffer.delete(docId);
|
|
@@ -7717,6 +7786,7 @@ function compact(state) {
|
|
|
7717
7786
|
state.osq.remove(docId);
|
|
7718
7787
|
}
|
|
7719
7788
|
}
|
|
7789
|
+
releaseReplacedVectors(state);
|
|
7720
7790
|
state.tombstones.clear();
|
|
7721
7791
|
}
|
|
7722
7792
|
function recalibrateWhileNoThreadSearches(state, graph) {
|
|
@@ -7748,6 +7818,7 @@ async function foldIntoGraph(state) {
|
|
|
7748
7818
|
const graph = state.hnsw;
|
|
7749
7819
|
if (graph === null || rebuildNeeded) {
|
|
7750
7820
|
if (state.osq?.isCalibrated()) recalibrateWhileNoThreadSearches(state, graph);
|
|
7821
|
+
else calibrateQuantizer(state);
|
|
7751
7822
|
await buildGraphFromStore(state);
|
|
7752
7823
|
} else {
|
|
7753
7824
|
await insertMissing(state, graph);
|
|
@@ -7794,8 +7865,9 @@ function maintenanceStatus(state) {
|
|
|
7794
7865
|
const storeSize = state.store.size;
|
|
7795
7866
|
const graphCount = state.hnsw ? 1 : 0;
|
|
7796
7867
|
const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
|
|
7797
|
-
const
|
|
7798
|
-
const
|
|
7868
|
+
const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
|
|
7869
|
+
const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
|
|
7870
|
+
const estimatedCompactMs = Math.round(tombstoned * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
|
|
7799
7871
|
const optimizeVectorCount = state.hnsw === null || graphNeedsRebuild(state) ? storeSize : state.buffer.size;
|
|
7800
7872
|
const estimatedOptimizeMs = Math.round(optimizeVectorCount * ESTIMATED_MS_PER_VECTOR_REBUILD * state.dimensionScale);
|
|
7801
7873
|
return {
|
|
@@ -8056,19 +8128,19 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
8056
8128
|
return scannedOutcome(scan2);
|
|
8057
8129
|
}
|
|
8058
8130
|
}
|
|
8059
|
-
const
|
|
8060
|
-
|
|
8061
|
-
|
|
8062
|
-
|
|
8063
|
-
|
|
8064
|
-
|
|
8065
|
-
|
|
8066
|
-
|
|
8067
|
-
|
|
8068
|
-
|
|
8069
|
-
const
|
|
8070
|
-
const buffered = bruteForceSearch(state, query, k, metric, minSimilarity,
|
|
8071
|
-
return
|
|
8131
|
+
const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample }).map((r) => ({ docId: r.docId, score: r.score }));
|
|
8132
|
+
return fetchedOutcome(
|
|
8133
|
+
state,
|
|
8134
|
+
withBufferedVectors(state, query, k, options, filter, hnswResults),
|
|
8135
|
+
minSimilarity,
|
|
8136
|
+
filter
|
|
8137
|
+
);
|
|
8138
|
+
}
|
|
8139
|
+
function withBufferedVectors(state, query, k, options, filter, graphResults) {
|
|
8140
|
+
if (state.buffer.size === 0) return graphResults;
|
|
8141
|
+
const candidates = bufferCandidates(state, filter);
|
|
8142
|
+
const buffered = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
|
|
8143
|
+
return mergeResults(graphResults, buffered.results, k);
|
|
8072
8144
|
}
|
|
8073
8145
|
|
|
8074
8146
|
// src/vector/vector-index/index.ts
|
|
@@ -8169,11 +8241,12 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
8169
8241
|
}
|
|
8170
8242
|
noteWrite(state);
|
|
8171
8243
|
state.tombstones.delete(docId);
|
|
8172
|
-
state.
|
|
8244
|
+
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
8245
|
+
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
8173
8246
|
if (previous !== void 0) {
|
|
8174
8247
|
state.hnsw?.markTombstoneOrdinal(previous);
|
|
8175
8248
|
state.freshGraph?.markTombstoneOrdinal(previous);
|
|
8176
|
-
state.osq?.removeOrdinal(previous);
|
|
8249
|
+
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
8177
8250
|
}
|
|
8178
8251
|
state.buffer.add(docId);
|
|
8179
8252
|
state.pendingLocations.delete(docId);
|
|
@@ -8243,10 +8316,13 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
8243
8316
|
efSearch: options.efSearch,
|
|
8244
8317
|
oversample: options.oversample
|
|
8245
8318
|
});
|
|
8246
|
-
if (viaWorkerCopy !== null) return fetchedOutcome(state, viaWorkerCopy, options.minSimilarity, filter);
|
|
8247
8319
|
if (confined && state.revision !== filterRevision) {
|
|
8248
8320
|
filter = filterForOptions(state, options);
|
|
8249
8321
|
}
|
|
8322
|
+
if (viaWorkerCopy !== null) {
|
|
8323
|
+
const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
|
|
8324
|
+
return fetchedOutcome(state, results, options.minSimilarity, filter);
|
|
8325
|
+
}
|
|
8250
8326
|
return searchWithFilter(state, query, k, options, filter);
|
|
8251
8327
|
}
|
|
8252
8328
|
return {
|
|
@@ -10942,4 +11018,4 @@ function createDirectExecutor(options) {
|
|
|
10942
11018
|
};
|
|
10943
11019
|
}
|
|
10944
11020
|
|
|
10945
|
-
export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES,
|
|
11021
|
+
export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerStopWords, registerTokenizer, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
|