@delali/narsil 0.3.1-next.80 → 0.3.1-next.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-CcTDWh_P.d.ts → capabilities-CAdkp2P-.d.ts} +2 -2
- package/dist/{chunk-HJASKX4F.mjs → chunk-2453RJYK.mjs} +2 -2
- package/dist/{chunk-4QDQEJDV.mjs → chunk-4D6OH4EF.mjs} +6 -6
- package/dist/{chunk-P2PYEZZE.mjs → chunk-4VBG6PEH.mjs} +3 -3
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-5XNIRLFC.mjs} +2 -2
- package/dist/{chunk-NSXQCE4K.mjs → chunk-7JZZ3HNV.mjs} +208 -48
- package/dist/{chunk-2E35UM3G.mjs → chunk-B3UOQJD3.mjs} +1 -1
- package/dist/{chunk-K4FZ55Q2.mjs → chunk-CUP2E4OH.mjs} +175 -99
- package/dist/{chunk-MEOOYKWL.mjs → chunk-GNYXCLJS.mjs} +4 -4
- package/dist/{chunk-OSF3SGJV.mjs → chunk-KJ7LB3TR.mjs} +2 -2
- package/dist/{chunk-DQABW5UQ.mjs → chunk-LS5IRM23.mjs} +1 -1
- package/dist/{chunk-4MQ5F4IL.mjs → chunk-OLXM65YH.mjs} +1 -1
- package/dist/{chunk-23HJ5QXQ.mjs → chunk-RZXNWQSM.mjs} +183 -94
- package/dist/{chunk-R2XCJIO7.mjs → chunk-SREPGC6O.mjs} +2 -2
- package/dist/{chunk-I64LXKO3.mjs → chunk-TPJU7NDJ.mjs} +1 -0
- package/dist/{chunk-YGLSR7BC.mjs → chunk-XYDP7CCJ.mjs} +5 -5
- package/dist/{chunk-QPYV22XH.mjs → chunk-XZF4LZ3X.mjs} +4 -3
- package/dist/{chunk-GG2FBGVZ.mjs → chunk-ZCKPRCYI.mjs} +1 -1
- package/dist/{client-CsybN673.d.ts → client-CKUtkLz3.d.ts} +2 -2
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +1 -0
- package/dist/{config-CFonR_gY.d.ts → config-BuIkeM5C.d.ts} +39 -3
- package/dist/{config-BCNQtszf.d.ts → config-CSDEPraV.d.ts} +1 -1
- package/dist/direct-executor-6HASZ36Y.mjs +19 -0
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +756 -64
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-B0oIF1_r.d.ts → engine-DMNZJAUq.d.ts} +1 -1
- package/dist/{types-ChrVy4lH.d.ts → index-DzONZUKE.d.ts} +79 -37
- package/dist/index.browser.mjs +426 -249
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +14 -14
- package/dist/{memory-B6SBOKlS.d.ts → memory-BPKNx9Tl.d.ts} +1 -0
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/server.d.ts +9 -9
- package/dist/server.mjs +18 -13
- package/dist/{tasks-CYAQe6qd.d.ts → tasks-DGyz586z.d.ts} +1 -1
- package/dist/{thread-P6QJ2Q5O.mjs → thread-33M5AOEI.mjs} +10 -10
- package/dist/vector/search-worker.d.ts +2 -0
- package/dist/vector/search-worker.mjs +8 -7
- package/dist/workers/entry.mjs +5 -5
- package/package.json +11 -7
- package/dist/direct-executor-VNMQ6NIH.mjs +0 -19
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-
|
|
2
|
-
import { getLanguage, detectRuntime } from './chunk-
|
|
3
|
-
import { MAX_DOC_ID_LENGTH } from './chunk-
|
|
4
|
-
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes,
|
|
5
|
-
import { noteFallback } from './chunk-
|
|
6
|
-
import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-
|
|
1
|
+
import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-XYDP7CCJ.mjs';
|
|
2
|
+
import { getLanguage, detectRuntime } from './chunk-OLXM65YH.mjs';
|
|
3
|
+
import { MAX_DOC_ID_LENGTH } from './chunk-ZCKPRCYI.mjs';
|
|
4
|
+
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, graphWithoutNodes, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, ordinalFilterHas, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, topLayerInParts, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-RZXNWQSM.mjs';
|
|
5
|
+
import { noteFallback } from './chunk-B3UOQJD3.mjs';
|
|
6
|
+
import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-SREPGC6O.mjs';
|
|
7
7
|
import { GEO_UNIT_METRES } from './chunk-WQBCBKY3.mjs';
|
|
8
|
-
import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex, getFieldValueByInternalId, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-
|
|
9
|
-
import { schemaFieldsOf, partialFieldsOf, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-
|
|
8
|
+
import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex, getFieldValueByInternalId, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-4VBG6PEH.mjs';
|
|
9
|
+
import { schemaFieldsOf, partialFieldsOf, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-KJ7LB3TR.mjs';
|
|
10
10
|
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
11
11
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
12
|
-
import { isWordIndexedBase, parsedTypeOf, isListBase } from './chunk-
|
|
12
|
+
import { isWordIndexedBase, parsedTypeOf, isListBase } from './chunk-LS5IRM23.mjs';
|
|
13
13
|
import { toReducedSortValue, readSortField, defaultSortMode, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-R6A773WK.mjs';
|
|
14
14
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
15
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
15
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
16
16
|
import { totalmem } from 'os';
|
|
17
17
|
import { getHeapStatistics } from 'v8';
|
|
18
18
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -6066,6 +6066,9 @@ function emptyFieldBeforeRestore(state) {
|
|
|
6066
6066
|
state.savedFiles = [];
|
|
6067
6067
|
state.savedSignature = null;
|
|
6068
6068
|
}
|
|
6069
|
+
function releaseReplacedVectors(state) {
|
|
6070
|
+
for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
|
|
6071
|
+
}
|
|
6069
6072
|
function adoptGraph(state, graph) {
|
|
6070
6073
|
if (graph === null && state.hnsw !== null) {
|
|
6071
6074
|
state.hnsw.clear();
|
|
@@ -6601,9 +6604,7 @@ function captureCloneCodes(state) {
|
|
|
6601
6604
|
const records = new Uint8Array(slots * layout.slotStride);
|
|
6602
6605
|
const present = new Uint8Array(slots);
|
|
6603
6606
|
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
6604
|
-
const
|
|
6605
|
-
if (docId === void 0) continue;
|
|
6606
|
-
const record = quantizer.recordOf(docId);
|
|
6607
|
+
const record = quantizer.recordAtOrdinal(ordinal);
|
|
6607
6608
|
if (record === void 0) continue;
|
|
6608
6609
|
records.set(record, ordinal * layout.slotStride);
|
|
6609
6610
|
present[ordinal] = 1;
|
|
@@ -6740,7 +6741,7 @@ async function dispatcherFor(state, graph) {
|
|
|
6740
6741
|
workerCount: Math.max(1, host.workerCount)
|
|
6741
6742
|
};
|
|
6742
6743
|
}
|
|
6743
|
-
if (liveSize(state) < WORKER_COPY_MIN_VECTORS) return null;
|
|
6744
|
+
if (liveSize(state) < WORKER_COPY_MIN_VECTORS || !sharedMemoryAvailable()) return null;
|
|
6744
6745
|
const handle = await shareGraph(state, graph, graph === state.hnsw);
|
|
6745
6746
|
const pool = state.workerCopyPool;
|
|
6746
6747
|
if (handle === null || pool === null || state.sharedHandles.get(graph)?.mode !== "shared") return null;
|
|
@@ -6830,12 +6831,40 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
6830
6831
|
state.buffer.delete(docId);
|
|
6831
6832
|
count += 1;
|
|
6832
6833
|
if (count % BUILD_CHUNK_SIZE === 0) {
|
|
6834
|
+
noteWrite(state);
|
|
6833
6835
|
await yieldToEventLoop();
|
|
6834
6836
|
}
|
|
6835
6837
|
}
|
|
6838
|
+
if (count > 0) noteWrite(state);
|
|
6836
6839
|
return !state.disposed;
|
|
6837
6840
|
}
|
|
6838
6841
|
|
|
6842
|
+
// src/vector/vector-index/dead-bits.ts
|
|
6843
|
+
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
6844
|
+
function deadBytesFor(count) {
|
|
6845
|
+
return Math.ceil(count / 8);
|
|
6846
|
+
}
|
|
6847
|
+
function isDead(dead, position) {
|
|
6848
|
+
return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
|
|
6849
|
+
}
|
|
6850
|
+
function countDead(dead, count) {
|
|
6851
|
+
if (dead === null) return 0;
|
|
6852
|
+
let found = 0;
|
|
6853
|
+
for (let position = 0; position < count; position++) {
|
|
6854
|
+
if (isDead(dead, position)) found += 1;
|
|
6855
|
+
}
|
|
6856
|
+
return found;
|
|
6857
|
+
}
|
|
6858
|
+
function deadBitsWhere(count, isDeadAt) {
|
|
6859
|
+
let dead = null;
|
|
6860
|
+
for (let position = 0; position < count; position++) {
|
|
6861
|
+
if (!isDeadAt(position)) continue;
|
|
6862
|
+
dead ??= new Uint8Array(deadBytesFor(count));
|
|
6863
|
+
dead[position >> 3] |= 1 << (position & 7);
|
|
6864
|
+
}
|
|
6865
|
+
return dead;
|
|
6866
|
+
}
|
|
6867
|
+
|
|
6839
6868
|
// src/vector/vector-index/disk.ts
|
|
6840
6869
|
function readsFromDisk(state) {
|
|
6841
6870
|
return state.storage === "disk";
|
|
@@ -6846,6 +6875,7 @@ function holdsVectorsOnDisk(state) {
|
|
|
6846
6875
|
function* locationsOf(state, layout) {
|
|
6847
6876
|
const stride = state.dimension * 4;
|
|
6848
6877
|
for (let i = 0; i < layout.docIds.length; i++) {
|
|
6878
|
+
if (isDead(layout.dead, i)) continue;
|
|
6849
6879
|
yield [layout.docIds[i], { path: layout.path, offset: layout.vectorsOffset + i * stride }];
|
|
6850
6880
|
}
|
|
6851
6881
|
}
|
|
@@ -7213,32 +7243,6 @@ function graphPartsOfEarlierPayload(payload) {
|
|
|
7213
7243
|
};
|
|
7214
7244
|
}
|
|
7215
7245
|
|
|
7216
|
-
// src/vector/vector-index/dead-bits.ts
|
|
7217
|
-
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
7218
|
-
function deadBytesFor(count) {
|
|
7219
|
-
return Math.ceil(count / 8);
|
|
7220
|
-
}
|
|
7221
|
-
function isDead(dead, position) {
|
|
7222
|
-
return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
|
|
7223
|
-
}
|
|
7224
|
-
function countDead(dead, count) {
|
|
7225
|
-
if (dead === null) return 0;
|
|
7226
|
-
let found = 0;
|
|
7227
|
-
for (let position = 0; position < count; position++) {
|
|
7228
|
-
if (isDead(dead, position)) found += 1;
|
|
7229
|
-
}
|
|
7230
|
-
return found;
|
|
7231
|
-
}
|
|
7232
|
-
function deadBitsWhere(count, isDeadAt) {
|
|
7233
|
-
let dead = null;
|
|
7234
|
-
for (let position = 0; position < count; position++) {
|
|
7235
|
-
if (!isDeadAt(position)) continue;
|
|
7236
|
-
dead ??= new Uint8Array(deadBytesFor(count));
|
|
7237
|
-
dead[position >> 3] |= 1 << (position & 7);
|
|
7238
|
-
}
|
|
7239
|
-
return dead;
|
|
7240
|
-
}
|
|
7241
|
-
|
|
7242
7246
|
// src/vector/vector-index/persistence.ts
|
|
7243
7247
|
function recordFor(state, quantizer, docId) {
|
|
7244
7248
|
const record = quantizer.recordOf(docId);
|
|
@@ -7260,7 +7264,7 @@ function sameCentroid(state, planned) {
|
|
|
7260
7264
|
}
|
|
7261
7265
|
return true;
|
|
7262
7266
|
}
|
|
7263
|
-
function partCodes(state, docIds, centroid) {
|
|
7267
|
+
function partCodes(state, docIds, centroid, tombstonedAt) {
|
|
7264
7268
|
const quantizer = state.osq;
|
|
7265
7269
|
const layout = state.store.handles.codeLayout;
|
|
7266
7270
|
if (centroid === null || quantizer === null || layout === null) return null;
|
|
@@ -7273,7 +7277,8 @@ function partCodes(state, docIds, centroid) {
|
|
|
7273
7277
|
}
|
|
7274
7278
|
const records = new Uint8Array(docIds.length * layout.slotStride);
|
|
7275
7279
|
for (let i = 0; i < docIds.length; i++) {
|
|
7276
|
-
const
|
|
7280
|
+
const tombstoned = tombstonedAt.get(i);
|
|
7281
|
+
const record = tombstoned === void 0 ? recordFor(state, quantizer, docIds[i]) : quantizer.recordAtOrdinal(tombstoned);
|
|
7277
7282
|
if (record !== void 0) records.set(record, i * layout.slotStride);
|
|
7278
7283
|
}
|
|
7279
7284
|
return { bits: quantizer.bits, centroid, records };
|
|
@@ -7430,33 +7435,39 @@ function isLive(state, ordinal) {
|
|
|
7430
7435
|
const docId = state.store.docIdForOrdinal(ordinal);
|
|
7431
7436
|
return docId !== void 0 && !state.tombstones.has(docId);
|
|
7432
7437
|
}
|
|
7433
|
-
function
|
|
7434
|
-
|
|
7435
|
-
|
|
7436
|
-
|
|
7438
|
+
function isGraphTombstone(state, ordinal) {
|
|
7439
|
+
return ordinal !== NO_LIVE_VECTOR_AT_POSITION && state.hnsw !== null && state.hnsw.holdsTombstone(ordinal);
|
|
7440
|
+
}
|
|
7441
|
+
function isCarried(state, ordinal) {
|
|
7442
|
+
return isLive(state, ordinal) || isGraphTombstone(state, ordinal);
|
|
7443
|
+
}
|
|
7444
|
+
function carriedCountOf(state, file) {
|
|
7445
|
+
let carried = 0;
|
|
7446
|
+
for (let i = 0; i < file.ordinals.length; i++) if (isCarried(state, file.ordinals[i])) carried += 1;
|
|
7447
|
+
return carried;
|
|
7437
7448
|
}
|
|
7438
7449
|
function deadBitsOf(state, file) {
|
|
7439
7450
|
return deadBitsWhere(file.ordinals.length, (position) => !isLive(state, file.ordinals[position]));
|
|
7440
7451
|
}
|
|
7441
|
-
function filesWorthKeeping(state, listed, calibration,
|
|
7452
|
+
function filesWorthKeeping(state, listed, calibration, carriedCounts) {
|
|
7442
7453
|
const kept = [];
|
|
7443
7454
|
for (const file of listed) {
|
|
7444
|
-
const
|
|
7445
|
-
const
|
|
7446
|
-
if (file.calibration !== calibration ||
|
|
7455
|
+
const carried = carriedCountOf(state, file);
|
|
7456
|
+
const deadWithoutNodeShare = file.ordinals.length === 0 ? 1 : 1 - carried / file.ordinals.length;
|
|
7457
|
+
if (file.calibration !== calibration || deadWithoutNodeShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
|
|
7447
7458
|
kept.push(file);
|
|
7448
|
-
|
|
7459
|
+
carriedCounts.push(carried);
|
|
7449
7460
|
}
|
|
7450
7461
|
return kept;
|
|
7451
7462
|
}
|
|
7452
|
-
function absorbSmallerPartialFiles(kept,
|
|
7463
|
+
function absorbSmallerPartialFiles(kept, carriedCounts, vectorsToWrite, capacity) {
|
|
7453
7464
|
let pending = vectorsToWrite;
|
|
7454
7465
|
for (let index = kept.length - 1; index >= 0 && pending > 0; index--) {
|
|
7455
7466
|
if (kept[index].ordinals.length >= capacity) continue;
|
|
7456
|
-
if (
|
|
7457
|
-
pending +=
|
|
7467
|
+
if (carriedCounts[index] > pending) return;
|
|
7468
|
+
pending += carriedCounts[index];
|
|
7458
7469
|
kept.splice(index, 1);
|
|
7459
|
-
|
|
7470
|
+
carriedCounts.splice(index, 1);
|
|
7460
7471
|
}
|
|
7461
7472
|
}
|
|
7462
7473
|
function ordinalsOutsideOf(state, kept) {
|
|
@@ -7470,14 +7481,15 @@ function ordinalsOutsideOf(state, kept) {
|
|
|
7470
7481
|
}
|
|
7471
7482
|
const outside = [];
|
|
7472
7483
|
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
7473
|
-
if (saved[ordinal] === 0 &&
|
|
7484
|
+
if (saved[ordinal] === 0 && isCarried(state, ordinal)) outside.push(ordinal);
|
|
7474
7485
|
}
|
|
7475
7486
|
return outside;
|
|
7476
7487
|
}
|
|
7477
|
-
function countOutside(state,
|
|
7488
|
+
function countOutside(state, carriedCounts) {
|
|
7478
7489
|
let held = 0;
|
|
7479
|
-
for (const
|
|
7480
|
-
|
|
7490
|
+
for (const carried of carriedCounts) held += carried;
|
|
7491
|
+
const tombstonedNodes = state.hnsw?.tombstoneCount ?? 0;
|
|
7492
|
+
return Math.max(0, state.store.size - state.tombstones.size + tombstonedNodes - held);
|
|
7481
7493
|
}
|
|
7482
7494
|
function numberEveryVector(state, kept, written) {
|
|
7483
7495
|
let total = written.length;
|
|
@@ -7488,7 +7500,7 @@ function numberEveryVector(state, kept, written) {
|
|
|
7488
7500
|
for (const file of kept) {
|
|
7489
7501
|
for (let i = 0; i < file.ordinals.length; i++, number++) {
|
|
7490
7502
|
const ordinal = file.ordinals[i];
|
|
7491
|
-
if (!
|
|
7503
|
+
if (!isCarried(state, ordinal)) continue;
|
|
7492
7504
|
numberOfOrdinal[ordinal] = number;
|
|
7493
7505
|
ordinalOfNumber[number] = ordinal;
|
|
7494
7506
|
}
|
|
@@ -7509,16 +7521,30 @@ function graphPartsOf(state, numbers) {
|
|
|
7509
7521
|
);
|
|
7510
7522
|
return parts.map((graph) => ({ v: VECTOR_GRAPH_PART_PAYLOAD_VERSION, fieldName: state.fieldName, graphs: [graph] }));
|
|
7511
7523
|
}
|
|
7512
|
-
function
|
|
7524
|
+
function graphPartsWithoutNodes(parts, numbers) {
|
|
7525
|
+
return parts.map((part, index) => ({
|
|
7526
|
+
...part,
|
|
7527
|
+
graphs: part.graphs.map((graph) => graphWithoutNodes(graph, index * VECTOR_GRAPH_PART_VECTORS, numbers))
|
|
7528
|
+
}));
|
|
7529
|
+
}
|
|
7530
|
+
function plannedVectorOf(state, ordinal) {
|
|
7531
|
+
const docId = state.store.docIdForOrdinal(ordinal) ?? state.store.view.docIdAt(ordinal) ?? "";
|
|
7532
|
+
return { ordinal, docId, tombstoned: !isLive(state, ordinal) };
|
|
7533
|
+
}
|
|
7534
|
+
function readFile(state, planned, centroid) {
|
|
7513
7535
|
const dimension = state.dimension;
|
|
7514
7536
|
const vectors = new Float32Array(planned.length * dimension);
|
|
7515
7537
|
const ordinals = new Int32Array(planned.length).fill(NO_LIVE_VECTOR_AT_POSITION);
|
|
7538
|
+
const tombstonedAt = /* @__PURE__ */ new Map();
|
|
7539
|
+
const docIds = planned.map((vector) => vector.docId);
|
|
7516
7540
|
for (let i = 0; i < planned.length; i++) {
|
|
7517
|
-
const
|
|
7541
|
+
const { ordinal, docId, tombstoned } = planned[i];
|
|
7542
|
+
const held = tombstoned || state.store.holdsOrdinal(ordinal) ? ordinal : state.store.getOrdinal(docId);
|
|
7518
7543
|
const entry = held === void 0 ? void 0 : state.store.entryForOrdinal(held);
|
|
7519
7544
|
if (held === void 0 || entry === void 0) continue;
|
|
7520
7545
|
vectors.set(entry.vector, i * dimension);
|
|
7521
7546
|
ordinals[i] = held;
|
|
7547
|
+
if (tombstoned) tombstonedAt.set(i, held);
|
|
7522
7548
|
}
|
|
7523
7549
|
return {
|
|
7524
7550
|
payload: {
|
|
@@ -7526,10 +7552,11 @@ function readFile(state, planned, docIds, centroid) {
|
|
|
7526
7552
|
fieldName: state.fieldName,
|
|
7527
7553
|
dimension,
|
|
7528
7554
|
docIds,
|
|
7529
|
-
codes: partCodes(state, docIds, centroid),
|
|
7555
|
+
codes: partCodes(state, docIds, centroid, tombstonedAt),
|
|
7530
7556
|
vectors: vectorsToBytes(vectors)
|
|
7531
7557
|
},
|
|
7532
|
-
ordinals
|
|
7558
|
+
ordinals,
|
|
7559
|
+
dead: deadBitsWhere(planned.length, (i) => planned[i].tombstoned || ordinals[i] === NO_LIVE_VECTOR_AT_POSITION)
|
|
7533
7560
|
};
|
|
7534
7561
|
}
|
|
7535
7562
|
function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
@@ -7537,14 +7564,20 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
|
7537
7564
|
const calibration = signature.calibration;
|
|
7538
7565
|
const trusted = registryMatches(state, listedKeys);
|
|
7539
7566
|
const capacity = fileCapacityOf(state);
|
|
7540
|
-
const
|
|
7541
|
-
const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration,
|
|
7542
|
-
absorbSmallerPartialFiles(keptFiles,
|
|
7567
|
+
const carriedCounts = [];
|
|
7568
|
+
const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, carriedCounts);
|
|
7569
|
+
absorbSmallerPartialFiles(keptFiles, carriedCounts, countOutside(state, carriedCounts), capacity);
|
|
7543
7570
|
const written = ordinalsOutsideOf(state, keptFiles);
|
|
7544
|
-
const
|
|
7571
|
+
const planned = written.map((ordinal) => plannedVectorOf(state, ordinal));
|
|
7545
7572
|
const centroid = codedCentroid(state);
|
|
7546
7573
|
const listedBefore = trusted && listedKeys !== null;
|
|
7547
7574
|
const unchanged = !rewritesGraph && listedBefore && sameSignature(state.savedSignature, signature) && written.length === 0;
|
|
7575
|
+
let firstWrittenNumber = 0;
|
|
7576
|
+
for (const file of keptFiles) firstWrittenNumber += file.ordinals.length;
|
|
7577
|
+
const graphParts = unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written));
|
|
7578
|
+
const vanished = /* @__PURE__ */ new Set();
|
|
7579
|
+
let vanishedWhenStripped = 0;
|
|
7580
|
+
let graphPartsToWrite = graphParts;
|
|
7548
7581
|
return {
|
|
7549
7582
|
unchanged,
|
|
7550
7583
|
keptFiles,
|
|
@@ -7555,12 +7588,22 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
|
7555
7588
|
dead: deadBitsOf(state, file)
|
|
7556
7589
|
})),
|
|
7557
7590
|
newFiles: Math.ceil(written.length / capacity),
|
|
7558
|
-
graphParts
|
|
7591
|
+
get graphParts() {
|
|
7592
|
+
if (graphParts !== null && vanished.size !== vanishedWhenStripped) {
|
|
7593
|
+
graphPartsToWrite = graphPartsWithoutNodes(graphParts, vanished);
|
|
7594
|
+
vanishedWhenStripped = vanished.size;
|
|
7595
|
+
}
|
|
7596
|
+
return graphPartsToWrite;
|
|
7597
|
+
},
|
|
7559
7598
|
signature,
|
|
7560
7599
|
readNewFile(index) {
|
|
7561
7600
|
const start = index * capacity;
|
|
7562
7601
|
const end = start + capacity;
|
|
7563
|
-
|
|
7602
|
+
const file = readFile(state, planned.slice(start, end), centroid);
|
|
7603
|
+
for (let i = 0; i < file.ordinals.length; i++) {
|
|
7604
|
+
if (file.ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) vanished.add(firstWrittenNumber + start + i);
|
|
7605
|
+
}
|
|
7606
|
+
return file;
|
|
7564
7607
|
}
|
|
7565
7608
|
};
|
|
7566
7609
|
}
|
|
@@ -7593,9 +7636,10 @@ function insertNodesOfLaterGraph(state, graph, later, ordinals) {
|
|
|
7593
7636
|
if (docId !== void 0 && !graph.has(docId)) graph.insertOrdinal(ordinal);
|
|
7594
7637
|
}
|
|
7595
7638
|
}
|
|
7596
|
-
function restoreGraph(state, payload, ordinalOfNumber) {
|
|
7639
|
+
function restoreGraph(state, payload, ordinalOfNumber, tombstoned) {
|
|
7597
7640
|
const first = payload?.graphs[0]?.parts[0];
|
|
7598
7641
|
if (payload === null || first === void 0) {
|
|
7642
|
+
releaseReplacedVectors(state);
|
|
7599
7643
|
bufferEveryVectorOutsideTheGraph(state);
|
|
7600
7644
|
return;
|
|
7601
7645
|
}
|
|
@@ -7606,11 +7650,13 @@ function restoreGraph(state, payload, ordinalOfNumber) {
|
|
|
7606
7650
|
state.osq ?? void 0
|
|
7607
7651
|
);
|
|
7608
7652
|
graph.deserializeNumbered(payload.graphs[0], ordinalOfNumber);
|
|
7653
|
+
for (const ordinal of tombstoned) graph.markTombstoneOrdinal(ordinal);
|
|
7609
7654
|
adoptGraph(state, graph);
|
|
7610
7655
|
for (const later of payload.graphs.slice(1)) insertNodesOfLaterGraph(state, graph, later, ordinalOfNumber);
|
|
7611
7656
|
bufferEveryVectorOutsideTheGraph(state);
|
|
7612
7657
|
if (state.buffer.size <= graph.size) return;
|
|
7613
7658
|
adoptGraph(state, null);
|
|
7659
|
+
releaseReplacedVectors(state);
|
|
7614
7660
|
state.buffer.clear();
|
|
7615
7661
|
bufferEveryVectorOutsideTheGraph(state);
|
|
7616
7662
|
}
|
|
@@ -7619,6 +7665,9 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7619
7665
|
const cold = readsFromDisk(state) && (shape.holdsGraph || shape.liveVectors >= state.promotionThreshold);
|
|
7620
7666
|
const restoresCodes = state.osq !== null && osqBitsOf(state.quantizationMode) !== null && shape.holdsGraph;
|
|
7621
7667
|
const files = [];
|
|
7668
|
+
const graphNodes = shape.graph?.graphs[0] ?? null;
|
|
7669
|
+
const tombstoned = /* @__PURE__ */ new Set();
|
|
7670
|
+
let nextNumber = 0;
|
|
7622
7671
|
let sharedCodes = null;
|
|
7623
7672
|
let codesAgree = true;
|
|
7624
7673
|
function restoreRecords(payload, ordinals) {
|
|
@@ -7640,9 +7689,14 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7640
7689
|
const stride = layout.slotStride;
|
|
7641
7690
|
for (let i = 0; i < ordinals.length; i++) {
|
|
7642
7691
|
if (ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) continue;
|
|
7643
|
-
|
|
7692
|
+
const record = codes.records.subarray(i * stride, (i + 1) * stride);
|
|
7693
|
+
if (tombstoned.has(ordinals[i])) quantizer.restoreRecordAtOrdinal(ordinals[i], record);
|
|
7694
|
+
else quantizer.restoreRecord(payload.docIds[i], record);
|
|
7644
7695
|
}
|
|
7645
7696
|
}
|
|
7697
|
+
function graphHoldsNodeFor(number) {
|
|
7698
|
+
return graphNodes !== null && topLayerInParts(graphNodes, number) >= 0;
|
|
7699
|
+
}
|
|
7646
7700
|
function addFile(file) {
|
|
7647
7701
|
const { payload, location } = file;
|
|
7648
7702
|
if (payload.dimension !== state.dimension) {
|
|
@@ -7657,13 +7711,21 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7657
7711
|
const vectors = bytesToVectors(payload.vectors);
|
|
7658
7712
|
const fileIndex = onDisk && cold ? state.store.addVectorFile(location.path) : null;
|
|
7659
7713
|
const ordinals = new Int32Array(payload.docIds.length).fill(NO_LIVE_VECTOR_AT_POSITION);
|
|
7714
|
+
const firstNumber = nextNumber;
|
|
7715
|
+
nextNumber += payload.docIds.length;
|
|
7660
7716
|
for (let i = 0; i < payload.docIds.length; i++) {
|
|
7661
|
-
|
|
7717
|
+
const dead = isDead(file.dead, i);
|
|
7718
|
+
if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
|
|
7662
7719
|
const docId = payload.docIds[i];
|
|
7663
|
-
const replaced = state.store.getOrdinal(docId);
|
|
7664
|
-
if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
|
|
7665
7720
|
const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
|
|
7666
7721
|
const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
|
|
7722
|
+
if (dead) {
|
|
7723
|
+
ordinals[i] = state.store.insertReplaced(docId, vector, fileIndex === null ? void 0 : { fileIndex, offset });
|
|
7724
|
+
tombstoned.add(ordinals[i]);
|
|
7725
|
+
continue;
|
|
7726
|
+
}
|
|
7727
|
+
const replaced = state.store.getOrdinal(docId);
|
|
7728
|
+
if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
|
|
7667
7729
|
if (fileIndex !== null) {
|
|
7668
7730
|
ordinals[i] = state.store.insertCold(docId, magnitude(vector), { fileIndex, offset });
|
|
7669
7731
|
continue;
|
|
@@ -7674,21 +7736,27 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7674
7736
|
restoreRecords(payload, ordinals);
|
|
7675
7737
|
files.push({ id: file.id, key: file.key, ordinals, coded: payload.codes !== null });
|
|
7676
7738
|
}
|
|
7677
|
-
function finish(
|
|
7739
|
+
function finish() {
|
|
7678
7740
|
const codesRestored = restoresCodes && codesAgree && sharedCodes !== null;
|
|
7679
|
-
if (restoresCodes && !codesRestored)
|
|
7741
|
+
if (restoresCodes && !codesRestored) {
|
|
7742
|
+
recalibrateFromStore(state);
|
|
7743
|
+
for (const ordinal of tombstoned) state.osq?.writeCodes(ordinal);
|
|
7744
|
+
}
|
|
7680
7745
|
let total = 0;
|
|
7681
7746
|
for (const file of files) total += file.ordinals.length;
|
|
7682
7747
|
const ordinalOfNumber = new Int32Array(total);
|
|
7683
7748
|
let number = 0;
|
|
7684
7749
|
for (const file of files) {
|
|
7685
7750
|
for (let i = 0; i < file.ordinals.length; i++) {
|
|
7686
|
-
|
|
7751
|
+
const ordinal = file.ordinals[i];
|
|
7752
|
+
if (!state.store.holdsOrdinal(ordinal) && !tombstoned.has(ordinal)) {
|
|
7753
|
+
file.ordinals[i] = NO_LIVE_VECTOR_AT_POSITION;
|
|
7754
|
+
}
|
|
7687
7755
|
}
|
|
7688
7756
|
ordinalOfNumber.set(file.ordinals, number);
|
|
7689
7757
|
number += file.ordinals.length;
|
|
7690
7758
|
}
|
|
7691
|
-
restoreGraph(state, graph, ordinalOfNumber);
|
|
7759
|
+
restoreGraph(state, shape.graph, ordinalOfNumber, tombstoned);
|
|
7692
7760
|
const current = signatureOf(state).calibration;
|
|
7693
7761
|
const codedUnder = codesRestored && current !== null ? current : CALIBRATION_NEVER_CURRENT;
|
|
7694
7762
|
const saved = files.map((file) => ({
|
|
@@ -7705,11 +7773,12 @@ function beginCheckpointRestore(state, shape) {
|
|
|
7705
7773
|
|
|
7706
7774
|
// src/vector/vector-index/maintenance.ts
|
|
7707
7775
|
function compact(state) {
|
|
7708
|
-
if (state.tombstones.size === 0) return;
|
|
7776
|
+
if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
|
|
7709
7777
|
if (state.hnsw) {
|
|
7710
7778
|
state.compactedNodeCount += state.hnsw.tombstoneCount;
|
|
7711
7779
|
state.hnsw.compactTombstones();
|
|
7712
7780
|
}
|
|
7781
|
+
state.freshGraph?.compactTombstones();
|
|
7713
7782
|
for (const docId of state.tombstones) {
|
|
7714
7783
|
state.store.remove(docId);
|
|
7715
7784
|
state.buffer.delete(docId);
|
|
@@ -7717,6 +7786,7 @@ function compact(state) {
|
|
|
7717
7786
|
state.osq.remove(docId);
|
|
7718
7787
|
}
|
|
7719
7788
|
}
|
|
7789
|
+
releaseReplacedVectors(state);
|
|
7720
7790
|
state.tombstones.clear();
|
|
7721
7791
|
}
|
|
7722
7792
|
function recalibrateWhileNoThreadSearches(state, graph) {
|
|
@@ -7748,6 +7818,7 @@ async function foldIntoGraph(state) {
|
|
|
7748
7818
|
const graph = state.hnsw;
|
|
7749
7819
|
if (graph === null || rebuildNeeded) {
|
|
7750
7820
|
if (state.osq?.isCalibrated()) recalibrateWhileNoThreadSearches(state, graph);
|
|
7821
|
+
else calibrateQuantizer(state);
|
|
7751
7822
|
await buildGraphFromStore(state);
|
|
7752
7823
|
} else {
|
|
7753
7824
|
await insertMissing(state, graph);
|
|
@@ -7794,8 +7865,9 @@ function maintenanceStatus(state) {
|
|
|
7794
7865
|
const storeSize = state.store.size;
|
|
7795
7866
|
const graphCount = state.hnsw ? 1 : 0;
|
|
7796
7867
|
const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
|
|
7797
|
-
const
|
|
7798
|
-
const
|
|
7868
|
+
const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
|
|
7869
|
+
const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
|
|
7870
|
+
const estimatedCompactMs = Math.round(tombstoned * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
|
|
7799
7871
|
const optimizeVectorCount = state.hnsw === null || graphNeedsRebuild(state) ? storeSize : state.buffer.size;
|
|
7800
7872
|
const estimatedOptimizeMs = Math.round(optimizeVectorCount * ESTIMATED_MS_PER_VECTOR_REBUILD * state.dimensionScale);
|
|
7801
7873
|
return {
|
|
@@ -8056,19 +8128,19 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
8056
8128
|
return scannedOutcome(scan2);
|
|
8057
8129
|
}
|
|
8058
8130
|
}
|
|
8059
|
-
const
|
|
8060
|
-
|
|
8061
|
-
|
|
8062
|
-
|
|
8063
|
-
|
|
8064
|
-
|
|
8065
|
-
|
|
8066
|
-
|
|
8067
|
-
|
|
8068
|
-
|
|
8069
|
-
const
|
|
8070
|
-
const buffered = bruteForceSearch(state, query, k, metric, minSimilarity,
|
|
8071
|
-
return
|
|
8131
|
+
const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample }).map((r) => ({ docId: r.docId, score: r.score }));
|
|
8132
|
+
return fetchedOutcome(
|
|
8133
|
+
state,
|
|
8134
|
+
withBufferedVectors(state, query, k, options, filter, hnswResults),
|
|
8135
|
+
minSimilarity,
|
|
8136
|
+
filter
|
|
8137
|
+
);
|
|
8138
|
+
}
|
|
8139
|
+
function withBufferedVectors(state, query, k, options, filter, graphResults) {
|
|
8140
|
+
if (state.buffer.size === 0) return graphResults;
|
|
8141
|
+
const candidates = bufferCandidates(state, filter);
|
|
8142
|
+
const buffered = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
|
|
8143
|
+
return mergeResults(graphResults, buffered.results, k);
|
|
8072
8144
|
}
|
|
8073
8145
|
|
|
8074
8146
|
// src/vector/vector-index/index.ts
|
|
@@ -8169,11 +8241,12 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
8169
8241
|
}
|
|
8170
8242
|
noteWrite(state);
|
|
8171
8243
|
state.tombstones.delete(docId);
|
|
8172
|
-
state.
|
|
8244
|
+
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
8245
|
+
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
8173
8246
|
if (previous !== void 0) {
|
|
8174
8247
|
state.hnsw?.markTombstoneOrdinal(previous);
|
|
8175
8248
|
state.freshGraph?.markTombstoneOrdinal(previous);
|
|
8176
|
-
state.osq?.removeOrdinal(previous);
|
|
8249
|
+
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
8177
8250
|
}
|
|
8178
8251
|
state.buffer.add(docId);
|
|
8179
8252
|
state.pendingLocations.delete(docId);
|
|
@@ -8243,10 +8316,13 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
8243
8316
|
efSearch: options.efSearch,
|
|
8244
8317
|
oversample: options.oversample
|
|
8245
8318
|
});
|
|
8246
|
-
if (viaWorkerCopy !== null) return fetchedOutcome(state, viaWorkerCopy, options.minSimilarity, filter);
|
|
8247
8319
|
if (confined && state.revision !== filterRevision) {
|
|
8248
8320
|
filter = filterForOptions(state, options);
|
|
8249
8321
|
}
|
|
8322
|
+
if (viaWorkerCopy !== null) {
|
|
8323
|
+
const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
|
|
8324
|
+
return fetchedOutcome(state, results, options.minSimilarity, filter);
|
|
8325
|
+
}
|
|
8250
8326
|
return searchWithFilter(state, query, k, options, filter);
|
|
8251
8327
|
}
|
|
8252
8328
|
return {
|
|
@@ -10942,4 +11018,4 @@ function createDirectExecutor(options) {
|
|
|
10942
11018
|
};
|
|
10943
11019
|
}
|
|
10944
11020
|
|
|
10945
|
-
export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES,
|
|
11021
|
+
export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerStopWords, registerTokenizer, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-
|
|
2
|
-
import { getLanguage } from './chunk-
|
|
3
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-
|
|
1
|
+
import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-2453RJYK.mjs';
|
|
2
|
+
import { getLanguage } from './chunk-OLXM65YH.mjs';
|
|
3
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-ZCKPRCYI.mjs';
|
|
4
4
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
6
6
|
import { encode, decode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
// src/distribution/cluster/index-settings.ts
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { EARLIER_RELEASE_MIN_M, MIN_M, MAX_M } from './chunk-KEHK2O6P.mjs';
|
|
2
|
-
import { DEFAULT_PATTERN_VALUE_LIMIT, isPlainObject, isWordIndexedBase, MAX_TEXT_FIELDS, MAX_VECTOR_DIMENSION, parseFieldType, MAX_DOCUMENT_BYTES, MAX_NESTING_DEPTH, PROTOTYPE_POLLUTION_KEYS, FIELD_NAME_PATTERN, RESERVED_ROOT_FIELDS, MAX_DOCUMENT_NESTING_DEPTH, VECTOR_PATTERN } from './chunk-
|
|
2
|
+
import { DEFAULT_PATTERN_VALUE_LIMIT, isPlainObject, isWordIndexedBase, MAX_TEXT_FIELDS, MAX_VECTOR_DIMENSION, parseFieldType, MAX_DOCUMENT_BYTES, MAX_NESTING_DEPTH, PROTOTYPE_POLLUTION_KEYS, FIELD_NAME_PATTERN, RESERVED_ROOT_FIELDS, MAX_DOCUMENT_NESTING_DEPTH, VECTOR_PATTERN } from './chunk-LS5IRM23.mjs';
|
|
3
3
|
import { compareCodePoints, holdsUnpairedSurrogate, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
5
5
|
import { encode } from '@msgpack/msgpack';
|
|
6
6
|
|
|
7
7
|
// src/schema/validator/schema.ts
|