@delali/narsil 0.3.1-next.112 → 0.3.1-next.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-UR5MNN53.mjs → chunk-2TMWIATH.mjs} +108 -67
- package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-73U5DWOZ.mjs} +39 -4
- package/dist/browser/{chunk-O5IKTA27.mjs → chunk-FODCC3MK.mjs} +1 -1
- package/dist/browser/{chunk-YRYBTRWB.mjs → chunk-I45FAGOF.mjs} +2 -2
- package/dist/browser/{durability-wiring-CPEMDDK7.mjs → durability-wiring-CSMTKEYX.mjs} +3 -3
- package/dist/browser/{module-DYKR4INA.mjs → module-G2PPZV3B.mjs} +127 -33
- package/dist/browser/{restore-queue-2H4QDQV4.mjs → restore-queue-N7N42PBS.mjs} +1 -1
- package/dist/browser/{runtime-FL5K4JOH.mjs → runtime-BX6XRFYR.mjs} +2 -2
- package/dist/browser/{snapshot-ISZMBDCN.mjs → snapshot-EBH6RXQO.mjs} +6 -5
- package/dist/browser/{write-IABOLE4N.mjs → write-5ZKFINX3.mjs} +2 -2
- package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
- package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
- package/dist/{chunk-R2MRGYU5.mjs → chunk-4ZKBKPBV.mjs} +22 -2
- package/dist/{chunk-GLZI636G.mjs → chunk-5VVSHCXT.mjs} +27 -12
- package/dist/{chunk-3COL24MT.mjs → chunk-7EWQZQDT.mjs} +1 -1
- package/dist/{chunk-QI6JW4PM.mjs → chunk-BCNUJVTT.mjs} +1 -1
- package/dist/{chunk-3RRATITJ.mjs → chunk-BCSWODXP.mjs} +75 -55
- package/dist/{thread-WGBCVJII.mjs → chunk-DEXID4ZT.mjs} +301 -200
- package/dist/chunk-EBIHRL7B.mjs +21 -0
- package/dist/{chunk-YIDSHBQS.mjs → chunk-HC5NURS6.mjs} +33 -4
- package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
- package/dist/{chunk-63HFWQEA.mjs → chunk-IE43SRL3.mjs} +276 -102
- package/dist/chunk-IWVEG5CX.mjs +67 -0
- package/dist/{chunk-2IFU7SXG.mjs → chunk-KGVC6KJ3.mjs} +4 -4
- package/dist/{chunk-7CI34JAP.mjs → chunk-KVLX65KD.mjs} +4 -4
- package/dist/{chunk-5WEBVYOA.mjs → chunk-N53HN5VA.mjs} +16 -9
- package/dist/chunk-N6J6EJW3.mjs +198 -0
- package/dist/{chunk-PT7VRWS7.mjs → chunk-NX7KU77V.mjs} +3 -2
- package/dist/{chunk-QIFOXLWF.mjs → chunk-OGFSUXYY.mjs} +2 -2
- package/dist/chunk-PELHVTO7.mjs +24 -0
- package/dist/{chunk-V7ISFBFC.mjs → chunk-Q62NDKEC.mjs} +5 -4
- package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
- package/dist/{chunk-BVOUSX3U.mjs → chunk-UDZYIWDP.mjs} +3 -3
- package/dist/{chunk-HNAP2KTG.mjs → chunk-VFAXPDBH.mjs} +1 -0
- package/dist/{chunk-IBTKJIV7.mjs → chunk-VOKEYJHU.mjs} +1 -1
- package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
- package/dist/{chunk-IM76DLA7.mjs → chunk-YCFQ5754.mjs} +2 -2
- package/dist/{chunk-VLRT47GU.mjs → chunk-YCSRB3IN.mjs} +3 -3
- package/dist/{chunk-CQUMNIIG.mjs → chunk-Z4DZ4KK2.mjs} +1 -0
- package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
- package/dist/{chunk-2RAQKITH.mjs → chunk-ZR63OGT2.mjs} +1 -1
- package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
- package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
- package/dist/{direct-executor-ZSTKRA6Z.mjs → direct-executor-OUY4RPTU.mjs} +7 -6
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +79 -42
- package/dist/{durability-wiring-EQTFL3B4.mjs → durability-wiring-5V6DVHFK.mjs} +17 -16
- package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
- package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +10 -9
- package/dist/long-query-thread-HMQM2FWL.mjs +118 -0
- package/dist/{module-WP4VVGV2.mjs → module-3IMTXNPW.mjs} +92 -29
- package/dist/pattern.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -13
- package/dist/react.d.ts +6 -6
- package/dist/{rerank-3POMPQNX.mjs → rerank-NYX5S7OG.mjs} +3 -3
- package/dist/{restore-queue-WZTMTJP2.mjs → restore-queue-J6PCG4ON.mjs} +1 -1
- package/dist/{runtime-C2435MPK.mjs → runtime-XNYW6AWX.mjs} +2 -2
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +110 -258
- package/dist/{snapshot-EU6NNB2P.mjs → snapshot-57WF53MK.mjs} +18 -17
- package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
- package/dist/thread-FDEHU5G3.mjs +31 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +19 -7
- package/dist/{write-CKRWS3P5.mjs → write-YEI7A5OR.mjs} +11 -10
- package/package.json +7 -7
|
@@ -1,14 +1,17 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { noteWaitingForMemory, admitsBackgroundBytes } from './chunk-PELHVTO7.mjs';
|
|
2
|
+
import { chargeHeapBytes } from './chunk-IWVEG5CX.mjs';
|
|
3
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, WORKER_COPY_MIN_VECTORS, allLiveDocIds, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-NX7KU77V.mjs';
|
|
2
4
|
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-OCB7TMUQ.mjs';
|
|
3
|
-
import { createVectorStore } from './chunk-
|
|
4
|
-
export { openSharedVectorField } from './chunk-
|
|
5
|
-
import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-
|
|
6
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
5
|
+
import { createVectorStore } from './chunk-4ZKBKPBV.mjs';
|
|
6
|
+
export { openSharedVectorField } from './chunk-4ZKBKPBV.mjs';
|
|
7
|
+
import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-HC5NURS6.mjs';
|
|
8
|
+
export { releaseLocksHeldBy } from './chunk-HC5NURS6.mjs';
|
|
7
9
|
import { noteFallback } from './chunk-DJHHF2L5.mjs';
|
|
8
10
|
import './chunk-37CD3OKS.mjs';
|
|
9
|
-
import { spawnNodeWorker } from './chunk-
|
|
11
|
+
import { spawnNodeWorker } from './chunk-W55X2BYN.mjs';
|
|
10
12
|
import { resolveWorkerCount, resolveWorkerEntry, workerResourceLimits } from './chunk-6PN3U6XW.mjs';
|
|
11
13
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
14
|
+
import './chunk-EBIHRL7B.mjs';
|
|
12
15
|
import './chunk-LXWDQLTW.mjs';
|
|
13
16
|
import './chunk-KOKM4PQN.mjs';
|
|
14
17
|
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, UNPLACED_FLAG_CHUNK_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-QSK35UC2.mjs';
|
|
@@ -18,6 +21,8 @@ import './chunk-E74YLAIG.mjs';
|
|
|
18
21
|
import './chunk-VJS4JFBQ.mjs';
|
|
19
22
|
import './chunk-R6A773WK.mjs';
|
|
20
23
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
24
|
+
import './chunk-RIIUZJQ3.mjs';
|
|
25
|
+
import './chunk-22YPZVMG.mjs';
|
|
21
26
|
import { noteThreadSlotLive, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
|
|
22
27
|
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-OBWUWZRA.mjs';
|
|
23
28
|
|
|
@@ -189,12 +194,13 @@ function readStoreFlags(store) {
|
|
|
189
194
|
}
|
|
190
195
|
|
|
191
196
|
// src/vector/shared-field/searcher.ts
|
|
192
|
-
function distancesInTypeScript(store, query, metric, ordinals) {
|
|
197
|
+
function distancesInTypeScript(store, query, metric, ordinals, clock) {
|
|
193
198
|
noteFallback("score");
|
|
194
199
|
const arenaQuery = store.prepareQueryArena(query);
|
|
195
200
|
const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
|
|
196
201
|
const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
|
|
197
202
|
for (let i = 0; i < ordinals.length; i++) {
|
|
203
|
+
if (clock !== void 0) clock.tick(query.length);
|
|
198
204
|
if (arenaQuery) {
|
|
199
205
|
distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
|
|
200
206
|
continue;
|
|
@@ -204,10 +210,12 @@ function distancesInTypeScript(store, query, metric, ordinals) {
|
|
|
204
210
|
}
|
|
205
211
|
return distances;
|
|
206
212
|
}
|
|
207
|
-
function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
|
|
213
|
+
function scoreCandidates(store, candidates, query, k, metric, minSimilarity, clock) {
|
|
208
214
|
const { ordinals, docIds } = candidates;
|
|
209
215
|
if (ordinals.length === 0) return { results: [], matched: 0 };
|
|
210
|
-
|
|
216
|
+
clock?.check();
|
|
217
|
+
const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals, clock);
|
|
218
|
+
clock?.check();
|
|
211
219
|
const heap = createBoundedMaxHeap(highScoreFirst, k);
|
|
212
220
|
let matched = 0;
|
|
213
221
|
for (let i = 0; i < ordinals.length; i++) {
|
|
@@ -278,6 +286,7 @@ function createSharedVectorSearcher(options) {
|
|
|
278
286
|
}
|
|
279
287
|
function search2(query, k, searchOptions, filter) {
|
|
280
288
|
const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = searchOptions;
|
|
289
|
+
const clock = searchOptions.clock;
|
|
281
290
|
const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
|
|
282
291
|
const liveDocuments = flags.liveDocuments();
|
|
283
292
|
if (liveDocuments <= 0) return empty;
|
|
@@ -293,23 +302,27 @@ function createSharedVectorSearcher(options) {
|
|
|
293
302
|
const graph = view.graph;
|
|
294
303
|
if (graph === null) {
|
|
295
304
|
const ordinals = filter === void 0 ? flags.unplacedOrdinals() : ordinalFilterValues(filter);
|
|
296
|
-
return scanned(
|
|
305
|
+
return scanned(
|
|
306
|
+
scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity, clock)
|
|
307
|
+
);
|
|
297
308
|
}
|
|
298
309
|
if (filter !== void 0) {
|
|
299
310
|
const graphLiveSize = view.liveSize;
|
|
300
311
|
const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
|
|
301
312
|
if (selectivity < view.handles.filterThreshold) {
|
|
302
313
|
const candidates = candidatesOf(ordinalFilterValues(filter), filter);
|
|
303
|
-
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity));
|
|
314
|
+
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity, clock));
|
|
304
315
|
}
|
|
305
316
|
}
|
|
317
|
+
const unplaced = flags.unplacedOrdinals();
|
|
306
318
|
let resolvedEveryHit = true;
|
|
307
319
|
const graphResults = [];
|
|
308
320
|
for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, {
|
|
309
321
|
filter,
|
|
310
322
|
efSearch,
|
|
311
323
|
oversample,
|
|
312
|
-
rescoreWindow
|
|
324
|
+
rescoreWindow,
|
|
325
|
+
clock
|
|
313
326
|
})) {
|
|
314
327
|
const docId = view.docIdOf(hit.ord);
|
|
315
328
|
if (docId === void 0) {
|
|
@@ -318,9 +331,8 @@ function createSharedVectorSearcher(options) {
|
|
|
318
331
|
}
|
|
319
332
|
graphResults.push({ docId, score: hit.score });
|
|
320
333
|
}
|
|
321
|
-
const unplaced = flags.unplacedOrdinals();
|
|
322
334
|
if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
|
|
323
|
-
const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity);
|
|
335
|
+
const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity, clock);
|
|
324
336
|
return fetched(mergeResults(graphResults, scan.results, k), resolvedEveryHit);
|
|
325
337
|
}
|
|
326
338
|
return {
|
|
@@ -1206,6 +1218,35 @@ async function releasePendingLocations(state) {
|
|
|
1206
1218
|
await releaseInTurn(state, () => releaseLocations(state, paths, pending));
|
|
1207
1219
|
}
|
|
1208
1220
|
|
|
1221
|
+
// src/vector/vector-index/memory-charge.ts
|
|
1222
|
+
function admitShuffledRebuild(state) {
|
|
1223
|
+
const bytes = state.hnsw?.graphBytes ?? 0;
|
|
1224
|
+
if (!admitsBackgroundBytes(bytes)) {
|
|
1225
|
+
state.waitingForMemory = true;
|
|
1226
|
+
noteWaitingForMemory(state, { indexName: state.indexName, fieldName: state.fieldName });
|
|
1227
|
+
return false;
|
|
1228
|
+
}
|
|
1229
|
+
stopWaitingForMemory(state);
|
|
1230
|
+
chargeHeapBytes(bytes - state.rebuildChargeBytes);
|
|
1231
|
+
state.rebuildChargeBytes = bytes;
|
|
1232
|
+
return true;
|
|
1233
|
+
}
|
|
1234
|
+
function stopWaitingForMemory(state) {
|
|
1235
|
+
if (!state.waitingForMemory) return;
|
|
1236
|
+
state.waitingForMemory = false;
|
|
1237
|
+
noteWaitingForMemory(state, null);
|
|
1238
|
+
}
|
|
1239
|
+
function releaseRebuildCharge(state) {
|
|
1240
|
+
chargeHeapBytes(-state.rebuildChargeBytes);
|
|
1241
|
+
state.rebuildChargeBytes = 0;
|
|
1242
|
+
}
|
|
1243
|
+
function releaseMemoryCharges(state) {
|
|
1244
|
+
releaseRebuildCharge(state);
|
|
1245
|
+
stopWaitingForMemory(state);
|
|
1246
|
+
if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
|
|
1247
|
+
state.rebuildWaitTimer = null;
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1209
1250
|
// src/vector/vector-index/build.ts
|
|
1210
1251
|
function bufferWhatTheGraphLacks(state, graph) {
|
|
1211
1252
|
for (const docId of allLiveDocIds(state)) {
|
|
@@ -1288,8 +1329,25 @@ async function rebuildInShuffledOrder(state) {
|
|
|
1288
1329
|
const adopted = await buildGraphFromStore(state);
|
|
1289
1330
|
if (adopted && replacing.loadedWithoutCount) state.onLoadedGraphRepaired();
|
|
1290
1331
|
}
|
|
1332
|
+
function retryWhenMemoryAllows(state) {
|
|
1333
|
+
if (state.rebuildWaitTimer !== null || state.disposed) return;
|
|
1334
|
+
const timer = setTimeout(() => {
|
|
1335
|
+
state.rebuildWaitTimer = null;
|
|
1336
|
+
startShuffledRebuild(state);
|
|
1337
|
+
}, SHUFFLED_REBUILD_MEMORY_RETRY_MS);
|
|
1338
|
+
timer.unref?.();
|
|
1339
|
+
state.rebuildWaitTimer = timer;
|
|
1340
|
+
}
|
|
1291
1341
|
function startShuffledRebuild(state) {
|
|
1292
|
-
if (state.shuffledRebuild !== null || state.disposed
|
|
1342
|
+
if (state.shuffledRebuild !== null || state.disposed) return;
|
|
1343
|
+
if (!shuffledRebuildDue(state)) {
|
|
1344
|
+
stopWaitingForMemory(state);
|
|
1345
|
+
return;
|
|
1346
|
+
}
|
|
1347
|
+
if (!admitShuffledRebuild(state)) {
|
|
1348
|
+
retryWhenMemoryAllows(state);
|
|
1349
|
+
return;
|
|
1350
|
+
}
|
|
1293
1351
|
state.shuffledRebuild = rebuildInShuffledOrder(state).catch((error) => {
|
|
1294
1352
|
console.warn(
|
|
1295
1353
|
`Rebuilding the graph of "${state.indexName}/${state.fieldName}" in a shuffled order failed, so the field keeps the graph it holds:`,
|
|
@@ -1297,6 +1355,7 @@ function startShuffledRebuild(state) {
|
|
|
1297
1355
|
);
|
|
1298
1356
|
}).finally(() => {
|
|
1299
1357
|
state.shuffledRebuild = null;
|
|
1358
|
+
releaseRebuildCharge(state);
|
|
1300
1359
|
scheduleBuild(state);
|
|
1301
1360
|
scheduleWorkerCopyLoad(state);
|
|
1302
1361
|
});
|
|
@@ -1422,9 +1481,12 @@ function releaseHeldMemory(state) {
|
|
|
1422
1481
|
state.savedFiles = [];
|
|
1423
1482
|
state.savedSignature = null;
|
|
1424
1483
|
state.store.release();
|
|
1484
|
+
releaseMemoryCharges(state);
|
|
1425
1485
|
}
|
|
1426
1486
|
function disposeIndex(state) {
|
|
1427
1487
|
state.disposed = true;
|
|
1488
|
+
if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
|
|
1489
|
+
state.rebuildWaitTimer = null;
|
|
1428
1490
|
if (state.buildTimer !== null) clearTimeout(state.buildTimer);
|
|
1429
1491
|
state.buildTimer = null;
|
|
1430
1492
|
invalidateWorkerCopies(state);
|
|
@@ -1763,6 +1825,7 @@ function maintenanceStatus(state) {
|
|
|
1763
1825
|
graphCount,
|
|
1764
1826
|
bufferSize: state.buffer.size,
|
|
1765
1827
|
building: state.building || state.buildScheduled || state.buildTimer !== null || state.shuffledRebuild !== null,
|
|
1828
|
+
waitingForMemory: state.waitingForMemory,
|
|
1766
1829
|
estimatedCompactMs,
|
|
1767
1830
|
estimatedOptimizeMs
|
|
1768
1831
|
};
|
|
@@ -1777,8 +1840,8 @@ function estimateMemoryBytes(state) {
|
|
|
1777
1840
|
}
|
|
1778
1841
|
|
|
1779
1842
|
// src/vector/vector-index/search.ts
|
|
1780
|
-
function* bufferCandidates(state, filter) {
|
|
1781
|
-
for (const docId of
|
|
1843
|
+
function* bufferCandidates(state, buffered, filter) {
|
|
1844
|
+
for (const docId of buffered) {
|
|
1782
1845
|
if (state.tombstones.has(docId)) continue;
|
|
1783
1846
|
if (filter) {
|
|
1784
1847
|
const ordinal = state.store.getOrdinal(docId);
|
|
@@ -1903,18 +1966,13 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
1903
1966
|
}
|
|
1904
1967
|
}
|
|
1905
1968
|
const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow }).map((r) => ({ docId: r.docId, score: r.score }));
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
withBufferedVectors(state, query, k, options, filter, hnswResults),
|
|
1909
|
-
minSimilarity,
|
|
1910
|
-
filter
|
|
1911
|
-
);
|
|
1969
|
+
const results = state.buffer.size === 0 ? hnswResults : withBufferedVectors(state, query, k, options, filter, hnswResults, state.buffer);
|
|
1970
|
+
return fetchedOutcome(state, results, minSimilarity, filter);
|
|
1912
1971
|
}
|
|
1913
|
-
function withBufferedVectors(state, query, k, options, filter, graphResults) {
|
|
1914
|
-
|
|
1915
|
-
const
|
|
1916
|
-
|
|
1917
|
-
return mergeResults(graphResults, buffered.results, k);
|
|
1972
|
+
function withBufferedVectors(state, query, k, options, filter, graphResults, buffered) {
|
|
1973
|
+
const candidates = bufferCandidates(state, buffered, filter);
|
|
1974
|
+
const scan = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
|
|
1975
|
+
return mergeResults(graphResults, scan.results, k);
|
|
1918
1976
|
}
|
|
1919
1977
|
|
|
1920
1978
|
// src/vector/vector-index/index.ts
|
|
@@ -1965,6 +2023,9 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
1965
2023
|
hnsw: null,
|
|
1966
2024
|
freshGraph: null,
|
|
1967
2025
|
...noBuildHistory(),
|
|
2026
|
+
rebuildChargeBytes: 0,
|
|
2027
|
+
waitingForMemory: false,
|
|
2028
|
+
rebuildWaitTimer: null,
|
|
1968
2029
|
onLoadedGraphRepaired,
|
|
1969
2030
|
compactedNodeCount: 0,
|
|
1970
2031
|
building: false,
|
|
@@ -2067,6 +2128,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2067
2128
|
}
|
|
2068
2129
|
}
|
|
2069
2130
|
const filterRevision = state.revision;
|
|
2131
|
+
const buffered = state.buffer.size === 0 ? null : [...state.buffer];
|
|
2070
2132
|
scheduleWorkerCopyLoad(state);
|
|
2071
2133
|
const viaWorkerCopy = await searchViaWorkerCopies(state, query, k, options.metric, options.minSimilarity, {
|
|
2072
2134
|
filter,
|
|
@@ -2078,7 +2140,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2078
2140
|
filter = filterForOptions(state, options);
|
|
2079
2141
|
}
|
|
2080
2142
|
if (viaWorkerCopy !== null) {
|
|
2081
|
-
const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
|
|
2143
|
+
const results = buffered === null ? viaWorkerCopy : withBufferedVectors(state, query, k, options, filter, viaWorkerCopy, buffered);
|
|
2082
2144
|
return fetchedOutcome(state, results, options.minSimilarity, filter);
|
|
2083
2145
|
}
|
|
2084
2146
|
return searchWithFilter(state, query, k, options, filter);
|
|
@@ -2122,6 +2184,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2122
2184
|
},
|
|
2123
2185
|
maintenanceStatus: () => maintenanceStatus(state),
|
|
2124
2186
|
estimateMemoryBytes: () => estimateMemoryBytes(state),
|
|
2187
|
+
growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
|
|
2125
2188
|
deserialize: (parts) => {
|
|
2126
2189
|
invalidateWorkerCopies(state);
|
|
2127
2190
|
deserialize(state, parts);
|
package/dist/pattern.mjs
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
|
-
import { createDurableDirectory } from '../../chunk-
|
|
2
|
-
import '../../chunk-
|
|
1
|
+
import { createDurableDirectory } from '../../chunk-YCFQ5754.mjs';
|
|
2
|
+
import '../../chunk-W55X2BYN.mjs';
|
|
3
3
|
import '../../chunk-HLPRHOSO.mjs';
|
|
4
|
-
import { CHECKPOINT_WORKER_HEARTBEAT_MS, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-
|
|
5
|
-
import '../../chunk-
|
|
4
|
+
import { CHECKPOINT_WORKER_HEARTBEAT_MS, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-KGVC6KJ3.mjs';
|
|
5
|
+
import '../../chunk-7EWQZQDT.mjs';
|
|
6
6
|
import '../../chunk-P36MUFEJ.mjs';
|
|
7
7
|
import '../../chunk-YH5OIL7B.mjs';
|
|
8
|
-
import '../../chunk-
|
|
9
|
-
import '../../chunk-
|
|
8
|
+
import '../../chunk-BCSWODXP.mjs';
|
|
9
|
+
import '../../chunk-5VVSHCXT.mjs';
|
|
10
10
|
import '../../chunk-N57XXGPG.mjs';
|
|
11
11
|
import '../../chunk-6PN3U6XW.mjs';
|
|
12
12
|
import '../../chunk-AV4IJKD2.mjs';
|
|
13
|
-
import '../../chunk-
|
|
14
|
-
import '../../chunk-
|
|
13
|
+
import '../../chunk-Q62NDKEC.mjs';
|
|
14
|
+
import '../../chunk-N53HN5VA.mjs';
|
|
15
15
|
import '../../chunk-FUUP45IS.mjs';
|
|
16
|
-
import '../../chunk-
|
|
16
|
+
import '../../chunk-ZATRDAYU.mjs';
|
|
17
|
+
import '../../chunk-EBIHRL7B.mjs';
|
|
17
18
|
import '../../chunk-HJSN7UYG.mjs';
|
|
18
19
|
import '../../chunk-LXWDQLTW.mjs';
|
|
19
20
|
import '../../chunk-TGP5M5R7.mjs';
|
|
@@ -21,7 +22,7 @@ import '../../chunk-U2BKYEC7.mjs';
|
|
|
21
22
|
import '../../chunk-KOKM4PQN.mjs';
|
|
22
23
|
import '../../chunk-QSK35UC2.mjs';
|
|
23
24
|
import '../../chunk-VK45J47R.mjs';
|
|
24
|
-
import '../../chunk-
|
|
25
|
+
import '../../chunk-VFAXPDBH.mjs';
|
|
25
26
|
import '../../chunk-WQBCBKY3.mjs';
|
|
26
27
|
import '../../chunk-KCQG2PJT.mjs';
|
|
27
28
|
import '../../chunk-DG62H7OD.mjs';
|
|
@@ -36,9 +37,9 @@ import '../../chunk-7X5JST5L.mjs';
|
|
|
36
37
|
import '../../chunk-OFYKBKKV.mjs';
|
|
37
38
|
import '../../chunk-R6A773WK.mjs';
|
|
38
39
|
import '../../chunk-YTJCGKM4.mjs';
|
|
39
|
-
import '../../chunk-
|
|
40
|
-
import '../../chunk-
|
|
41
|
-
import '../../chunk-
|
|
40
|
+
import '../../chunk-Z4DZ4KK2.mjs';
|
|
41
|
+
import '../../chunk-BCNUJVTT.mjs';
|
|
42
|
+
import '../../chunk-22YPZVMG.mjs';
|
|
42
43
|
import '../../chunk-RFBLQLLZ.mjs';
|
|
43
44
|
import '../../chunk-L5ORED7V.mjs';
|
|
44
45
|
import '../../chunk-OBWUWZRA.mjs';
|
package/dist/react.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { N as NarsilError, L as ListParams, g as ListResult, I as IndexInfo, k as IndexStats, Q as QueryParams, P as PreflightResult, e as QueryResult, h as SuggestParams, j as SuggestResult } from './events-
|
|
2
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './events-
|
|
3
|
-
import { e as TaskRecord, d as TaskProgress, b as ImportResult, c as TaskListQuery, T as TaskListPage } from './tasks-
|
|
4
|
-
export { I as ImportError, f as TaskStatus, g as TaskType } from './tasks-
|
|
1
|
+
import { N as NarsilError, L as ListParams, g as ListResult, I as IndexInfo, k as IndexStats, Q as QueryParams, P as PreflightResult, e as QueryResult, h as SuggestParams, j as SuggestResult } from './events-D8GPB1-H.js';
|
|
2
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './events-D8GPB1-H.js';
|
|
3
|
+
import { e as TaskRecord, d as TaskProgress, b as ImportResult, c as TaskListQuery, T as TaskListPage } from './tasks-5_LCFEzF.js';
|
|
4
|
+
export { I as ImportError, f as TaskStatus, g as TaskType } from './tasks-5_LCFEzF.js';
|
|
5
5
|
import { ReactNode, ReactElement } from 'react';
|
|
6
|
-
import { N as Narsil } from './engine-
|
|
7
|
-
import { N as NarsilClient, I as ImportSource } from './client-
|
|
6
|
+
import { N as Narsil } from './engine-CCBBGmMR.js';
|
|
7
|
+
import { N as NarsilClient, I as ImportSource } from './client-BGzLNt78.js';
|
|
8
8
|
import { A as AnyDocument } from './schema-4W_7tk1f.js';
|
|
9
9
|
import './synonyms-CpOAHPa-.js';
|
|
10
10
|
import './adapters-BQFQ4PRz.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { runRerankedQuery, rerankTextOf } from './chunk-
|
|
1
|
+
import { runRerankedQuery, rerankTextOf } from './chunk-OGFSUXYY.mjs';
|
|
2
2
|
import './chunk-KZ3NX2P7.mjs';
|
|
3
3
|
import { rerankOnHost } from './chunk-542TD4BL.mjs';
|
|
4
|
-
import './chunk-
|
|
4
|
+
import './chunk-ZR63OGT2.mjs';
|
|
5
5
|
import './chunk-HJSN7UYG.mjs';
|
|
6
6
|
import './chunk-LXWDQLTW.mjs';
|
|
7
7
|
import './chunk-E74YLAIG.mjs';
|
|
@@ -22,7 +22,7 @@ async function rerankClusterQuery(deps, indexName, params, firstStage, readDocum
|
|
|
22
22
|
host,
|
|
23
23
|
schema,
|
|
24
24
|
maxWindow: core.rerankSettings.maxWindow,
|
|
25
|
-
binding: host.queryBindingOf(params),
|
|
25
|
+
binding: () => host.queryBindingOf(params),
|
|
26
26
|
clampsPins: false,
|
|
27
27
|
fusesRankings: (request) => request.vector !== void 0 && (request.term !== void 0 || request.mode === "hybrid"),
|
|
28
28
|
firstStage,
|
|
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
18
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
19
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
20
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-57WF53MK.mjs');
|
|
22
22
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
23
|
});
|
|
24
24
|
const settled = restored.then(
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { runRerankedQuery, rerankTextOf } from './chunk-
|
|
1
|
+
import { runRerankedQuery, rerankTextOf } from './chunk-OGFSUXYY.mjs';
|
|
2
2
|
export { withRerankQueryText } from './chunk-KZ3NX2P7.mjs';
|
|
3
3
|
import { rerankOnHost } from './chunk-542TD4BL.mjs';
|
|
4
4
|
export { rerankOnHost } from './chunk-542TD4BL.mjs';
|
|
5
|
-
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-
|
|
5
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-ZR63OGT2.mjs';
|
|
6
6
|
import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
7
7
|
import './chunk-LXWDQLTW.mjs';
|
|
8
8
|
import { HYBRID_RESCORE_WINDOW } from './chunk-E74YLAIG.mjs';
|
package/dist/server.d.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './events-
|
|
2
|
-
import { c as ServerOptions, N as NarsilServer } from './capabilities-
|
|
3
|
-
export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, d as RequestContext, e as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, f as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-
|
|
4
|
-
import { N as Narsil } from './engine-
|
|
5
|
-
import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-
|
|
6
|
-
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-
|
|
1
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './events-D8GPB1-H.js';
|
|
2
|
+
import { c as ServerOptions, N as NarsilServer } from './capabilities-DOgB3pPf.js';
|
|
3
|
+
export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, d as RequestContext, e as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, f as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-DOgB3pPf.js';
|
|
4
|
+
import { N as Narsil } from './engine-CCBBGmMR.js';
|
|
5
|
+
import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-5_LCFEzF.js';
|
|
6
|
+
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-5_LCFEzF.js';
|
|
7
7
|
import './schema-4W_7tk1f.js';
|
|
8
8
|
import './adapters-BQFQ4PRz.js';
|
|
9
9
|
import './synonyms-CpOAHPa-.js';
|
|
10
|
-
import './config-
|
|
11
|
-
import './config-
|
|
10
|
+
import './config-CS5ADSrx.js';
|
|
11
|
+
import './config-CSYMTTm1.js';
|
|
12
12
|
import './index-Cl5qQauj.js';
|
|
13
13
|
import './types-DvncmvuQ.js';
|
|
14
14
|
|