@delali/narsil 0.3.1-next.113 → 0.3.1-next.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-DC2SZO4D.mjs → chunk-2TMWIATH.mjs} +107 -67
- package/dist/browser/{chunk-YZPOBLAS.mjs → chunk-73U5DWOZ.mjs} +4 -3
- package/dist/browser/{chunk-4FVSBSXY.mjs → chunk-FODCC3MK.mjs} +1 -1
- package/dist/browser/{chunk-3YPLBK4C.mjs → chunk-I45FAGOF.mjs} +2 -2
- package/dist/browser/{durability-wiring-2RG22WQJ.mjs → durability-wiring-CSMTKEYX.mjs} +3 -3
- package/dist/browser/{module-5EWHBKHZ.mjs → module-G2PPZV3B.mjs} +27 -24
- package/dist/browser/{restore-queue-WCFUB4NS.mjs → restore-queue-N7N42PBS.mjs} +1 -1
- package/dist/browser/{runtime-FL5K4JOH.mjs → runtime-BX6XRFYR.mjs} +2 -2
- package/dist/browser/{snapshot-BE3F2PNC.mjs → snapshot-EBH6RXQO.mjs} +4 -4
- package/dist/browser/{write-MV7K7OCU.mjs → write-5ZKFINX3.mjs} +2 -2
- package/dist/{chunk-H2GNWBLB.mjs → chunk-4ZKBKPBV.mjs} +1 -1
- package/dist/{chunk-GLZI636G.mjs → chunk-5VVSHCXT.mjs} +27 -12
- package/dist/{chunk-LVNBY2TS.mjs → chunk-7EWQZQDT.mjs} +1 -1
- package/dist/{chunk-4PFOOQFQ.mjs → chunk-BCNUJVTT.mjs} +1 -1
- package/dist/{chunk-PABYSTKO.mjs → chunk-BCSWODXP.mjs} +71 -54
- package/dist/{thread-S4SUCV4Y.mjs → chunk-DEXID4ZT.mjs} +300 -202
- package/dist/{chunk-4XWIW3F3.mjs → chunk-HC5NURS6.mjs} +4 -3
- package/dist/{chunk-AOLD3LPV.mjs → chunk-IE43SRL3.mjs} +4 -1
- package/dist/{chunk-27ZEY73B.mjs → chunk-KGVC6KJ3.mjs} +4 -4
- package/dist/{chunk-QGVMEMCG.mjs → chunk-KVLX65KD.mjs} +4 -4
- package/dist/{chunk-5WEBVYOA.mjs → chunk-N53HN5VA.mjs} +16 -9
- package/dist/chunk-N6J6EJW3.mjs +198 -0
- package/dist/{chunk-CU4OB64V.mjs → chunk-NX7KU77V.mjs} +1 -1
- package/dist/{chunk-QIFOXLWF.mjs → chunk-OGFSUXYY.mjs} +2 -2
- package/dist/{chunk-5QIRLJH4.mjs → chunk-Q62NDKEC.mjs} +4 -3
- package/dist/{chunk-DZXGNLK5.mjs → chunk-UDZYIWDP.mjs} +3 -3
- package/dist/{chunk-HNAP2KTG.mjs → chunk-VFAXPDBH.mjs} +1 -0
- package/dist/{chunk-OM5G2J6X.mjs → chunk-VOKEYJHU.mjs} +1 -1
- package/dist/{chunk-OABTAUCW.mjs → chunk-YCFQ5754.mjs} +1 -1
- package/dist/{chunk-JUAFWJON.mjs → chunk-YCSRB3IN.mjs} +3 -3
- package/dist/{chunk-CQUMNIIG.mjs → chunk-Z4DZ4KK2.mjs} +1 -0
- package/dist/{chunk-2RAQKITH.mjs → chunk-ZR63OGT2.mjs} +1 -1
- package/dist/{direct-executor-R4HOSVPE.mjs → direct-executor-OUY4RPTU.mjs} +4 -4
- package/dist/distribution.mjs +17 -17
- package/dist/{durability-wiring-XZI6355I.mjs → durability-wiring-5V6DVHFK.mjs} +13 -13
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +7 -7
- package/dist/long-query-thread-HMQM2FWL.mjs +118 -0
- package/dist/{module-WVKSJL74.mjs → module-3IMTXNPW.mjs} +30 -27
- package/dist/pattern.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +10 -10
- package/dist/{rerank-3POMPQNX.mjs → rerank-NYX5S7OG.mjs} +3 -3
- package/dist/{restore-queue-JAZAB5IQ.mjs → restore-queue-J6PCG4ON.mjs} +1 -1
- package/dist/{runtime-C2435MPK.mjs → runtime-XNYW6AWX.mjs} +2 -2
- package/dist/server.mjs +93 -199
- package/dist/{snapshot-PDUGHQ25.mjs → snapshot-57WF53MK.mjs} +14 -14
- package/dist/thread-FDEHU5G3.mjs +31 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +17 -5
- package/dist/{write-KVJ7YJJ6.mjs → write-YEI7A5OR.mjs} +8 -8
- package/package.json +7 -7
package/dist/index.mjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
export { VERSION, createNarsil } from './chunk-
|
|
2
|
-
import './chunk-
|
|
1
|
+
export { VERSION, createNarsil } from './chunk-BCSWODXP.mjs';
|
|
2
|
+
import './chunk-5VVSHCXT.mjs';
|
|
3
3
|
import './chunk-N57XXGPG.mjs';
|
|
4
4
|
import './chunk-6PN3U6XW.mjs';
|
|
5
5
|
import './chunk-AV4IJKD2.mjs';
|
|
6
|
-
export { registerSynonyms } from './chunk-
|
|
7
|
-
export { registerPartialWordSearch, registerPatternSearch } from './chunk-
|
|
6
|
+
export { registerSynonyms } from './chunk-Q62NDKEC.mjs';
|
|
7
|
+
export { registerPartialWordSearch, registerPatternSearch } from './chunk-N53HN5VA.mjs';
|
|
8
8
|
export { getLanguage, registerLanguage } from './chunk-FUUP45IS.mjs';
|
|
9
9
|
import './chunk-ZATRDAYU.mjs';
|
|
10
10
|
import './chunk-EBIHRL7B.mjs';
|
|
@@ -15,7 +15,7 @@ import './chunk-U2BKYEC7.mjs';
|
|
|
15
15
|
export { isSimdAvailable } from './chunk-KOKM4PQN.mjs';
|
|
16
16
|
import './chunk-QSK35UC2.mjs';
|
|
17
17
|
import './chunk-VK45J47R.mjs';
|
|
18
|
-
import './chunk-
|
|
18
|
+
import './chunk-VFAXPDBH.mjs';
|
|
19
19
|
import './chunk-WQBCBKY3.mjs';
|
|
20
20
|
import './chunk-KCQG2PJT.mjs';
|
|
21
21
|
import './chunk-DG62H7OD.mjs';
|
|
@@ -30,8 +30,8 @@ import './chunk-7X5JST5L.mjs';
|
|
|
30
30
|
export { clearNormalizationCache, configureNormalizationCache, getNormalizationCacheSize, resetNormalizationCache } from './chunk-OFYKBKKV.mjs';
|
|
31
31
|
import './chunk-R6A773WK.mjs';
|
|
32
32
|
import './chunk-YTJCGKM4.mjs';
|
|
33
|
-
import './chunk-
|
|
34
|
-
import './chunk-
|
|
33
|
+
import './chunk-Z4DZ4KK2.mjs';
|
|
34
|
+
import './chunk-BCNUJVTT.mjs';
|
|
35
35
|
import './chunk-22YPZVMG.mjs';
|
|
36
36
|
import './chunk-RFBLQLLZ.mjs';
|
|
37
37
|
import './chunk-L5ORED7V.mjs';
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { createRelayClient, createThreadReadEngine, searchHandler, isLongQueryPortNote } from './chunk-DEXID4ZT.mjs';
|
|
2
|
+
import { answerRequest } from './chunk-N6J6EJW3.mjs';
|
|
3
|
+
import './chunk-IE43SRL3.mjs';
|
|
4
|
+
import './chunk-KZ3NX2P7.mjs';
|
|
5
|
+
import './chunk-HFJXEFFV.mjs';
|
|
6
|
+
import './chunk-IWVEG5CX.mjs';
|
|
7
|
+
import './chunk-DJHHF2L5.mjs';
|
|
8
|
+
import './chunk-5VVSHCXT.mjs';
|
|
9
|
+
import './chunk-N53HN5VA.mjs';
|
|
10
|
+
import './chunk-ZATRDAYU.mjs';
|
|
11
|
+
import './chunk-EBIHRL7B.mjs';
|
|
12
|
+
import './chunk-HJSN7UYG.mjs';
|
|
13
|
+
import './chunk-LXWDQLTW.mjs';
|
|
14
|
+
import './chunk-QSK35UC2.mjs';
|
|
15
|
+
import './chunk-VK45J47R.mjs';
|
|
16
|
+
import './chunk-VFAXPDBH.mjs';
|
|
17
|
+
import './chunk-WQBCBKY3.mjs';
|
|
18
|
+
import './chunk-KCQG2PJT.mjs';
|
|
19
|
+
import './chunk-DG62H7OD.mjs';
|
|
20
|
+
import './chunk-U476IXKQ.mjs';
|
|
21
|
+
import './chunk-E74YLAIG.mjs';
|
|
22
|
+
import './chunk-VJS4JFBQ.mjs';
|
|
23
|
+
import './chunk-GES7U2PM.mjs';
|
|
24
|
+
import './chunk-6ODIY6ZB.mjs';
|
|
25
|
+
import './chunk-X5CV2E2B.mjs';
|
|
26
|
+
import './chunk-OFYKBKKV.mjs';
|
|
27
|
+
import './chunk-R6A773WK.mjs';
|
|
28
|
+
import './chunk-YTJCGKM4.mjs';
|
|
29
|
+
import './chunk-RIIUZJQ3.mjs';
|
|
30
|
+
import './chunk-22YPZVMG.mjs';
|
|
31
|
+
import './chunk-L5ORED7V.mjs';
|
|
32
|
+
import './chunk-OBWUWZRA.mjs';
|
|
33
|
+
|
|
34
|
+
// src/server/request-threads/long-query-thread.ts
|
|
35
|
+
var serving = null;
|
|
36
|
+
function isMovedRequest(message) {
|
|
37
|
+
if (typeof message !== "object" || message === null) return false;
|
|
38
|
+
const { type, requestId } = message;
|
|
39
|
+
return type === "request" && typeof requestId === "number";
|
|
40
|
+
}
|
|
41
|
+
function streamNote(message) {
|
|
42
|
+
if (typeof message !== "object" || message === null) return null;
|
|
43
|
+
const { type, requestId } = message;
|
|
44
|
+
if (typeof requestId !== "number") return null;
|
|
45
|
+
return type === "stream-credit" || type === "stream-cancel" ? { type, requestId } : null;
|
|
46
|
+
}
|
|
47
|
+
function answerMovesOn(port, handlers) {
|
|
48
|
+
const streams = /* @__PURE__ */ new Map();
|
|
49
|
+
port.on("message", (message) => {
|
|
50
|
+
const note = streamNote(message);
|
|
51
|
+
if (note !== null) {
|
|
52
|
+
if (note.type === "stream-credit") streams.get(note.requestId)?.credit();
|
|
53
|
+
else streams.get(note.requestId)?.cancel();
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
if (!isMovedRequest(message)) return;
|
|
57
|
+
const handler = message.route === "search" || message.route === "preflight" ? handlers[message.route] : null;
|
|
58
|
+
if (handler === null) return;
|
|
59
|
+
void answerRequest(handler, message, port, streams).then((answer) => {
|
|
60
|
+
if (answer !== null) port.postMessage(answer);
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
function serveLongQueries(executor, settings) {
|
|
65
|
+
if (serving !== null) throw new Error("This worker already answers long queries");
|
|
66
|
+
const relay = createRelayClient(settings.port, settings.touchIntervalMs);
|
|
67
|
+
const local = createThreadReadEngine({
|
|
68
|
+
executor,
|
|
69
|
+
relay,
|
|
70
|
+
searchHooks: false,
|
|
71
|
+
patternWorkCap: settings.patternWorkCap,
|
|
72
|
+
maxRerankWindow: settings.maxRerankWindow
|
|
73
|
+
});
|
|
74
|
+
const afterWrites = (answer) => async (indexName, params) => {
|
|
75
|
+
await relay.awaitWrites(indexName);
|
|
76
|
+
return answer(indexName, params);
|
|
77
|
+
};
|
|
78
|
+
const shared = {
|
|
79
|
+
limits: settings.limits,
|
|
80
|
+
allowRegex: settings.allowRegex,
|
|
81
|
+
move: async () => false
|
|
82
|
+
};
|
|
83
|
+
const query = afterWrites((name, params) => local.queryHere(name, params));
|
|
84
|
+
const preflight = afterWrites((name, params) => local.preflightHere(name, params));
|
|
85
|
+
const handlers = {
|
|
86
|
+
search: searchHandler({
|
|
87
|
+
...shared,
|
|
88
|
+
fallback: (ctx) => relay.request("search", ctx),
|
|
89
|
+
staysOnThread: (name, params) => local.canAnswer(name, params),
|
|
90
|
+
answer: query,
|
|
91
|
+
answerHere: query
|
|
92
|
+
}),
|
|
93
|
+
preflight: searchHandler({
|
|
94
|
+
...shared,
|
|
95
|
+
fallback: (ctx) => relay.request("preflight", ctx),
|
|
96
|
+
staysOnThread: (name, params) => local.canCount(name, params),
|
|
97
|
+
answer: preflight,
|
|
98
|
+
answerHere: preflight
|
|
99
|
+
})
|
|
100
|
+
};
|
|
101
|
+
const thread = { relay, ports: /* @__PURE__ */ new Set() };
|
|
102
|
+
settings.port.on("message", (message) => {
|
|
103
|
+
if (!isLongQueryPortNote(message) || serving !== thread) return;
|
|
104
|
+
thread.ports.add(message.port);
|
|
105
|
+
message.port.on("close", () => thread.ports.delete(message.port));
|
|
106
|
+
answerMovesOn(message.port, handlers);
|
|
107
|
+
});
|
|
108
|
+
serving = thread;
|
|
109
|
+
}
|
|
110
|
+
function closeLongQueryThread() {
|
|
111
|
+
if (serving === null) return;
|
|
112
|
+
const { relay, ports } = serving;
|
|
113
|
+
serving = null;
|
|
114
|
+
for (const port of ports) port.close();
|
|
115
|
+
relay.close();
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export { closeLongQueryThread, serveLongQueries };
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { noteWaitingForMemory, admitsBackgroundBytes } from './chunk-PELHVTO7.mjs';
|
|
2
2
|
import { chargeHeapBytes } from './chunk-IWVEG5CX.mjs';
|
|
3
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, WORKER_COPY_MIN_VECTORS, allLiveDocIds, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-
|
|
3
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, WORKER_COPY_MIN_VECTORS, allLiveDocIds, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-NX7KU77V.mjs';
|
|
4
4
|
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-OCB7TMUQ.mjs';
|
|
5
|
-
import { createVectorStore } from './chunk-
|
|
6
|
-
export { openSharedVectorField } from './chunk-
|
|
7
|
-
import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-
|
|
8
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
5
|
+
import { createVectorStore } from './chunk-4ZKBKPBV.mjs';
|
|
6
|
+
export { openSharedVectorField } from './chunk-4ZKBKPBV.mjs';
|
|
7
|
+
import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-HC5NURS6.mjs';
|
|
8
|
+
export { releaseLocksHeldBy } from './chunk-HC5NURS6.mjs';
|
|
9
9
|
import { noteFallback } from './chunk-DJHHF2L5.mjs';
|
|
10
10
|
import './chunk-37CD3OKS.mjs';
|
|
11
11
|
import { spawnNodeWorker } from './chunk-W55X2BYN.mjs';
|
|
@@ -194,12 +194,13 @@ function readStoreFlags(store) {
|
|
|
194
194
|
}
|
|
195
195
|
|
|
196
196
|
// src/vector/shared-field/searcher.ts
|
|
197
|
-
function distancesInTypeScript(store, query, metric, ordinals) {
|
|
197
|
+
function distancesInTypeScript(store, query, metric, ordinals, clock) {
|
|
198
198
|
noteFallback("score");
|
|
199
199
|
const arenaQuery = store.prepareQueryArena(query);
|
|
200
200
|
const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
|
|
201
201
|
const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
|
|
202
202
|
for (let i = 0; i < ordinals.length; i++) {
|
|
203
|
+
if (clock !== void 0) clock.tick(query.length);
|
|
203
204
|
if (arenaQuery) {
|
|
204
205
|
distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
|
|
205
206
|
continue;
|
|
@@ -209,10 +210,12 @@ function distancesInTypeScript(store, query, metric, ordinals) {
|
|
|
209
210
|
}
|
|
210
211
|
return distances;
|
|
211
212
|
}
|
|
212
|
-
function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
|
|
213
|
+
function scoreCandidates(store, candidates, query, k, metric, minSimilarity, clock) {
|
|
213
214
|
const { ordinals, docIds } = candidates;
|
|
214
215
|
if (ordinals.length === 0) return { results: [], matched: 0 };
|
|
215
|
-
|
|
216
|
+
clock?.check();
|
|
217
|
+
const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals, clock);
|
|
218
|
+
clock?.check();
|
|
216
219
|
const heap = createBoundedMaxHeap(highScoreFirst, k);
|
|
217
220
|
let matched = 0;
|
|
218
221
|
for (let i = 0; i < ordinals.length; i++) {
|
|
@@ -283,6 +286,7 @@ function createSharedVectorSearcher(options) {
|
|
|
283
286
|
}
|
|
284
287
|
function search2(query, k, searchOptions, filter) {
|
|
285
288
|
const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = searchOptions;
|
|
289
|
+
const clock = searchOptions.clock;
|
|
286
290
|
const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
|
|
287
291
|
const liveDocuments = flags.liveDocuments();
|
|
288
292
|
if (liveDocuments <= 0) return empty;
|
|
@@ -298,23 +302,27 @@ function createSharedVectorSearcher(options) {
|
|
|
298
302
|
const graph = view.graph;
|
|
299
303
|
if (graph === null) {
|
|
300
304
|
const ordinals = filter === void 0 ? flags.unplacedOrdinals() : ordinalFilterValues(filter);
|
|
301
|
-
return scanned(
|
|
305
|
+
return scanned(
|
|
306
|
+
scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity, clock)
|
|
307
|
+
);
|
|
302
308
|
}
|
|
303
309
|
if (filter !== void 0) {
|
|
304
310
|
const graphLiveSize = view.liveSize;
|
|
305
311
|
const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
|
|
306
312
|
if (selectivity < view.handles.filterThreshold) {
|
|
307
313
|
const candidates = candidatesOf(ordinalFilterValues(filter), filter);
|
|
308
|
-
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity));
|
|
314
|
+
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity, clock));
|
|
309
315
|
}
|
|
310
316
|
}
|
|
317
|
+
const unplaced = flags.unplacedOrdinals();
|
|
311
318
|
let resolvedEveryHit = true;
|
|
312
319
|
const graphResults = [];
|
|
313
320
|
for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, {
|
|
314
321
|
filter,
|
|
315
322
|
efSearch,
|
|
316
323
|
oversample,
|
|
317
|
-
rescoreWindow
|
|
324
|
+
rescoreWindow,
|
|
325
|
+
clock
|
|
318
326
|
})) {
|
|
319
327
|
const docId = view.docIdOf(hit.ord);
|
|
320
328
|
if (docId === void 0) {
|
|
@@ -323,9 +331,8 @@ function createSharedVectorSearcher(options) {
|
|
|
323
331
|
}
|
|
324
332
|
graphResults.push({ docId, score: hit.score });
|
|
325
333
|
}
|
|
326
|
-
const unplaced = flags.unplacedOrdinals();
|
|
327
334
|
if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
|
|
328
|
-
const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity);
|
|
335
|
+
const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity, clock);
|
|
329
336
|
return fetched(mergeResults(graphResults, scan.results, k), resolvedEveryHit);
|
|
330
337
|
}
|
|
331
338
|
return {
|
|
@@ -1833,8 +1840,8 @@ function estimateMemoryBytes(state) {
|
|
|
1833
1840
|
}
|
|
1834
1841
|
|
|
1835
1842
|
// src/vector/vector-index/search.ts
|
|
1836
|
-
function* bufferCandidates(state, filter) {
|
|
1837
|
-
for (const docId of
|
|
1843
|
+
function* bufferCandidates(state, buffered, filter) {
|
|
1844
|
+
for (const docId of buffered) {
|
|
1838
1845
|
if (state.tombstones.has(docId)) continue;
|
|
1839
1846
|
if (filter) {
|
|
1840
1847
|
const ordinal = state.store.getOrdinal(docId);
|
|
@@ -1959,18 +1966,13 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
1959
1966
|
}
|
|
1960
1967
|
}
|
|
1961
1968
|
const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow }).map((r) => ({ docId: r.docId, score: r.score }));
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
withBufferedVectors(state, query, k, options, filter, hnswResults),
|
|
1965
|
-
minSimilarity,
|
|
1966
|
-
filter
|
|
1967
|
-
);
|
|
1969
|
+
const results = state.buffer.size === 0 ? hnswResults : withBufferedVectors(state, query, k, options, filter, hnswResults, state.buffer);
|
|
1970
|
+
return fetchedOutcome(state, results, minSimilarity, filter);
|
|
1968
1971
|
}
|
|
1969
|
-
function withBufferedVectors(state, query, k, options, filter, graphResults) {
|
|
1970
|
-
|
|
1971
|
-
const
|
|
1972
|
-
|
|
1973
|
-
return mergeResults(graphResults, buffered.results, k);
|
|
1972
|
+
function withBufferedVectors(state, query, k, options, filter, graphResults, buffered) {
|
|
1973
|
+
const candidates = bufferCandidates(state, buffered, filter);
|
|
1974
|
+
const scan = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
|
|
1975
|
+
return mergeResults(graphResults, scan.results, k);
|
|
1974
1976
|
}
|
|
1975
1977
|
|
|
1976
1978
|
// src/vector/vector-index/index.ts
|
|
@@ -2126,6 +2128,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2126
2128
|
}
|
|
2127
2129
|
}
|
|
2128
2130
|
const filterRevision = state.revision;
|
|
2131
|
+
const buffered = state.buffer.size === 0 ? null : [...state.buffer];
|
|
2129
2132
|
scheduleWorkerCopyLoad(state);
|
|
2130
2133
|
const viaWorkerCopy = await searchViaWorkerCopies(state, query, k, options.metric, options.minSimilarity, {
|
|
2131
2134
|
filter,
|
|
@@ -2137,7 +2140,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2137
2140
|
filter = filterForOptions(state, options);
|
|
2138
2141
|
}
|
|
2139
2142
|
if (viaWorkerCopy !== null) {
|
|
2140
|
-
const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
|
|
2143
|
+
const results = buffered === null ? viaWorkerCopy : withBufferedVectors(state, query, k, options, filter, viaWorkerCopy, buffered);
|
|
2141
2144
|
return fetchedOutcome(state, results, options.minSimilarity, filter);
|
|
2142
2145
|
}
|
|
2143
2146
|
return searchWithFilter(state, query, k, options, filter);
|
package/dist/pattern.mjs
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import { createDurableDirectory } from '../../chunk-
|
|
1
|
+
import { createDurableDirectory } from '../../chunk-YCFQ5754.mjs';
|
|
2
2
|
import '../../chunk-W55X2BYN.mjs';
|
|
3
3
|
import '../../chunk-HLPRHOSO.mjs';
|
|
4
|
-
import { CHECKPOINT_WORKER_HEARTBEAT_MS, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-
|
|
5
|
-
import '../../chunk-
|
|
4
|
+
import { CHECKPOINT_WORKER_HEARTBEAT_MS, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-KGVC6KJ3.mjs';
|
|
5
|
+
import '../../chunk-7EWQZQDT.mjs';
|
|
6
6
|
import '../../chunk-P36MUFEJ.mjs';
|
|
7
7
|
import '../../chunk-YH5OIL7B.mjs';
|
|
8
|
-
import '../../chunk-
|
|
9
|
-
import '../../chunk-
|
|
8
|
+
import '../../chunk-BCSWODXP.mjs';
|
|
9
|
+
import '../../chunk-5VVSHCXT.mjs';
|
|
10
10
|
import '../../chunk-N57XXGPG.mjs';
|
|
11
11
|
import '../../chunk-6PN3U6XW.mjs';
|
|
12
12
|
import '../../chunk-AV4IJKD2.mjs';
|
|
13
|
-
import '../../chunk-
|
|
14
|
-
import '../../chunk-
|
|
13
|
+
import '../../chunk-Q62NDKEC.mjs';
|
|
14
|
+
import '../../chunk-N53HN5VA.mjs';
|
|
15
15
|
import '../../chunk-FUUP45IS.mjs';
|
|
16
16
|
import '../../chunk-ZATRDAYU.mjs';
|
|
17
17
|
import '../../chunk-EBIHRL7B.mjs';
|
|
@@ -22,7 +22,7 @@ import '../../chunk-U2BKYEC7.mjs';
|
|
|
22
22
|
import '../../chunk-KOKM4PQN.mjs';
|
|
23
23
|
import '../../chunk-QSK35UC2.mjs';
|
|
24
24
|
import '../../chunk-VK45J47R.mjs';
|
|
25
|
-
import '../../chunk-
|
|
25
|
+
import '../../chunk-VFAXPDBH.mjs';
|
|
26
26
|
import '../../chunk-WQBCBKY3.mjs';
|
|
27
27
|
import '../../chunk-KCQG2PJT.mjs';
|
|
28
28
|
import '../../chunk-DG62H7OD.mjs';
|
|
@@ -37,8 +37,8 @@ import '../../chunk-7X5JST5L.mjs';
|
|
|
37
37
|
import '../../chunk-OFYKBKKV.mjs';
|
|
38
38
|
import '../../chunk-R6A773WK.mjs';
|
|
39
39
|
import '../../chunk-YTJCGKM4.mjs';
|
|
40
|
-
import '../../chunk-
|
|
41
|
-
import '../../chunk-
|
|
40
|
+
import '../../chunk-Z4DZ4KK2.mjs';
|
|
41
|
+
import '../../chunk-BCNUJVTT.mjs';
|
|
42
42
|
import '../../chunk-22YPZVMG.mjs';
|
|
43
43
|
import '../../chunk-RFBLQLLZ.mjs';
|
|
44
44
|
import '../../chunk-L5ORED7V.mjs';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { runRerankedQuery, rerankTextOf } from './chunk-
|
|
1
|
+
import { runRerankedQuery, rerankTextOf } from './chunk-OGFSUXYY.mjs';
|
|
2
2
|
import './chunk-KZ3NX2P7.mjs';
|
|
3
3
|
import { rerankOnHost } from './chunk-542TD4BL.mjs';
|
|
4
|
-
import './chunk-
|
|
4
|
+
import './chunk-ZR63OGT2.mjs';
|
|
5
5
|
import './chunk-HJSN7UYG.mjs';
|
|
6
6
|
import './chunk-LXWDQLTW.mjs';
|
|
7
7
|
import './chunk-E74YLAIG.mjs';
|
|
@@ -22,7 +22,7 @@ async function rerankClusterQuery(deps, indexName, params, firstStage, readDocum
|
|
|
22
22
|
host,
|
|
23
23
|
schema,
|
|
24
24
|
maxWindow: core.rerankSettings.maxWindow,
|
|
25
|
-
binding: host.queryBindingOf(params),
|
|
25
|
+
binding: () => host.queryBindingOf(params),
|
|
26
26
|
clampsPins: false,
|
|
27
27
|
fusesRankings: (request) => request.vector !== void 0 && (request.term !== void 0 || request.mode === "hybrid"),
|
|
28
28
|
firstStage,
|
|
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
18
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
19
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
20
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-57WF53MK.mjs');
|
|
22
22
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
23
|
});
|
|
24
24
|
const settled = restored.then(
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { runRerankedQuery, rerankTextOf } from './chunk-
|
|
1
|
+
import { runRerankedQuery, rerankTextOf } from './chunk-OGFSUXYY.mjs';
|
|
2
2
|
export { withRerankQueryText } from './chunk-KZ3NX2P7.mjs';
|
|
3
3
|
import { rerankOnHost } from './chunk-542TD4BL.mjs';
|
|
4
4
|
export { rerankOnHost } from './chunk-542TD4BL.mjs';
|
|
5
|
-
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-
|
|
5
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-ZR63OGT2.mjs';
|
|
6
6
|
import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
7
7
|
import './chunk-LXWDQLTW.mjs';
|
|
8
8
|
import { HYBRID_RESCORE_WINDOW } from './chunk-E74YLAIG.mjs';
|