@delali/narsil 0.3.1-next.113 → 0.3.1-next.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-DC2SZO4D.mjs → chunk-2TMWIATH.mjs} +107 -67
- package/dist/browser/{chunk-YZPOBLAS.mjs → chunk-73U5DWOZ.mjs} +4 -3
- package/dist/browser/{chunk-4FVSBSXY.mjs → chunk-FODCC3MK.mjs} +1 -1
- package/dist/browser/{chunk-3YPLBK4C.mjs → chunk-I45FAGOF.mjs} +2 -2
- package/dist/browser/{durability-wiring-2RG22WQJ.mjs → durability-wiring-CSMTKEYX.mjs} +3 -3
- package/dist/browser/{module-5EWHBKHZ.mjs → module-G2PPZV3B.mjs} +27 -24
- package/dist/browser/{restore-queue-WCFUB4NS.mjs → restore-queue-N7N42PBS.mjs} +1 -1
- package/dist/browser/{runtime-FL5K4JOH.mjs → runtime-BX6XRFYR.mjs} +2 -2
- package/dist/browser/{snapshot-BE3F2PNC.mjs → snapshot-EBH6RXQO.mjs} +4 -4
- package/dist/browser/{write-MV7K7OCU.mjs → write-5ZKFINX3.mjs} +2 -2
- package/dist/{chunk-H2GNWBLB.mjs → chunk-4ZKBKPBV.mjs} +1 -1
- package/dist/{chunk-GLZI636G.mjs → chunk-5VVSHCXT.mjs} +27 -12
- package/dist/{chunk-LVNBY2TS.mjs → chunk-7EWQZQDT.mjs} +1 -1
- package/dist/{chunk-4PFOOQFQ.mjs → chunk-BCNUJVTT.mjs} +1 -1
- package/dist/{chunk-PABYSTKO.mjs → chunk-BCSWODXP.mjs} +71 -54
- package/dist/{thread-S4SUCV4Y.mjs → chunk-DEXID4ZT.mjs} +300 -202
- package/dist/{chunk-4XWIW3F3.mjs → chunk-HC5NURS6.mjs} +4 -3
- package/dist/{chunk-AOLD3LPV.mjs → chunk-IE43SRL3.mjs} +4 -1
- package/dist/{chunk-27ZEY73B.mjs → chunk-KGVC6KJ3.mjs} +4 -4
- package/dist/{chunk-QGVMEMCG.mjs → chunk-KVLX65KD.mjs} +4 -4
- package/dist/{chunk-5WEBVYOA.mjs → chunk-N53HN5VA.mjs} +16 -9
- package/dist/chunk-N6J6EJW3.mjs +198 -0
- package/dist/{chunk-CU4OB64V.mjs → chunk-NX7KU77V.mjs} +1 -1
- package/dist/{chunk-QIFOXLWF.mjs → chunk-OGFSUXYY.mjs} +2 -2
- package/dist/{chunk-5QIRLJH4.mjs → chunk-Q62NDKEC.mjs} +4 -3
- package/dist/{chunk-DZXGNLK5.mjs → chunk-UDZYIWDP.mjs} +3 -3
- package/dist/{chunk-HNAP2KTG.mjs → chunk-VFAXPDBH.mjs} +1 -0
- package/dist/{chunk-OM5G2J6X.mjs → chunk-VOKEYJHU.mjs} +1 -1
- package/dist/{chunk-OABTAUCW.mjs → chunk-YCFQ5754.mjs} +1 -1
- package/dist/{chunk-JUAFWJON.mjs → chunk-YCSRB3IN.mjs} +3 -3
- package/dist/{chunk-CQUMNIIG.mjs → chunk-Z4DZ4KK2.mjs} +1 -0
- package/dist/{chunk-2RAQKITH.mjs → chunk-ZR63OGT2.mjs} +1 -1
- package/dist/{direct-executor-R4HOSVPE.mjs → direct-executor-OUY4RPTU.mjs} +4 -4
- package/dist/distribution.mjs +17 -17
- package/dist/{durability-wiring-XZI6355I.mjs → durability-wiring-5V6DVHFK.mjs} +13 -13
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +7 -7
- package/dist/long-query-thread-HMQM2FWL.mjs +118 -0
- package/dist/{module-WVKSJL74.mjs → module-3IMTXNPW.mjs} +30 -27
- package/dist/pattern.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +10 -10
- package/dist/{rerank-3POMPQNX.mjs → rerank-NYX5S7OG.mjs} +3 -3
- package/dist/{restore-queue-JAZAB5IQ.mjs → restore-queue-J6PCG4ON.mjs} +1 -1
- package/dist/{runtime-C2435MPK.mjs → runtime-XNYW6AWX.mjs} +2 -2
- package/dist/server.mjs +93 -199
- package/dist/{snapshot-PDUGHQ25.mjs → snapshot-57WF53MK.mjs} +14 -14
- package/dist/thread-FDEHU5G3.mjs +31 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +17 -5
- package/dist/{write-KVJ7YJJ6.mjs → write-YEI7A5OR.mjs} +8 -8
- package/package.json +7 -7
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-
|
|
3
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-73U5DWOZ.mjs';
|
|
3
|
+
export { releaseLocksHeldBy } from './chunk-73U5DWOZ.mjs';
|
|
4
4
|
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
5
5
|
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-2CBQ2PCC.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-LJS6PREA.mjs';
|
|
@@ -173,12 +173,13 @@ function readStoreFlags(store) {
|
|
|
173
173
|
}
|
|
174
174
|
|
|
175
175
|
// src/vector/shared-field/searcher.ts
|
|
176
|
-
function distancesInTypeScript(store, query, metric, ordinals) {
|
|
176
|
+
function distancesInTypeScript(store, query, metric, ordinals, clock) {
|
|
177
177
|
noteFallback("score");
|
|
178
178
|
const arenaQuery = store.prepareQueryArena(query);
|
|
179
179
|
const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
|
|
180
180
|
const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
|
|
181
181
|
for (let i = 0; i < ordinals.length; i++) {
|
|
182
|
+
if (clock !== void 0) clock.tick(query.length);
|
|
182
183
|
if (arenaQuery) {
|
|
183
184
|
distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
|
|
184
185
|
continue;
|
|
@@ -188,10 +189,12 @@ function distancesInTypeScript(store, query, metric, ordinals) {
|
|
|
188
189
|
}
|
|
189
190
|
return distances;
|
|
190
191
|
}
|
|
191
|
-
function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
|
|
192
|
+
function scoreCandidates(store, candidates, query, k, metric, minSimilarity, clock) {
|
|
192
193
|
const { ordinals, docIds } = candidates;
|
|
193
194
|
if (ordinals.length === 0) return { results: [], matched: 0 };
|
|
194
|
-
|
|
195
|
+
clock?.check();
|
|
196
|
+
const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals, clock);
|
|
197
|
+
clock?.check();
|
|
195
198
|
const heap = createBoundedMaxHeap(highScoreFirst, k);
|
|
196
199
|
let matched = 0;
|
|
197
200
|
for (let i = 0; i < ordinals.length; i++) {
|
|
@@ -262,6 +265,7 @@ function createSharedVectorSearcher(options) {
|
|
|
262
265
|
}
|
|
263
266
|
function search2(query, k, searchOptions, filter) {
|
|
264
267
|
const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = searchOptions;
|
|
268
|
+
const clock = searchOptions.clock;
|
|
265
269
|
const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
|
|
266
270
|
const liveDocuments = flags.liveDocuments();
|
|
267
271
|
if (liveDocuments <= 0) return empty;
|
|
@@ -277,23 +281,27 @@ function createSharedVectorSearcher(options) {
|
|
|
277
281
|
const graph = view.graph;
|
|
278
282
|
if (graph === null) {
|
|
279
283
|
const ordinals = filter === void 0 ? flags.unplacedOrdinals() : ordinalFilterValues(filter);
|
|
280
|
-
return scanned(
|
|
284
|
+
return scanned(
|
|
285
|
+
scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity, clock)
|
|
286
|
+
);
|
|
281
287
|
}
|
|
282
288
|
if (filter !== void 0) {
|
|
283
289
|
const graphLiveSize = view.liveSize;
|
|
284
290
|
const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
|
|
285
291
|
if (selectivity < view.handles.filterThreshold) {
|
|
286
292
|
const candidates = candidatesOf(ordinalFilterValues(filter), filter);
|
|
287
|
-
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity));
|
|
293
|
+
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity, clock));
|
|
288
294
|
}
|
|
289
295
|
}
|
|
296
|
+
const unplaced = flags.unplacedOrdinals();
|
|
290
297
|
let resolvedEveryHit = true;
|
|
291
298
|
const graphResults = [];
|
|
292
299
|
for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, {
|
|
293
300
|
filter,
|
|
294
301
|
efSearch,
|
|
295
302
|
oversample,
|
|
296
|
-
rescoreWindow
|
|
303
|
+
rescoreWindow,
|
|
304
|
+
clock
|
|
297
305
|
})) {
|
|
298
306
|
const docId = view.docIdOf(hit.ord);
|
|
299
307
|
if (docId === void 0) {
|
|
@@ -302,9 +310,8 @@ function createSharedVectorSearcher(options) {
|
|
|
302
310
|
}
|
|
303
311
|
graphResults.push({ docId, score: hit.score });
|
|
304
312
|
}
|
|
305
|
-
const unplaced = flags.unplacedOrdinals();
|
|
306
313
|
if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
|
|
307
|
-
const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity);
|
|
314
|
+
const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity, clock);
|
|
308
315
|
return fetched(mergeResults(graphResults, scan.results, k), resolvedEveryHit);
|
|
309
316
|
}
|
|
310
317
|
return {
|
|
@@ -2824,8 +2831,8 @@ function estimateMemoryBytes(state) {
|
|
|
2824
2831
|
}
|
|
2825
2832
|
|
|
2826
2833
|
// src/vector/vector-index/search.ts
|
|
2827
|
-
function* bufferCandidates(state, filter) {
|
|
2828
|
-
for (const docId of
|
|
2834
|
+
function* bufferCandidates(state, buffered, filter) {
|
|
2835
|
+
for (const docId of buffered) {
|
|
2829
2836
|
if (state.tombstones.has(docId)) continue;
|
|
2830
2837
|
if (filter) {
|
|
2831
2838
|
const ordinal = state.store.getOrdinal(docId);
|
|
@@ -2950,18 +2957,13 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
2950
2957
|
}
|
|
2951
2958
|
}
|
|
2952
2959
|
const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow }).map((r) => ({ docId: r.docId, score: r.score }));
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
withBufferedVectors(state, query, k, options, filter, hnswResults),
|
|
2956
|
-
minSimilarity,
|
|
2957
|
-
filter
|
|
2958
|
-
);
|
|
2960
|
+
const results = state.buffer.size === 0 ? hnswResults : withBufferedVectors(state, query, k, options, filter, hnswResults, state.buffer);
|
|
2961
|
+
return fetchedOutcome(state, results, minSimilarity, filter);
|
|
2959
2962
|
}
|
|
2960
|
-
function withBufferedVectors(state, query, k, options, filter, graphResults) {
|
|
2961
|
-
|
|
2962
|
-
const
|
|
2963
|
-
|
|
2964
|
-
return mergeResults(graphResults, buffered.results, k);
|
|
2963
|
+
function withBufferedVectors(state, query, k, options, filter, graphResults, buffered) {
|
|
2964
|
+
const candidates = bufferCandidates(state, buffered, filter);
|
|
2965
|
+
const scan = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
|
|
2966
|
+
return mergeResults(graphResults, scan.results, k);
|
|
2965
2967
|
}
|
|
2966
2968
|
|
|
2967
2969
|
// src/vector/vector-index/index.ts
|
|
@@ -3117,6 +3119,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3117
3119
|
}
|
|
3118
3120
|
}
|
|
3119
3121
|
const filterRevision = state.revision;
|
|
3122
|
+
const buffered = state.buffer.size === 0 ? null : [...state.buffer];
|
|
3120
3123
|
scheduleWorkerCopyLoad(state);
|
|
3121
3124
|
const viaWorkerCopy = await searchViaWorkerCopies(state, query, k, options.metric, options.minSimilarity, {
|
|
3122
3125
|
filter,
|
|
@@ -3128,7 +3131,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3128
3131
|
filter = filterForOptions(state, options);
|
|
3129
3132
|
}
|
|
3130
3133
|
if (viaWorkerCopy !== null) {
|
|
3131
|
-
const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
|
|
3134
|
+
const results = buffered === null ? viaWorkerCopy : withBufferedVectors(state, query, k, options, filter, viaWorkerCopy, buffered);
|
|
3132
3135
|
return fetchedOutcome(state, results, options.minSimilarity, filter);
|
|
3133
3136
|
}
|
|
3134
3137
|
return searchWithFilter(state, query, k, options, filter);
|
|
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
18
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
19
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
20
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-EBH6RXQO.mjs');
|
|
22
22
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
23
|
});
|
|
24
24
|
const settled = restored.then(
|
|
@@ -49,7 +49,7 @@ function pageByPosition(placed, organic, { start, limit, pinnedIds, binding, enc
|
|
|
49
49
|
sortKey: null,
|
|
50
50
|
sortSignature: null,
|
|
51
51
|
depth,
|
|
52
|
-
binding
|
|
52
|
+
binding: binding()
|
|
53
53
|
});
|
|
54
54
|
return { page, cursor };
|
|
55
55
|
}
|
|
@@ -229,7 +229,7 @@ function rerankTextOf(values) {
|
|
|
229
229
|
// src/rerank/pipeline.ts
|
|
230
230
|
function rerankCursorDepth(cursor, binding, host) {
|
|
231
231
|
const decoded = host.decodePageCursor(cursor);
|
|
232
|
-
host.requireMatchingCursor(decoded, cursor, null, true, binding);
|
|
232
|
+
host.requireMatchingCursor(decoded, cursor, null, true, binding());
|
|
233
233
|
return decoded.depth ?? 0;
|
|
234
234
|
}
|
|
235
235
|
function requireWithinReach(cursorDepth, offset, limit) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { collectAndReturnPages, vectorsPerFile } from './chunk-
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-
|
|
1
|
+
import { collectAndReturnPages, vectorsPerFile } from './chunk-73U5DWOZ.mjs';
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-I45FAGOF.mjs';
|
|
3
|
+
import './chunk-FODCC3MK.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-2TMWIATH.mjs';
|
|
5
5
|
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
6
6
|
import { MAX_FILE_BYTES } from './chunk-2CBQ2PCC.mjs';
|
|
7
7
|
import './chunk-JPTQZWJQ.mjs';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-FODCC3MK.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-2TMWIATH.mjs';
|
|
3
3
|
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
4
4
|
import './chunk-2CBQ2PCC.mjs';
|
|
5
5
|
import './chunk-JPTQZWJQ.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, grownByteLength, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-
|
|
1
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, grownByteLength, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-HC5NURS6.mjs';
|
|
2
2
|
import { MAX_VECTOR_ORDINALS } from './chunk-QSK35UC2.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-OBWUWZRA.mjs';
|
|
4
4
|
import { readSync, closeSync, openSync } from 'fs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex } from './chunk-
|
|
1
|
+
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex } from './chunk-N53HN5VA.mjs';
|
|
2
2
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-LXWDQLTW.mjs';
|
|
3
3
|
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QSK35UC2.mjs';
|
|
4
4
|
import { flattenSchema, partialFieldsOf } from './chunk-DG62H7OD.mjs';
|
|
@@ -63,6 +63,7 @@ function writeEncodedString(stream, value) {
|
|
|
63
63
|
stream.length += encoded.length;
|
|
64
64
|
}
|
|
65
65
|
function writeString(stream, value) {
|
|
66
|
+
if (stream.clock !== void 0) stream.clock.tick(value.length);
|
|
66
67
|
ensureCapacity(stream, LENGTH_BYTES + value.length);
|
|
67
68
|
const start = stream.length + LENGTH_BYTES;
|
|
68
69
|
const bytes = stream.bytes;
|
|
@@ -83,6 +84,7 @@ var NumberList = class {
|
|
|
83
84
|
}
|
|
84
85
|
};
|
|
85
86
|
function writeNumberList(stream, list) {
|
|
87
|
+
if (stream.clock !== void 0) stream.clock.tick(list.members.length);
|
|
86
88
|
writeByte(stream, LIST_TAG);
|
|
87
89
|
writeUint32(stream, list.members.length);
|
|
88
90
|
flush(stream);
|
|
@@ -154,15 +156,15 @@ function boundRerankOf(rerank) {
|
|
|
154
156
|
}
|
|
155
157
|
var FILTERS_SLOT = 2;
|
|
156
158
|
var BOUND_SLOT_COUNT = 16;
|
|
157
|
-
function bindingOf(values) {
|
|
158
|
-
const stream = { bytes: new Uint8Array(256), length: 0, hash: FNV_OFFSET_BASIS };
|
|
159
|
+
function bindingOf(values, clock) {
|
|
160
|
+
const stream = { bytes: new Uint8Array(256), length: 0, hash: FNV_OFFSET_BASIS, clock };
|
|
159
161
|
for (const value of values) {
|
|
160
162
|
writeValue(stream, value);
|
|
161
163
|
}
|
|
162
164
|
flush(stream);
|
|
163
165
|
return (stream.hash >>> 0).toString(16).padStart(8, "0");
|
|
164
166
|
}
|
|
165
|
-
function queryBindingOf(params) {
|
|
167
|
+
function queryBindingOf(params, clock) {
|
|
166
168
|
const values = [
|
|
167
169
|
params.term,
|
|
168
170
|
params.fields,
|
|
@@ -182,7 +184,14 @@ function queryBindingOf(params) {
|
|
|
182
184
|
params.scoring ?? "local"
|
|
183
185
|
];
|
|
184
186
|
if (params.rerank !== void 0) values.push(boundRerankOf(params.rerank));
|
|
185
|
-
return bindingOf(values);
|
|
187
|
+
return bindingOf(values, clock);
|
|
188
|
+
}
|
|
189
|
+
function lazyQueryBinding(params, clock) {
|
|
190
|
+
let binding;
|
|
191
|
+
return () => {
|
|
192
|
+
binding ??= queryBindingOf(params, clock);
|
|
193
|
+
return binding;
|
|
194
|
+
};
|
|
186
195
|
}
|
|
187
196
|
function listBindingOf(filters) {
|
|
188
197
|
const values = new Array(BOUND_SLOT_COUNT).fill(void 0);
|
|
@@ -262,7 +271,7 @@ function requireValidTypoTolerance(params) {
|
|
|
262
271
|
|
|
263
272
|
// src/engine/query/shared.ts
|
|
264
273
|
function withPatternWork(context) {
|
|
265
|
-
return { ...context, patternWork: createPatternWorkMeter(context.patternWorkCap) };
|
|
274
|
+
return { ...context, patternWork: createPatternWorkMeter(context.patternWorkCap, context.clock) };
|
|
266
275
|
}
|
|
267
276
|
function partitionsFor(manager, partitionIds) {
|
|
268
277
|
return partitionsIn(manager, partitionIds);
|
|
@@ -1181,6 +1190,7 @@ function filteredSearchOf(params, context) {
|
|
|
1181
1190
|
}
|
|
1182
1191
|
function searchField(vecIndex, vectorConfig, queryVector, k, context, filtered, rescoreWindow) {
|
|
1183
1192
|
const options = {
|
|
1193
|
+
clock: context.patternWork.clock,
|
|
1184
1194
|
metric: vectorConfig.metric ?? "cosine",
|
|
1185
1195
|
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1186
1196
|
efSearch: vectorConfig.efSearch,
|
|
@@ -1328,7 +1338,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
1328
1338
|
let anchorId = null;
|
|
1329
1339
|
if (params.searchAfter !== void 0) {
|
|
1330
1340
|
const decoded = decodePageCursor(params.searchAfter);
|
|
1331
|
-
requireMatchingCursor(decoded, params.searchAfter, sortSignature, true, context.cursorBinding);
|
|
1341
|
+
requireMatchingCursor(decoded, params.searchAfter, sortSignature, true, context.cursorBinding());
|
|
1332
1342
|
anchorKey = decoded.sortKey;
|
|
1333
1343
|
anchorId = decoded.anchor;
|
|
1334
1344
|
}
|
|
@@ -1345,8 +1355,11 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
1345
1355
|
const partitionFacets = [];
|
|
1346
1356
|
const partitionFacetConfig = params.facets !== void 0 && partitions.length > 1 ? everyValueFacetConfig(params.facets) : params.facets;
|
|
1347
1357
|
let count = 0;
|
|
1358
|
+
const clock = context.patternWork.clock;
|
|
1348
1359
|
for (const partition of partitions) {
|
|
1360
|
+
clock?.check();
|
|
1349
1361
|
const matches = fulltextMatches(partition, params, language, schema, options);
|
|
1362
|
+
clock?.check();
|
|
1350
1363
|
if (partitionFacetConfig !== void 0) {
|
|
1351
1364
|
partitionFacets.push(
|
|
1352
1365
|
partition.computeFacets(
|
|
@@ -1358,6 +1371,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
1358
1371
|
}
|
|
1359
1372
|
if (matches === null || matches.count === 0) continue;
|
|
1360
1373
|
count += matches.count;
|
|
1374
|
+
clock?.check();
|
|
1361
1375
|
const page2 = partition.sortedPage({
|
|
1362
1376
|
fields,
|
|
1363
1377
|
directions,
|
|
@@ -1370,6 +1384,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
1370
1384
|
});
|
|
1371
1385
|
for (const entry of page2) merged.push(entry);
|
|
1372
1386
|
}
|
|
1387
|
+
clock?.check();
|
|
1373
1388
|
merged.sort((a, b) => compareComparableKeys(a.key, b.key, directions) || compareCodePoints(a.id, b.id));
|
|
1374
1389
|
const hasMore = merged.length > offset + limit;
|
|
1375
1390
|
const page = merged.slice(offset, offset + limit);
|
|
@@ -1381,7 +1396,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
1381
1396
|
score: null,
|
|
1382
1397
|
sortKey: last.key,
|
|
1383
1398
|
sortSignature,
|
|
1384
|
-
binding: context.cursorBinding
|
|
1399
|
+
binding: context.cursorBinding()
|
|
1385
1400
|
});
|
|
1386
1401
|
}
|
|
1387
1402
|
return {
|
|
@@ -1396,18 +1411,18 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
1396
1411
|
function requireCursorDepth(cursor, signature, binding) {
|
|
1397
1412
|
if (cursor === void 0) return 0;
|
|
1398
1413
|
const decoded = decodePageCursor(cursor);
|
|
1399
|
-
requireMatchingCursor(decoded, cursor, signature, true, binding);
|
|
1414
|
+
requireMatchingCursor(decoded, cursor, signature, true, binding());
|
|
1400
1415
|
return decoded.depth ?? 0;
|
|
1401
1416
|
}
|
|
1402
1417
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
1403
1418
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
1404
|
-
return import('./runtime-
|
|
1419
|
+
return import('./runtime-XNYW6AWX.mjs').then(
|
|
1405
1420
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
1406
1421
|
);
|
|
1407
1422
|
}
|
|
1408
1423
|
function executeQuery(request, callerContext) {
|
|
1409
1424
|
if (request.rerank === void 0) return runQuery(request, callerContext);
|
|
1410
|
-
return import('./runtime-
|
|
1425
|
+
return import('./runtime-XNYW6AWX.mjs').then(
|
|
1411
1426
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
1412
1427
|
);
|
|
1413
1428
|
}
|
|
@@ -1702,4 +1717,4 @@ function pickDisplaySurface(surfaceOccurrences) {
|
|
|
1702
1717
|
return best;
|
|
1703
1718
|
}
|
|
1704
1719
|
|
|
1705
|
-
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, requireSurfaceFormsForPartialFields, resolveHybridFusion, runtimeHost, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
|
|
1720
|
+
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, lazyQueryBinding, listBindingOf, now, queryBindingOf, requireSurfaceFormsForPartialFields, resolveHybridFusion, runtimeHost, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-BCSWODXP.mjs';
|
|
2
2
|
import { getNestedValue } from './chunk-KCQG2PJT.mjs';
|
|
3
3
|
import { requireValidFieldTypes } from './chunk-DG62H7OD.mjs';
|
|
4
4
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
@@ -14,7 +14,7 @@ function requireLoadedVectorCode() {
|
|
|
14
14
|
function loadVectorCode() {
|
|
15
15
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
16
16
|
if (loading === null) {
|
|
17
|
-
loading = import('./module-
|
|
17
|
+
loading = import('./module-3IMTXNPW.mjs').then(
|
|
18
18
|
(code) => {
|
|
19
19
|
loaded = code;
|
|
20
20
|
return code;
|