@delali/narsil 0.3.1-next.150 → 0.3.1-next.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-ZCJNYBJM.mjs → analysis-change-2OYM7ARW.mjs} +4 -4
- package/dist/{react-native/chunks/analysis-change-entry-BNRU6STO.mjs → analysis-change-entry-QD7YNDEE.mjs} +1 -1
- package/dist/browser/{analysis-change-M65AZQQG.mjs → analysis-change-2V2HZ2NY.mjs} +2 -2
- package/dist/{analysis-change-entry-YN3ELL6C.mjs → browser/analysis-change-entry-JL5U2N3Q.mjs} +1 -1
- package/dist/browser/{chunk-SRL47USS.mjs → chunk-2A2IPRC3.mjs} +1 -1
- package/dist/browser/chunk-3I27KXQD.mjs +34 -0
- package/dist/browser/{chunk-KAEVGLYN.mjs → chunk-55WYRLDZ.mjs} +15 -0
- package/dist/browser/{chunk-6OWX3OP2.mjs → chunk-7ZJTRUVW.mjs} +1 -1
- package/dist/browser/{chunk-QMNZFFD3.mjs → chunk-B3JM6Z74.mjs} +6 -2
- package/dist/browser/{chunk-BRWY3JCN.mjs → chunk-CSCKZBVJ.mjs} +4 -29
- package/dist/browser/{chunk-D6UFGBF5.mjs → chunk-GJKXW3SF.mjs} +43 -24
- package/dist/browser/{chunk-5V74FFZE.mjs → chunk-HT7J73QL.mjs} +1 -1
- package/dist/browser/{chunk-BVX2MEHA.mjs → chunk-KWYQ6KX4.mjs} +17 -17
- package/dist/browser/{chunk-2JABMB4S.mjs → chunk-PM4MUM5N.mjs} +2 -2
- package/dist/browser/{chunk-W643QV3L.mjs → chunk-QJCO3TJU.mjs} +8 -4
- package/dist/browser/{chunk-ODQP7P3H.mjs → chunk-TQ5UMFG5.mjs} +1 -1
- package/dist/browser/{chunk-4DYLCICG.mjs → chunk-VBX2TI5V.mjs} +2 -2
- package/dist/browser/{chunk-XILIT5MT.mjs → chunk-VD7JEUIR.mjs} +5 -5
- package/dist/browser/{cold-module-A5S6JFDN.mjs → cold-module-YECAK4MN.mjs} +18 -12
- package/dist/browser/{durability-wiring-W5AVV3RF.mjs → durability-wiring-LM7WFL6X.mjs} +61 -50
- package/dist/browser/{import-PTT63AGK.mjs → import-QOY6ZH6X.mjs} +3 -3
- package/dist/browser/{module-CEKA7MQH.mjs → module-KAMRAJLE.mjs} +6 -6
- package/dist/browser/{module-X5PGOE47.mjs → module-XQLEOYNM.mjs} +24 -23
- package/dist/{react-native/chunks/restore-queue-OI5EH7R3.mjs → browser/restore-queue-UQ6AAUVX.mjs} +2 -2
- package/dist/browser/{snapshot-DQ6A4PQP.mjs → snapshot-UGZVMP4H.mjs} +11 -10
- package/dist/browser/{write-MAGRAIEH.mjs → write-I5P5MW4O.mjs} +7 -7
- package/dist/{chunk-VQDZVWN3.mjs → chunk-2PNYKG2D.mjs} +16 -16
- package/dist/{chunk-U6NE6RCV.mjs → chunk-3E4RRF5J.mjs} +1 -1
- package/dist/{chunk-ZYKUD5NX.mjs → chunk-4QCSHLNM.mjs} +56 -6
- package/dist/{chunk-YKEHQUSZ.mjs → chunk-52QQWEBS.mjs} +2 -2
- package/dist/{chunk-KZ5GQE4Z.mjs → chunk-6RKSY3BX.mjs} +6 -6
- package/dist/{chunk-6C5EX2QR.mjs → chunk-7LDMWKP6.mjs} +1 -1
- package/dist/{chunk-ADBS5XRI.mjs → chunk-7TE2AJIH.mjs} +6 -6
- package/dist/{chunk-QN5WVV5K.mjs → chunk-CSHHU7PA.mjs} +5 -5
- package/dist/{chunk-ZAYWOHVG.mjs → chunk-DGRM4EZZ.mjs} +1 -1
- package/dist/{chunk-7N223GWV.mjs → chunk-DMNPYF54.mjs} +6 -2
- package/dist/{chunk-7HKWO4BJ.mjs → chunk-FDG4KXX5.mjs} +1 -1
- package/dist/{chunk-QEYF2SQ6.mjs → chunk-FNBNZCGP.mjs} +1 -1
- package/dist/{chunk-ZEKDO7YX.mjs → chunk-G5KHSM6E.mjs} +15 -0
- package/dist/{chunk-3BVGJ4JR.mjs → chunk-IJSFWGVW.mjs} +3 -28
- package/dist/{chunk-B7XIOAMK.mjs → chunk-J4BDAH2W.mjs} +1 -1
- package/dist/{chunk-CYYSZPVV.mjs → chunk-KQJVNPRM.mjs} +18 -19
- package/dist/chunk-MATMVH65.mjs +9 -0
- package/dist/{react-native/chunks/chunk-R4EEMYM6.mjs → chunk-NITCPDFL.mjs} +2 -2
- package/dist/{chunk-U2726TL5.mjs → chunk-NPXVQS33.mjs} +6 -2
- package/dist/{chunk-DOY6EG76.mjs → chunk-RBEHLIKX.mjs} +2 -2
- package/dist/{chunk-F22IOCR2.mjs → chunk-SEAJS664.mjs} +1 -1
- package/dist/{chunk-O4KUNDVF.mjs → chunk-UGVUW27S.mjs} +3 -3
- package/dist/chunk-VD7J552I.mjs +28 -0
- package/dist/{chunk-L7TIAYPX.mjs → chunk-ZHAXP7SD.mjs} +7 -7
- package/dist/{chunk-XKPNKR3V.mjs → chunk-ZS6CMXAE.mjs} +31 -10
- package/dist/{cold-module-2WWGBFJC.mjs → cold-module-PYSSBIL7.mjs} +18 -12
- package/dist/{direct-executor-QLVHMELJ.mjs → direct-executor-WYALQSMD.mjs} +2 -2
- package/dist/distribution.mjs +23 -21
- package/dist/{durability-wiring-GNWQSZI3.mjs → durability-wiring-7VBZEJLG.mjs} +19 -45
- package/dist/{import-DX55PN4G.mjs → import-GJOMB2HC.mjs} +4 -4
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +3 -3
- package/dist/index.react-native.mjs +1 -1
- package/dist/{module-V5K4ZP23.mjs → module-6RLIKSZQ.mjs} +8 -8
- package/dist/{module-HW6IXUM6.mjs → module-AVQEIB7O.mjs} +23 -21
- package/dist/{paged-partition-archive-FDIIWZNR.mjs → paged-partition-archive-KGRW366Y.mjs} +11 -10
- package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
- package/dist/react-native/chunks/{analysis-change-HGW3L6NY.mjs → analysis-change-DXPZRGWF.mjs} +2 -2
- package/dist/{browser/analysis-change-entry-H3YIOLXZ.mjs → react-native/chunks/analysis-change-entry-CXNRW324.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-5FV4ZEVA.mjs → chunk-7IQR3SZO.mjs} +43 -24
- package/dist/react-native/chunks/{chunk-7VWQLEFV.mjs → chunk-CLE3LZ77.mjs} +6 -2
- package/dist/react-native/chunks/{chunk-SLGIAD6B.mjs → chunk-D7LFYPYI.mjs} +4 -4
- package/dist/{chunk-KZQIVGW7.mjs → react-native/chunks/chunk-DFQJ4IJB.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-B6OD63Z3.mjs → chunk-FS7BYE23.mjs} +8 -4
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +34 -0
- package/dist/react-native/chunks/{chunk-YJSMEKRE.mjs → chunk-IDFMNRIC.mjs} +17 -17
- package/dist/react-native/chunks/{chunk-FKGG4UOF.mjs → chunk-JYL35A4A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GV5KFQ6M.mjs → chunk-KLZGAOLT.mjs} +4 -29
- package/dist/react-native/chunks/{chunk-HURV3ASY.mjs → chunk-KOCTCUXW.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-KUJOBRKS.mjs → chunk-UCJU6WNH.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LQ6Q4L5B.mjs → chunk-VGUWIZFQ.mjs} +15 -0
- package/dist/react-native/chunks/{cold-module-TSFWLYWC.mjs → cold-module-KFGEWYEL.mjs} +16 -10
- package/dist/react-native/chunks/{durability-wiring-PZR6KOQK.mjs → durability-wiring-QKJB2QBT.mjs} +61 -50
- package/dist/react-native/chunks/{import-VTODLPLO.mjs → import-XFBCCENV.mjs} +2 -2
- package/dist/react-native/chunks/{module-VEK2KCKU.mjs → module-FHJ2GOBI.mjs} +4 -4
- package/dist/react-native/chunks/{module-JBBH6PZH.mjs → module-UJ5VUARS.mjs} +24 -23
- package/dist/{browser/restore-queue-DD4LWZK7.mjs → react-native/chunks/restore-queue-H5VCN453.mjs} +2 -2
- package/dist/react-native/chunks/{snapshot-AUZMEV36.mjs → snapshot-KVBD4LRC.mjs} +11 -10
- package/dist/react-native/chunks/{write-DDC2QBRJ.mjs → write-ES5A3WUA.mjs} +5 -5
- package/dist/{restore-queue-BQWWNNQI.mjs → restore-queue-AOF5OTQV.mjs} +2 -2
- package/dist/server.mjs +5 -5
- package/dist/{snapshot-EHXTOTW5.mjs → snapshot-WMX6JCLP.mjs} +18 -16
- package/dist/workers/entry.mjs +2 -2
- package/dist/{write-SDJXXQ2G.mjs → write-SKJ6HW7O.mjs} +7 -7
- package/package.json +7 -7
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-
|
|
1
|
+
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-6RKSY3BX.mjs';
|
|
2
2
|
import { gunzip, gzip } from './chunk-ZXNNR73R.mjs';
|
|
3
|
-
import { SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
3
|
+
import { SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-DMNPYF54.mjs';
|
|
4
|
+
import { defaultQuantizationFor } from './chunk-MATMVH65.mjs';
|
|
5
|
+
import { DEFAULT_VECTOR_METRIC } from './chunk-VD7J552I.mjs';
|
|
6
|
+
import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-G5KHSM6E.mjs';
|
|
7
|
+
import { readSegmentFile, STABLE_HEAD_ATTEMPTS } from './chunk-CSHHU7PA.mjs';
|
|
8
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestKey, segmentsPrefix, vectorFileKey, graphPartKey } from './chunk-7LDMWKP6.mjs';
|
|
9
|
+
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks } from './chunk-2PNYKG2D.mjs';
|
|
10
|
+
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-UGVUW27S.mjs';
|
|
9
11
|
import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-3HV7M5OJ.mjs';
|
|
10
12
|
import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-SOUZMPME.mjs';
|
|
11
13
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
@@ -15,6 +17,21 @@ import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
|
15
17
|
import { decode, encode } from '@msgpack/msgpack';
|
|
16
18
|
|
|
17
19
|
// src/engine/durability-metadata.ts
|
|
20
|
+
var VECTOR_TYPE = /^vector\[(\d+)\]$/;
|
|
21
|
+
function vectorFieldsOf(schema, promotion) {
|
|
22
|
+
const fields = {};
|
|
23
|
+
for (const [fieldPath, type] of Object.entries(schema)) {
|
|
24
|
+
const match = VECTOR_TYPE.exec(type);
|
|
25
|
+
if (match === null) continue;
|
|
26
|
+
const dimension = Number(match[1]);
|
|
27
|
+
fields[fieldPath] = {
|
|
28
|
+
dimension,
|
|
29
|
+
metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
|
|
30
|
+
quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
return Object.keys(fields).length === 0 ? null : fields;
|
|
34
|
+
}
|
|
18
35
|
function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
19
36
|
const metadata = {
|
|
20
37
|
indexName,
|
|
@@ -30,6 +47,10 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
30
47
|
engineVersion: VERSION
|
|
31
48
|
};
|
|
32
49
|
metadata.documentCount = documentCount ?? config.documentCount;
|
|
50
|
+
const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
|
|
51
|
+
if (vectorFields !== null) {
|
|
52
|
+
metadata.vectorFields = vectorFields;
|
|
53
|
+
}
|
|
33
54
|
if (config.embedding) {
|
|
34
55
|
metadata.embedding = config.embedding;
|
|
35
56
|
}
|
|
@@ -1420,7 +1441,7 @@ function removeFromPartition(manager, partitionId, docIds) {
|
|
|
1420
1441
|
}
|
|
1421
1442
|
}
|
|
1422
1443
|
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
1423
|
-
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-
|
|
1444
|
+
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-6RLIKSZQ.mjs');
|
|
1424
1445
|
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
1425
1446
|
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
1426
1447
|
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
@@ -1434,7 +1455,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
1434
1455
|
const { directory } = store;
|
|
1435
1456
|
for (const ref of partition.segments) {
|
|
1436
1457
|
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
1437
|
-
const { isPagedSegmentFile } = await import('./module-
|
|
1458
|
+
const { isPagedSegmentFile } = await import('./module-6RLIKSZQ.mjs');
|
|
1438
1459
|
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
1439
1460
|
await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
1440
1461
|
await yieldToEventLoop();
|
|
@@ -1490,7 +1511,7 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
1490
1511
|
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
1491
1512
|
);
|
|
1492
1513
|
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
1493
|
-
const { startReadingOpeningSectionsAhead } = await import('./module-
|
|
1514
|
+
const { startReadingOpeningSectionsAhead } = await import('./module-6RLIKSZQ.mjs');
|
|
1494
1515
|
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
1495
1516
|
}
|
|
1496
1517
|
for (const partition of manifest.partitions) {
|
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
import { readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile } from './chunk-
|
|
2
|
-
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-
|
|
1
|
+
import { readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile } from './chunk-7TE2AJIH.mjs';
|
|
2
|
+
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-7TE2AJIH.mjs';
|
|
3
3
|
import './chunk-3SBJVQLL.mjs';
|
|
4
4
|
import './chunk-NYSONUB7.mjs';
|
|
5
|
-
import { loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-
|
|
6
|
-
import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-
|
|
5
|
+
import { loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-DMNPYF54.mjs';
|
|
6
|
+
import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-G5KHSM6E.mjs';
|
|
7
7
|
import './chunk-FOCV4CBZ.mjs';
|
|
8
8
|
import { magnitude, defaultRescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, osqQuantize, createOsqScratch, scalarQueryProducts, osqEstimate, osqDistance, toDistance } from './chunk-AWNHDBKV.mjs';
|
|
9
9
|
export { createHNSWIndex, osqBitsOf } from './chunk-AWNHDBKV.mjs';
|
|
10
10
|
import { stageQueryLevels, readTrailer } from './chunk-37CD3OKS.mjs';
|
|
11
11
|
import './chunk-DJHHF2L5.mjs';
|
|
12
|
-
import { COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, COLD_WALK_NODES_READ_AHEAD } from './chunk-
|
|
13
|
-
import './chunk-
|
|
14
|
-
import './chunk-
|
|
15
|
-
import { yieldToEventLoop, HEADER_SIZE, crc32 } from './chunk-
|
|
12
|
+
import { COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, COLD_WALK_NODES_READ_AHEAD } from './chunk-CSHHU7PA.mjs';
|
|
13
|
+
import './chunk-NPXVQS33.mjs';
|
|
14
|
+
import './chunk-7LDMWKP6.mjs';
|
|
15
|
+
import { yieldToEventLoop, HEADER_SIZE, crc32, readHeader } from './chunk-2PNYKG2D.mjs';
|
|
16
16
|
import './chunk-6F6USYII.mjs';
|
|
17
|
-
import './chunk-
|
|
17
|
+
import './chunk-UGVUW27S.mjs';
|
|
18
18
|
import './chunk-3HV7M5OJ.mjs';
|
|
19
19
|
import './chunk-KAKQ74LS.mjs';
|
|
20
20
|
import './chunk-HJSN7UYG.mjs';
|
|
@@ -27,7 +27,7 @@ import './chunk-2HJWQPR6.mjs';
|
|
|
27
27
|
import './chunk-X5CV2E2B.mjs';
|
|
28
28
|
import './chunk-TGP5M5R7.mjs';
|
|
29
29
|
import './chunk-U2BKYEC7.mjs';
|
|
30
|
-
import './chunk-
|
|
30
|
+
import './chunk-DGRM4EZZ.mjs';
|
|
31
31
|
import './chunk-QHPYS427.mjs';
|
|
32
32
|
import './chunk-QYRNLLLI.mjs';
|
|
33
33
|
import './chunk-ZXNQEW43.mjs';
|
|
@@ -339,9 +339,15 @@ var BIN16_MAX_BYTES = 65535;
|
|
|
339
339
|
var VECTOR_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
340
340
|
var NEIGHBOUR_PAGE_SIZES = /* @__PURE__ */ new Set([RECORD_PAGE_BYTES]);
|
|
341
341
|
var LEVEL_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES]);
|
|
342
|
+
function holdsRawPayload(path) {
|
|
343
|
+
const header = new Uint8Array(HEADER_SIZE);
|
|
344
|
+
readFileRange(path, 0, header);
|
|
345
|
+
return !readHeader(header).flags.compressionEnabled;
|
|
346
|
+
}
|
|
342
347
|
function readHead(path, ref) {
|
|
343
348
|
const { headOffset, headLength, headChecksum } = ref;
|
|
344
349
|
if (headOffset === void 0 || headLength === void 0 || headChecksum === void 0) return null;
|
|
350
|
+
if (!holdsRawPayload(path)) return null;
|
|
345
351
|
const head = new Uint8Array(headLength);
|
|
346
352
|
readFileRange(path, HEADER_SIZE + headOffset, head);
|
|
347
353
|
if (crc32(head) !== headChecksum) {
|
|
@@ -695,11 +701,11 @@ function openColdVectorField(input) {
|
|
|
695
701
|
var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
696
702
|
var FLOAT32_BYTES = 4;
|
|
697
703
|
var FLOAT64_BYTES2 = 8;
|
|
698
|
-
function waitBeforeColdLoad(
|
|
704
|
+
function waitBeforeColdLoad(loadingWanted) {
|
|
699
705
|
return new Promise((resolve) => {
|
|
700
706
|
const timer = setTimeout(resolve, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS);
|
|
701
707
|
if (typeof timer.unref === "function") timer.unref();
|
|
702
|
-
void
|
|
708
|
+
void loadingWanted.then(() => {
|
|
703
709
|
clearTimeout(timer);
|
|
704
710
|
resolve();
|
|
705
711
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { createDirectExecutor } from './chunk-
|
|
1
|
+
export { createDirectExecutor } from './chunk-UGVUW27S.mjs';
|
|
2
2
|
import './chunk-3HV7M5OJ.mjs';
|
|
3
3
|
import './chunk-KAKQ74LS.mjs';
|
|
4
4
|
import './chunk-Q2UL2TPC.mjs';
|
|
@@ -6,7 +6,7 @@ import './chunk-2HJWQPR6.mjs';
|
|
|
6
6
|
import './chunk-X5CV2E2B.mjs';
|
|
7
7
|
import './chunk-TGP5M5R7.mjs';
|
|
8
8
|
import './chunk-U2BKYEC7.mjs';
|
|
9
|
-
import './chunk-
|
|
9
|
+
import './chunk-DGRM4EZZ.mjs';
|
|
10
10
|
import './chunk-ZXNQEW43.mjs';
|
|
11
11
|
import './chunk-LQG5G2LY.mjs';
|
|
12
12
|
import './chunk-U46IZNCW.mjs';
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,37 +1,39 @@
|
|
|
1
|
-
import { archiveStream, buildArchiveFromMemory, memoryCheckpointStore, readSnapshotArchive, writeArchiveFile, sourceIndexNameOf } from './chunk-
|
|
1
|
+
import { archiveStream, buildArchiveFromMemory, memoryCheckpointStore, readSnapshotArchive, writeArchiveFile, sourceIndexNameOf } from './chunk-ZHAXP7SD.mjs';
|
|
2
2
|
import { createByteReader } from './chunk-CZQ33DLW.mjs';
|
|
3
|
-
import { routableAllocation, resolvePartitionId, resolvePrimaryAssignment, resolveNodeTargets, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-
|
|
4
|
-
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, resolveAnalysisTarget, namedSettingsOf, holdsSettings } from './chunk-
|
|
3
|
+
import { routableAllocation, resolvePartitionId, resolvePrimaryAssignment, resolveNodeTargets, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-J4BDAH2W.mjs';
|
|
4
|
+
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, resolveAnalysisTarget, namedSettingsOf, holdsSettings } from './chunk-RBEHLIKX.mjs';
|
|
5
5
|
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, archiveFootprint, storedBytesOfSegments } from './chunk-VP6XRMWR.mjs';
|
|
6
6
|
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-T4CJDBBE.mjs';
|
|
7
|
-
import { closePartitionWriter, replaySyncedWal } from './chunk-
|
|
8
|
-
import { indexMetadataOf, loadPartitionSegments, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentManifest } from './chunk-
|
|
9
|
-
import { readVectorFile } from './chunk-
|
|
10
|
-
import { bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, backupKeys, validateBackupName, BACKUP_MANIFEST_READS_AT_ONCE, storedFilesOf, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, readRestorePlacement, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-
|
|
7
|
+
import { closePartitionWriter, replaySyncedWal } from './chunk-4QCSHLNM.mjs';
|
|
8
|
+
import { indexMetadataOf, loadPartitionSegments, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentManifest } from './chunk-ZS6CMXAE.mjs';
|
|
9
|
+
import { readVectorFile } from './chunk-6RKSY3BX.mjs';
|
|
10
|
+
import { bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, backupKeys, validateBackupName, BACKUP_MANIFEST_READS_AT_ONCE, storedFilesOf, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, readRestorePlacement, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-52QQWEBS.mjs';
|
|
11
11
|
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, refuseEncodeWithoutRoom, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, MAX_SOURCE_ID_LENGTH, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, ADMISSION_TIMEOUT_MS, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-LDGGR7YT.mjs';
|
|
12
12
|
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-LDGGR7YT.mjs';
|
|
13
|
-
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, indexConfigFromSettings, metadataWithBm25, indexConfigKey, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, isRecord2, decodeIndexSettings, HIDDEN_GENERATION_PREFIX, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, TEARDOWN_CAS_ATTEMPTS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-
|
|
13
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, indexConfigFromSettings, metadataWithBm25, indexConfigKey, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, isRecord2, decodeIndexSettings, HIDDEN_GENERATION_PREFIX, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, TEARDOWN_CAS_ATTEMPTS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-3E4RRF5J.mjs';
|
|
14
14
|
import { restoredConfigFields, restoredEmbedding } from './chunk-IRWP524D.mjs';
|
|
15
15
|
import './chunk-ZXNNR73R.mjs';
|
|
16
|
-
import './chunk-
|
|
16
|
+
import './chunk-DMNPYF54.mjs';
|
|
17
17
|
import { addBackgroundAdmission, workWaitingForMemory } from './chunk-PELHVTO7.mjs';
|
|
18
|
-
import './chunk-
|
|
19
|
-
import
|
|
18
|
+
import './chunk-IJSFWGVW.mjs';
|
|
19
|
+
import './chunk-MATMVH65.mjs';
|
|
20
|
+
import './chunk-VD7J552I.mjs';
|
|
21
|
+
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-G5KHSM6E.mjs';
|
|
20
22
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
21
23
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
22
24
|
import './chunk-AWNHDBKV.mjs';
|
|
23
25
|
import './chunk-37CD3OKS.mjs';
|
|
24
26
|
import './chunk-DJHHF2L5.mjs';
|
|
25
27
|
import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
|
|
26
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-
|
|
28
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-KQJVNPRM.mjs';
|
|
27
29
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
28
|
-
import './chunk-
|
|
29
|
-
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-
|
|
30
|
-
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest, manifestFileRefs } from './chunk-
|
|
30
|
+
import './chunk-CSHHU7PA.mjs';
|
|
31
|
+
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-NPXVQS33.mjs';
|
|
32
|
+
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest, manifestFileRefs } from './chunk-7LDMWKP6.mjs';
|
|
31
33
|
import { TransportErrorCodes, ClusterMessageTypes, ReplicationMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-5QMHMELB.mjs';
|
|
32
|
-
import { bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, applyStoredIndexBm25, createEngineIndex, encodeIndexConfiguration, memoryFractionsOf, createMemoryMonitor, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, openChecksummedEnvelope, storedArrayPositionGap, emptySerializablePartition, arrayPositionsAreStale, crc32Init, crc32Final, crc32Update, yieldToEventLoop, providedDocId, decodeIndexConfiguration, engineCoreOf } from './chunk-
|
|
34
|
+
import { bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, applyStoredIndexBm25, createEngineIndex, encodeIndexConfiguration, memoryFractionsOf, createMemoryMonitor, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, openChecksummedEnvelope, storedArrayPositionGap, emptySerializablePartition, arrayPositionsAreStale, crc32Init, crc32Final, crc32Update, yieldToEventLoop, providedDocId, decodeIndexConfiguration, engineCoreOf } from './chunk-2PNYKG2D.mjs';
|
|
33
35
|
import { runtimeHost, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, readRecentMemoryFigures, validateBM25Shape, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, now, listBindingOf, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, hitsKeptPerGroup, queryBindingOf, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-6F6USYII.mjs';
|
|
34
|
-
import { synonymModule, generateId } from './chunk-
|
|
36
|
+
import { synonymModule, generateId } from './chunk-UGVUW27S.mjs';
|
|
35
37
|
import { oversampledShardSize, bm25SetsValue, normalizeSort, sortModesOf, readSortValues, resolveProjection, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor, requireWithinResultWindow, holdsQuotedText, sortSignatureEntry, mergePartitionStats } from './chunk-3HV7M5OJ.mjs';
|
|
36
38
|
import { fnv1a, fnv1aUint32s } from './chunk-KAKQ74LS.mjs';
|
|
37
39
|
import './chunk-HJSN7UYG.mjs';
|
|
@@ -44,7 +46,7 @@ import { getLanguage, resolveIndexAnalysis, configOfNamedSettings } from './chun
|
|
|
44
46
|
import './chunk-X5CV2E2B.mjs';
|
|
45
47
|
import './chunk-TGP5M5R7.mjs';
|
|
46
48
|
import './chunk-U2BKYEC7.mjs';
|
|
47
|
-
import './chunk-
|
|
49
|
+
import './chunk-DGRM4EZZ.mjs';
|
|
48
50
|
import './chunk-QHPYS427.mjs';
|
|
49
51
|
import { collectAndReturnPages } from './chunk-QYRNLLLI.mjs';
|
|
50
52
|
import './chunk-ZXNQEW43.mjs';
|
|
@@ -13302,7 +13304,7 @@ async function openOnce(core, indexName, partitionId) {
|
|
|
13302
13304
|
if (directory === void 0) throw durabilityRequired(indexName, partitionId);
|
|
13303
13305
|
if ((checkpoint.manifest.analysisGeneration ?? 0) !== entry.analysisGeneration) return null;
|
|
13304
13306
|
const config = await encodeIndexConfiguration(entry);
|
|
13305
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
13307
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-KGRW366Y.mjs');
|
|
13306
13308
|
const archive = await openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager);
|
|
13307
13309
|
handedOver = true;
|
|
13308
13310
|
try {
|
|
@@ -13981,7 +13983,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
13981
13983
|
);
|
|
13982
13984
|
}
|
|
13983
13985
|
try {
|
|
13984
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
13986
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-KGRW366Y.mjs');
|
|
13985
13987
|
const archive = await openPagedPartitionArchive(checkpoint2, directory, indexName, partitionId, manager);
|
|
13986
13988
|
return {
|
|
13987
13989
|
stream: archiveStream({ ...archive, config, sourceIndexName: indexName }),
|
|
@@ -14190,7 +14192,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
14190
14192
|
if (directory !== null) {
|
|
14191
14193
|
const durable = core.durability?.manager;
|
|
14192
14194
|
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
14193
|
-
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-
|
|
14195
|
+
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-KGRW366Y.mjs');
|
|
14194
14196
|
await removeReceivedFilesNothingReads(
|
|
14195
14197
|
directory,
|
|
14196
14198
|
core.requireManager(indexName),
|
|
@@ -1,23 +1,25 @@
|
|
|
1
|
-
import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-
|
|
2
|
-
import { closePartitionWriter, drainIndexStateForUnload, listPersistedIndexes, removeLeftoverRewriteFiles, loadMetadata, snapshotCheckpointFor, replayWal } from './chunk-
|
|
3
|
-
import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, isRecord, invalidManifest, unpackEnvelopeBytes, isId, isUint32, isSequenceNumber, writeLiveVectors, growPartitionsTo, metadataConfigOf, durableMetadataConfigOf, readCurrentManifest } from './chunk-
|
|
4
|
-
import { VECTOR_FILE_ENVELOPE_VERSION } from './chunk-
|
|
1
|
+
import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-FNBNZCGP.mjs';
|
|
2
|
+
import { closePartitionWriter, drainIndexStateForUnload, listPersistedIndexes, removeLeftoverRewriteFiles, loadMetadata, snapshotCheckpointFor, replayWal, createMarkerWriter } from './chunk-4QCSHLNM.mjs';
|
|
3
|
+
import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, isRecord, invalidManifest, unpackEnvelopeBytes, isId, isUint32, isSequenceNumber, writeLiveVectors, growPartitionsTo, metadataConfigOf, durableMetadataConfigOf, readCurrentManifest } from './chunk-ZS6CMXAE.mjs';
|
|
4
|
+
import { VECTOR_FILE_ENVELOPE_VERSION } from './chunk-6RKSY3BX.mjs';
|
|
5
5
|
import './chunk-ZXNNR73R.mjs';
|
|
6
|
-
import './chunk-
|
|
7
|
-
import
|
|
6
|
+
import './chunk-DMNPYF54.mjs';
|
|
7
|
+
import './chunk-MATMVH65.mjs';
|
|
8
|
+
import './chunk-VD7J552I.mjs';
|
|
9
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-G5KHSM6E.mjs';
|
|
8
10
|
import './chunk-37CD3OKS.mjs';
|
|
9
|
-
import { createDurableDirectory } from './chunk-
|
|
11
|
+
import { createDurableDirectory } from './chunk-FDG4KXX5.mjs';
|
|
10
12
|
import { spawnNodeWorker } from './chunk-BEZH7VEH.mjs';
|
|
11
13
|
import { getFs } from './chunk-HLPRHOSO.mjs';
|
|
12
|
-
import { nextSegmentKey, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, commitCheckpointManifest, removeCheckpointGarbage, writeSegmentHeader,
|
|
14
|
+
import { nextSegmentKey, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, commitCheckpointManifest, removeCheckpointGarbage, writeSegmentHeader, SEGMENT_HEADER_SIZE, buildEntry, captureWholePartition, checkpointLastSeqNo, readCommitMarker } from './chunk-KQJVNPRM.mjs';
|
|
13
15
|
import './chunk-JDDXV6UF.mjs';
|
|
14
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, readSegmentFile, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-
|
|
15
|
-
import './chunk-
|
|
16
|
-
import { directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-
|
|
16
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, readSegmentFile, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-CSHHU7PA.mjs';
|
|
17
|
+
import './chunk-NPXVQS33.mjs';
|
|
18
|
+
import { directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-7LDMWKP6.mjs';
|
|
17
19
|
import './chunk-5QMHMELB.mjs';
|
|
18
|
-
import { toError, openChecksummedEnvelope, HEADER_SIZE } from './chunk-
|
|
20
|
+
import { toError, openChecksummedEnvelope, HEADER_SIZE } from './chunk-2PNYKG2D.mjs';
|
|
19
21
|
import './chunk-6F6USYII.mjs';
|
|
20
|
-
import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
22
|
+
import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-UGVUW27S.mjs';
|
|
21
23
|
import './chunk-3HV7M5OJ.mjs';
|
|
22
24
|
import { fnv1a } from './chunk-KAKQ74LS.mjs';
|
|
23
25
|
import './chunk-HJSN7UYG.mjs';
|
|
@@ -30,7 +32,7 @@ import './chunk-2HJWQPR6.mjs';
|
|
|
30
32
|
import './chunk-X5CV2E2B.mjs';
|
|
31
33
|
import './chunk-TGP5M5R7.mjs';
|
|
32
34
|
import './chunk-U2BKYEC7.mjs';
|
|
33
|
-
import './chunk-
|
|
35
|
+
import './chunk-DGRM4EZZ.mjs';
|
|
34
36
|
import { createRequestId } from './chunk-QHPYS427.mjs';
|
|
35
37
|
import './chunk-QYRNLLLI.mjs';
|
|
36
38
|
import './chunk-ZXNQEW43.mjs';
|
|
@@ -1115,7 +1117,7 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1115
1117
|
}
|
|
1116
1118
|
async function adoptWrittenSegments(input, manager, partitions, changes) {
|
|
1117
1119
|
const { indexName, indexState, hooks } = input;
|
|
1118
|
-
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-
|
|
1120
|
+
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-6RLIKSZQ.mjs');
|
|
1119
1121
|
const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
|
|
1120
1122
|
if (adoption === null) return [];
|
|
1121
1123
|
const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
@@ -1170,7 +1172,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1170
1172
|
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
1171
1173
|
if (!input.holdsCheckpointFiles()) {
|
|
1172
1174
|
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
|
|
1173
|
-
const stillRead = metadata.textStorage === "disk" ? (await import('./module-
|
|
1175
|
+
const stillRead = metadata.textStorage === "disk" ? (await import('./module-6RLIKSZQ.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
1174
1176
|
await removeCheckpointGarbage(
|
|
1175
1177
|
store,
|
|
1176
1178
|
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
|
|
@@ -1640,34 +1642,6 @@ function createGroupCommitCoordinator(sync) {
|
|
|
1640
1642
|
};
|
|
1641
1643
|
}
|
|
1642
1644
|
|
|
1643
|
-
// src/persistence/durability/marker-writer.ts
|
|
1644
|
-
function markerKey(indexName, partitionId) {
|
|
1645
|
-
return `${indexName}/wal/${partitionId}/commit`;
|
|
1646
|
-
}
|
|
1647
|
-
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
1648
|
-
const key = markerKey(indexName, partitionId);
|
|
1649
|
-
const handle = await directory.markerHandle(key);
|
|
1650
|
-
const existing = await handle.read();
|
|
1651
|
-
const current = readCommitMarker(existing);
|
|
1652
|
-
const createdFresh = existing.length === 0;
|
|
1653
|
-
let lastSlot = current?.slotIndex ?? null;
|
|
1654
|
-
let writeSeq = current?.state.writeSeq ?? 0;
|
|
1655
|
-
return {
|
|
1656
|
-
created: createdFresh,
|
|
1657
|
-
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
1658
|
-
async commit(state) {
|
|
1659
|
-
const slotIndex = nextMarkerSlot(lastSlot);
|
|
1660
|
-
writeSeq += 1;
|
|
1661
|
-
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
1662
|
-
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
1663
|
-
lastSlot = slotIndex;
|
|
1664
|
-
},
|
|
1665
|
-
async close() {
|
|
1666
|
-
await handle.close();
|
|
1667
|
-
}
|
|
1668
|
-
};
|
|
1669
|
-
}
|
|
1670
|
-
|
|
1671
1645
|
// src/persistence/durability/wal-writer.ts
|
|
1672
1646
|
var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
|
|
1673
1647
|
function segmentKey(indexName, partitionId, startSeqNo) {
|
|
@@ -2396,7 +2370,7 @@ var decoder = new TextDecoder();
|
|
|
2396
2370
|
async function countPagedPartition(store, partition) {
|
|
2397
2371
|
const { directory } = store;
|
|
2398
2372
|
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
2399
|
-
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-
|
|
2373
|
+
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-6RLIKSZQ.mjs');
|
|
2400
2374
|
for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
2401
2375
|
const scans = [];
|
|
2402
2376
|
try {
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport } from './chunk-
|
|
1
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport } from './chunk-SEAJS664.mjs';
|
|
2
2
|
import './chunk-NYSONUB7.mjs';
|
|
3
3
|
import './chunk-CZQ33DLW.mjs';
|
|
4
4
|
import './chunk-IRWP524D.mjs';
|
|
5
|
-
import './chunk-
|
|
5
|
+
import './chunk-2PNYKG2D.mjs';
|
|
6
6
|
import './chunk-6F6USYII.mjs';
|
|
7
|
-
import './chunk-
|
|
7
|
+
import './chunk-UGVUW27S.mjs';
|
|
8
8
|
import './chunk-3HV7M5OJ.mjs';
|
|
9
9
|
import './chunk-KAKQ74LS.mjs';
|
|
10
10
|
import './chunk-HJSN7UYG.mjs';
|
|
@@ -17,7 +17,7 @@ import './chunk-2HJWQPR6.mjs';
|
|
|
17
17
|
import './chunk-X5CV2E2B.mjs';
|
|
18
18
|
import './chunk-TGP5M5R7.mjs';
|
|
19
19
|
import './chunk-U2BKYEC7.mjs';
|
|
20
|
-
import './chunk-
|
|
20
|
+
import './chunk-DGRM4EZZ.mjs';
|
|
21
21
|
import './chunk-QHPYS427.mjs';
|
|
22
22
|
import './chunk-QYRNLLLI.mjs';
|
|
23
23
|
import './chunk-ZXNQEW43.mjs';
|
package/dist/index.browser.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './browser/chunk-
|
|
1
|
+
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './browser/chunk-KWYQ6KX4.mjs';
|
|
2
2
|
import './browser/chunk-MPK76GVS.mjs';
|
|
3
3
|
import './browser/chunk-JPTQZWJQ.mjs';
|
|
4
4
|
import './browser/chunk-26A5JFT2.mjs';
|
package/dist/index.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export { VERSION, createNarsil } from './chunk-
|
|
1
|
+
export { VERSION, createNarsil } from './chunk-2PNYKG2D.mjs';
|
|
2
2
|
import './chunk-6F6USYII.mjs';
|
|
3
|
-
export { registerSynonyms } from './chunk-
|
|
3
|
+
export { registerSynonyms } from './chunk-UGVUW27S.mjs';
|
|
4
4
|
export { registerPartialWordSearch, registerPatternSearch } from './chunk-3HV7M5OJ.mjs';
|
|
5
5
|
import './chunk-KAKQ74LS.mjs';
|
|
6
6
|
import './chunk-HJSN7UYG.mjs';
|
|
@@ -13,7 +13,7 @@ export { getLanguage, getStopWords, getTokenizer, hasStopWords, hasTokenizer, re
|
|
|
13
13
|
import './chunk-X5CV2E2B.mjs';
|
|
14
14
|
import './chunk-TGP5M5R7.mjs';
|
|
15
15
|
import './chunk-U2BKYEC7.mjs';
|
|
16
|
-
import './chunk-
|
|
16
|
+
import './chunk-DGRM4EZZ.mjs';
|
|
17
17
|
import './chunk-QHPYS427.mjs';
|
|
18
18
|
import './chunk-QYRNLLLI.mjs';
|
|
19
19
|
import './chunk-ZXNQEW43.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { VERSION, createNarsil, registerPartialWordSearch, registerPatternSearch, registerSynonyms } from './react-native/chunks/chunk-
|
|
1
|
+
export { VERSION, createNarsil, registerPartialWordSearch, registerPatternSearch, registerSynonyms } from './react-native/chunks/chunk-IDFMNRIC.mjs';
|
|
2
2
|
export { isSimdAvailable } from './react-native/chunks/chunk-LQAKO5QV.mjs';
|
|
3
3
|
import './react-native/chunks/chunk-NNE4VDMM.mjs';
|
|
4
4
|
import './react-native/chunks/chunk-L4M7QJNP.mjs';
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-
|
|
1
|
+
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-7TE2AJIH.mjs';
|
|
2
2
|
import './chunk-3SBJVQLL.mjs';
|
|
3
3
|
import './chunk-NYSONUB7.mjs';
|
|
4
|
-
import './chunk-
|
|
4
|
+
import './chunk-DMNPYF54.mjs';
|
|
5
5
|
import './chunk-FOCV4CBZ.mjs';
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-
|
|
8
|
-
import './chunk-
|
|
9
|
-
import './chunk-
|
|
6
|
+
import './chunk-CSHHU7PA.mjs';
|
|
7
|
+
import './chunk-NPXVQS33.mjs';
|
|
8
|
+
import './chunk-7LDMWKP6.mjs';
|
|
9
|
+
import './chunk-2PNYKG2D.mjs';
|
|
10
10
|
import './chunk-6F6USYII.mjs';
|
|
11
|
-
import './chunk-
|
|
11
|
+
import './chunk-UGVUW27S.mjs';
|
|
12
12
|
import './chunk-3HV7M5OJ.mjs';
|
|
13
13
|
import './chunk-KAKQ74LS.mjs';
|
|
14
14
|
import './chunk-HJSN7UYG.mjs';
|
|
@@ -21,7 +21,7 @@ import './chunk-2HJWQPR6.mjs';
|
|
|
21
21
|
import './chunk-X5CV2E2B.mjs';
|
|
22
22
|
import './chunk-TGP5M5R7.mjs';
|
|
23
23
|
import './chunk-U2BKYEC7.mjs';
|
|
24
|
-
import './chunk-
|
|
24
|
+
import './chunk-DGRM4EZZ.mjs';
|
|
25
25
|
import './chunk-QHPYS427.mjs';
|
|
26
26
|
import './chunk-QYRNLLLI.mjs';
|
|
27
27
|
import './chunk-ZXNQEW43.mjs';
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { noteWaitingForMemory, admitsBackgroundBytes } from './chunk-PELHVTO7.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED,
|
|
3
|
-
import {
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, noBuildHistory, recordCheckpoint, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, graphNeedsRebuild, yieldBuildTurn, threadsHoldCurrentLayout, emptyFieldBeforeRestore, recalibrateFromStore, signatureOf, allLiveDocIds, fieldHandlesOf, releaseReplacedVectors, adoptGraph, liveOrdinals, adoptLoadedCalibration, planCheckpoint, deserialize } from './chunk-IJSFWGVW.mjs';
|
|
3
|
+
import { defaultQuantizationFor } from './chunk-MATMVH65.mjs';
|
|
4
|
+
import { DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, DEFAULT_VECTOR_METRIC, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, DRIFT_CHECK_GROWTH_FACTOR, RECALIBRATION_DRIFT_RATIO, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-VD7J552I.mjs';
|
|
5
|
+
import { NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, isDead } from './chunk-G5KHSM6E.mjs';
|
|
4
6
|
import { createVectorStore } from './chunk-LJ4ENXOD.mjs';
|
|
5
7
|
export { openSharedVectorField } from './chunk-LJ4ENXOD.mjs';
|
|
6
8
|
import './chunk-FOCV4CBZ.mjs';
|
|
@@ -1896,11 +1898,11 @@ function beginCheckpointRestore(state, shape) {
|
|
|
1896
1898
|
}
|
|
1897
1899
|
|
|
1898
1900
|
// src/vector/vector-index/cold.ts
|
|
1899
|
-
function beginColdHold(state, field, loaded,
|
|
1901
|
+
function beginColdHold(state, field, loaded, loadWanted) {
|
|
1900
1902
|
const hold = {
|
|
1901
1903
|
field,
|
|
1902
1904
|
loaded,
|
|
1903
|
-
|
|
1905
|
+
loadWanted,
|
|
1904
1906
|
written: /* @__PURE__ */ new Map(),
|
|
1905
1907
|
removedNumbers: /* @__PURE__ */ new Set(),
|
|
1906
1908
|
removedIds: /* @__PURE__ */ new Set(),
|
|
@@ -1917,6 +1919,12 @@ function endColdHold(state) {
|
|
|
1917
1919
|
for (const write of hold.writes) write();
|
|
1918
1920
|
hold.field.close();
|
|
1919
1921
|
}
|
|
1922
|
+
async function untilColdLoaded(state) {
|
|
1923
|
+
while (state.cold !== null) {
|
|
1924
|
+
state.cold.loadWanted();
|
|
1925
|
+
await state.cold.loaded;
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1920
1928
|
function cancelColdHold(state) {
|
|
1921
1929
|
const hold = state.cold;
|
|
1922
1930
|
if (hold === null) return;
|
|
@@ -1931,15 +1939,15 @@ function openColdField(state, field, load) {
|
|
|
1931
1939
|
finished = resolve;
|
|
1932
1940
|
failed = reject;
|
|
1933
1941
|
});
|
|
1934
|
-
let
|
|
1935
|
-
const
|
|
1936
|
-
|
|
1942
|
+
let loadWanted = () => void 0;
|
|
1943
|
+
const loadingWanted = new Promise((resolve) => {
|
|
1944
|
+
loadWanted = resolve;
|
|
1937
1945
|
});
|
|
1938
|
-
const hold = beginColdHold(state, field, loaded,
|
|
1946
|
+
const hold = beginColdHold(state, field, loaded, loadWanted);
|
|
1939
1947
|
void load(
|
|
1940
1948
|
(shape) => beginCheckpointRestore(state, shape),
|
|
1941
1949
|
() => state.cold === hold,
|
|
1942
|
-
|
|
1950
|
+
loadingWanted
|
|
1943
1951
|
).then(
|
|
1944
1952
|
() => {
|
|
1945
1953
|
if (state.cold === hold) endColdHold(state);
|
|
@@ -2094,21 +2102,21 @@ function outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options) {
|
|
|
2094
2102
|
function coldSearch(state, hold, query, k, options) {
|
|
2095
2103
|
const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
|
|
2096
2104
|
if ("results" in walk) {
|
|
2097
|
-
hold.
|
|
2105
|
+
hold.loadWanted();
|
|
2098
2106
|
return walk;
|
|
2099
2107
|
}
|
|
2100
2108
|
const fromGraph = hold.field.search(query, k, options, walk.admits, hold.removedNumbers);
|
|
2101
|
-
hold.
|
|
2109
|
+
hold.loadWanted();
|
|
2102
2110
|
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
2103
2111
|
}
|
|
2104
2112
|
async function coldSearchConcurrently(state, hold, query, k, options) {
|
|
2105
2113
|
const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
|
|
2106
2114
|
if ("results" in walk) {
|
|
2107
|
-
hold.
|
|
2115
|
+
hold.loadWanted();
|
|
2108
2116
|
return walk;
|
|
2109
2117
|
}
|
|
2110
2118
|
const fromGraph = await hold.field.searchConcurrently(query, k, options, walk.admits, hold.removedNumbers);
|
|
2111
|
-
hold.
|
|
2119
|
+
hold.loadWanted();
|
|
2112
2120
|
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
2113
2121
|
}
|
|
2114
2122
|
|
|
@@ -2766,10 +2774,6 @@ function createVectorWrites(state) {
|
|
|
2766
2774
|
}
|
|
2767
2775
|
|
|
2768
2776
|
// src/vector/vector-index/index.ts
|
|
2769
|
-
function defaultQuantizationFor(dimension) {
|
|
2770
|
-
if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
|
|
2771
|
-
return "osq8";
|
|
2772
|
-
}
|
|
2773
2777
|
function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_WORKER_COPIES_ALLOWED, indexName = "", storage = "memory", onLoadedGraphRepaired = () => void 0) {
|
|
2774
2778
|
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
2775
2779
|
throw new NarsilError(
|
|
@@ -2782,7 +2786,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2782
2786
|
const filterThreshold = Math.max(0, Math.min(1, config?.filterThreshold ?? DEFAULT_FILTER_THRESHOLD));
|
|
2783
2787
|
const quantizationMode = config?.quantization ?? defaultQuantizationFor(dimension);
|
|
2784
2788
|
const codeBits = osqBitsOf(quantizationMode);
|
|
2785
|
-
const metric = config?.hnswConfig?.metric ??
|
|
2789
|
+
const metric = config?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC;
|
|
2786
2790
|
const rawHnswM = config?.hnswConfig?.m;
|
|
2787
2791
|
const hnswConfig = config?.hnswConfig ? { ...config.hnswConfig, m: rawHnswM !== void 0 ? Math.max(rawHnswM, 2) : void 0 } : void 0;
|
|
2788
2792
|
const dimensionScale = dimension / 256;
|
|
@@ -2852,9 +2856,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2852
2856
|
if (state.cold !== null) return coldHas(state.cold, docId);
|
|
2853
2857
|
return state.store.has(docId) && !state.tombstones.has(docId);
|
|
2854
2858
|
}
|
|
2855
|
-
|
|
2856
|
-
while (state.cold !== null) await state.cold.loaded;
|
|
2857
|
-
}
|
|
2859
|
+
const whenLoaded = () => untilColdLoaded(state);
|
|
2858
2860
|
async function searchParallel(query, k, options) {
|
|
2859
2861
|
if (state.graphsHeldOffThread !== null) await untilGraphReleased(state);
|
|
2860
2862
|
if (state.cold !== null) return coldSearchConcurrently(state, state.cold, query, k, options);
|