@delali/narsil 0.3.1-next.150 → 0.3.1-next.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-ZCJNYBJM.mjs → analysis-change-2OYM7ARW.mjs} +4 -4
- package/dist/{react-native/chunks/analysis-change-entry-BNRU6STO.mjs → analysis-change-entry-QD7YNDEE.mjs} +1 -1
- package/dist/browser/{analysis-change-M65AZQQG.mjs → analysis-change-2V2HZ2NY.mjs} +2 -2
- package/dist/{analysis-change-entry-YN3ELL6C.mjs → browser/analysis-change-entry-JL5U2N3Q.mjs} +1 -1
- package/dist/browser/{chunk-SRL47USS.mjs → chunk-2A2IPRC3.mjs} +1 -1
- package/dist/browser/chunk-3I27KXQD.mjs +34 -0
- package/dist/browser/{chunk-KAEVGLYN.mjs → chunk-55WYRLDZ.mjs} +15 -0
- package/dist/browser/{chunk-6OWX3OP2.mjs → chunk-7ZJTRUVW.mjs} +1 -1
- package/dist/browser/{chunk-QMNZFFD3.mjs → chunk-B3JM6Z74.mjs} +6 -2
- package/dist/browser/{chunk-BRWY3JCN.mjs → chunk-CSCKZBVJ.mjs} +4 -29
- package/dist/browser/{chunk-D6UFGBF5.mjs → chunk-GJKXW3SF.mjs} +43 -24
- package/dist/browser/{chunk-5V74FFZE.mjs → chunk-HT7J73QL.mjs} +1 -1
- package/dist/browser/{chunk-BVX2MEHA.mjs → chunk-KWYQ6KX4.mjs} +17 -17
- package/dist/browser/{chunk-2JABMB4S.mjs → chunk-PM4MUM5N.mjs} +2 -2
- package/dist/browser/{chunk-W643QV3L.mjs → chunk-QJCO3TJU.mjs} +8 -4
- package/dist/browser/{chunk-ODQP7P3H.mjs → chunk-TQ5UMFG5.mjs} +1 -1
- package/dist/browser/{chunk-4DYLCICG.mjs → chunk-VBX2TI5V.mjs} +2 -2
- package/dist/browser/{chunk-XILIT5MT.mjs → chunk-VD7JEUIR.mjs} +5 -5
- package/dist/browser/{cold-module-A5S6JFDN.mjs → cold-module-YECAK4MN.mjs} +18 -12
- package/dist/browser/{durability-wiring-W5AVV3RF.mjs → durability-wiring-LM7WFL6X.mjs} +61 -50
- package/dist/browser/{import-PTT63AGK.mjs → import-QOY6ZH6X.mjs} +3 -3
- package/dist/browser/{module-CEKA7MQH.mjs → module-KAMRAJLE.mjs} +6 -6
- package/dist/browser/{module-X5PGOE47.mjs → module-XQLEOYNM.mjs} +24 -23
- package/dist/{react-native/chunks/restore-queue-OI5EH7R3.mjs → browser/restore-queue-UQ6AAUVX.mjs} +2 -2
- package/dist/browser/{snapshot-DQ6A4PQP.mjs → snapshot-UGZVMP4H.mjs} +11 -10
- package/dist/browser/{write-MAGRAIEH.mjs → write-I5P5MW4O.mjs} +7 -7
- package/dist/{chunk-VQDZVWN3.mjs → chunk-2PNYKG2D.mjs} +16 -16
- package/dist/{chunk-U6NE6RCV.mjs → chunk-3E4RRF5J.mjs} +1 -1
- package/dist/{chunk-ZYKUD5NX.mjs → chunk-4QCSHLNM.mjs} +56 -6
- package/dist/{chunk-YKEHQUSZ.mjs → chunk-52QQWEBS.mjs} +2 -2
- package/dist/{chunk-KZ5GQE4Z.mjs → chunk-6RKSY3BX.mjs} +6 -6
- package/dist/{chunk-6C5EX2QR.mjs → chunk-7LDMWKP6.mjs} +1 -1
- package/dist/{chunk-ADBS5XRI.mjs → chunk-7TE2AJIH.mjs} +6 -6
- package/dist/{chunk-QN5WVV5K.mjs → chunk-CSHHU7PA.mjs} +5 -5
- package/dist/{chunk-ZAYWOHVG.mjs → chunk-DGRM4EZZ.mjs} +1 -1
- package/dist/{chunk-7N223GWV.mjs → chunk-DMNPYF54.mjs} +6 -2
- package/dist/{chunk-7HKWO4BJ.mjs → chunk-FDG4KXX5.mjs} +1 -1
- package/dist/{chunk-QEYF2SQ6.mjs → chunk-FNBNZCGP.mjs} +1 -1
- package/dist/{chunk-ZEKDO7YX.mjs → chunk-G5KHSM6E.mjs} +15 -0
- package/dist/{chunk-3BVGJ4JR.mjs → chunk-IJSFWGVW.mjs} +3 -28
- package/dist/{chunk-B7XIOAMK.mjs → chunk-J4BDAH2W.mjs} +1 -1
- package/dist/{chunk-CYYSZPVV.mjs → chunk-KQJVNPRM.mjs} +18 -19
- package/dist/chunk-MATMVH65.mjs +9 -0
- package/dist/{react-native/chunks/chunk-R4EEMYM6.mjs → chunk-NITCPDFL.mjs} +2 -2
- package/dist/{chunk-U2726TL5.mjs → chunk-NPXVQS33.mjs} +6 -2
- package/dist/{chunk-DOY6EG76.mjs → chunk-RBEHLIKX.mjs} +2 -2
- package/dist/{chunk-F22IOCR2.mjs → chunk-SEAJS664.mjs} +1 -1
- package/dist/{chunk-O4KUNDVF.mjs → chunk-UGVUW27S.mjs} +3 -3
- package/dist/chunk-VD7J552I.mjs +28 -0
- package/dist/{chunk-L7TIAYPX.mjs → chunk-ZHAXP7SD.mjs} +7 -7
- package/dist/{chunk-XKPNKR3V.mjs → chunk-ZS6CMXAE.mjs} +31 -10
- package/dist/{cold-module-2WWGBFJC.mjs → cold-module-PYSSBIL7.mjs} +18 -12
- package/dist/{direct-executor-QLVHMELJ.mjs → direct-executor-WYALQSMD.mjs} +2 -2
- package/dist/distribution.mjs +23 -21
- package/dist/{durability-wiring-GNWQSZI3.mjs → durability-wiring-7VBZEJLG.mjs} +19 -45
- package/dist/{import-DX55PN4G.mjs → import-GJOMB2HC.mjs} +4 -4
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +3 -3
- package/dist/index.react-native.mjs +1 -1
- package/dist/{module-V5K4ZP23.mjs → module-6RLIKSZQ.mjs} +8 -8
- package/dist/{module-HW6IXUM6.mjs → module-AVQEIB7O.mjs} +23 -21
- package/dist/{paged-partition-archive-FDIIWZNR.mjs → paged-partition-archive-KGRW366Y.mjs} +11 -10
- package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
- package/dist/react-native/chunks/{analysis-change-HGW3L6NY.mjs → analysis-change-DXPZRGWF.mjs} +2 -2
- package/dist/{browser/analysis-change-entry-H3YIOLXZ.mjs → react-native/chunks/analysis-change-entry-CXNRW324.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-5FV4ZEVA.mjs → chunk-7IQR3SZO.mjs} +43 -24
- package/dist/react-native/chunks/{chunk-7VWQLEFV.mjs → chunk-CLE3LZ77.mjs} +6 -2
- package/dist/react-native/chunks/{chunk-SLGIAD6B.mjs → chunk-D7LFYPYI.mjs} +4 -4
- package/dist/{chunk-KZQIVGW7.mjs → react-native/chunks/chunk-DFQJ4IJB.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-B6OD63Z3.mjs → chunk-FS7BYE23.mjs} +8 -4
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +34 -0
- package/dist/react-native/chunks/{chunk-YJSMEKRE.mjs → chunk-IDFMNRIC.mjs} +17 -17
- package/dist/react-native/chunks/{chunk-FKGG4UOF.mjs → chunk-JYL35A4A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GV5KFQ6M.mjs → chunk-KLZGAOLT.mjs} +4 -29
- package/dist/react-native/chunks/{chunk-HURV3ASY.mjs → chunk-KOCTCUXW.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-KUJOBRKS.mjs → chunk-UCJU6WNH.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LQ6Q4L5B.mjs → chunk-VGUWIZFQ.mjs} +15 -0
- package/dist/react-native/chunks/{cold-module-TSFWLYWC.mjs → cold-module-KFGEWYEL.mjs} +16 -10
- package/dist/react-native/chunks/{durability-wiring-PZR6KOQK.mjs → durability-wiring-QKJB2QBT.mjs} +61 -50
- package/dist/react-native/chunks/{import-VTODLPLO.mjs → import-XFBCCENV.mjs} +2 -2
- package/dist/react-native/chunks/{module-VEK2KCKU.mjs → module-FHJ2GOBI.mjs} +4 -4
- package/dist/react-native/chunks/{module-JBBH6PZH.mjs → module-UJ5VUARS.mjs} +24 -23
- package/dist/{browser/restore-queue-DD4LWZK7.mjs → react-native/chunks/restore-queue-H5VCN453.mjs} +2 -2
- package/dist/react-native/chunks/{snapshot-AUZMEV36.mjs → snapshot-KVBD4LRC.mjs} +11 -10
- package/dist/react-native/chunks/{write-DDC2QBRJ.mjs → write-ES5A3WUA.mjs} +5 -5
- package/dist/{restore-queue-BQWWNNQI.mjs → restore-queue-AOF5OTQV.mjs} +2 -2
- package/dist/server.mjs +5 -5
- package/dist/{snapshot-EHXTOTW5.mjs → snapshot-WMX6JCLP.mjs} +18 -16
- package/dist/workers/entry.mjs +2 -2
- package/dist/{write-SDJXXQ2G.mjs → write-SKJ6HW7O.mjs} +7 -7
- package/package.json +7 -7
|
@@ -1,18 +1,19 @@
|
|
|
1
|
-
import { resolvePartitionId } from './chunk-
|
|
2
|
-
import { readVectorFile, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-
|
|
3
|
-
import './chunk-
|
|
1
|
+
import { resolvePartitionId } from './chunk-J4BDAH2W.mjs';
|
|
2
|
+
import { readVectorFile, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-6RKSY3BX.mjs';
|
|
3
|
+
import './chunk-3E4RRF5J.mjs';
|
|
4
4
|
import './chunk-IRWP524D.mjs';
|
|
5
|
-
import { loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-
|
|
6
|
-
import { vectorsPerFile } from './chunk-
|
|
7
|
-
import
|
|
5
|
+
import { loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-DMNPYF54.mjs';
|
|
6
|
+
import { vectorsPerFile } from './chunk-IJSFWGVW.mjs';
|
|
7
|
+
import './chunk-VD7J552I.mjs';
|
|
8
|
+
import { isDead, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-G5KHSM6E.mjs';
|
|
8
9
|
import './chunk-AWNHDBKV.mjs';
|
|
9
10
|
import './chunk-37CD3OKS.mjs';
|
|
10
11
|
import './chunk-DJHHF2L5.mjs';
|
|
11
12
|
import { PARTITION_COPY_VECTOR_RUN_BYTES } from './chunk-JDDXV6UF.mjs';
|
|
12
|
-
import { segmentsPrefix, encodeSegmentManifest, directoryCheckpointStore, vectorFileKey } from './chunk-
|
|
13
|
-
import { HEADER_SIZE, crc32Init, crc32Final, engineVersionParts, writeHeader, RESERVED_SIZE, crc32Update } from './chunk-
|
|
13
|
+
import { segmentsPrefix, encodeSegmentManifest, directoryCheckpointStore, vectorFileKey } from './chunk-7LDMWKP6.mjs';
|
|
14
|
+
import { HEADER_SIZE, crc32Init, crc32Final, engineVersionParts, writeHeader, RESERVED_SIZE, crc32Update } from './chunk-2PNYKG2D.mjs';
|
|
14
15
|
import './chunk-6F6USYII.mjs';
|
|
15
|
-
import { generateId } from './chunk-
|
|
16
|
+
import { generateId } from './chunk-UGVUW27S.mjs';
|
|
16
17
|
import { isCompositePartition } from './chunk-3HV7M5OJ.mjs';
|
|
17
18
|
import './chunk-KAKQ74LS.mjs';
|
|
18
19
|
import './chunk-HJSN7UYG.mjs';
|
|
@@ -25,7 +26,7 @@ import './chunk-2HJWQPR6.mjs';
|
|
|
25
26
|
import './chunk-X5CV2E2B.mjs';
|
|
26
27
|
import './chunk-TGP5M5R7.mjs';
|
|
27
28
|
import './chunk-U2BKYEC7.mjs';
|
|
28
|
-
import './chunk-
|
|
29
|
+
import './chunk-DGRM4EZZ.mjs';
|
|
29
30
|
import './chunk-QHPYS427.mjs';
|
|
30
31
|
import './chunk-QYRNLLLI.mjs';
|
|
31
32
|
import './chunk-ZXNQEW43.mjs';
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { createDurableDirectory } from '../../chunk-
|
|
1
|
+
import { createDurableDirectory } from '../../chunk-FDG4KXX5.mjs';
|
|
2
2
|
import '../../chunk-BEZH7VEH.mjs';
|
|
3
3
|
import '../../chunk-HLPRHOSO.mjs';
|
|
4
|
-
import { writeCheckpointSegments } from '../../chunk-
|
|
4
|
+
import { writeCheckpointSegments } from '../../chunk-KQJVNPRM.mjs';
|
|
5
5
|
import '../../chunk-JDDXV6UF.mjs';
|
|
6
|
-
import { CHECKPOINT_WORKER_HEARTBEAT_MS } from '../../chunk-
|
|
7
|
-
import '../../chunk-
|
|
8
|
-
import { directoryCheckpointStore } from '../../chunk-
|
|
6
|
+
import { CHECKPOINT_WORKER_HEARTBEAT_MS } from '../../chunk-CSHHU7PA.mjs';
|
|
7
|
+
import '../../chunk-NPXVQS33.mjs';
|
|
8
|
+
import { directoryCheckpointStore } from '../../chunk-7LDMWKP6.mjs';
|
|
9
9
|
import '../../chunk-5QMHMELB.mjs';
|
|
10
|
-
import '../../chunk-
|
|
10
|
+
import '../../chunk-2PNYKG2D.mjs';
|
|
11
11
|
import '../../chunk-6F6USYII.mjs';
|
|
12
|
-
import '../../chunk-
|
|
12
|
+
import '../../chunk-UGVUW27S.mjs';
|
|
13
13
|
import '../../chunk-3HV7M5OJ.mjs';
|
|
14
14
|
import '../../chunk-KAKQ74LS.mjs';
|
|
15
15
|
import '../../chunk-HJSN7UYG.mjs';
|
|
@@ -22,7 +22,7 @@ import '../../chunk-2HJWQPR6.mjs';
|
|
|
22
22
|
import '../../chunk-X5CV2E2B.mjs';
|
|
23
23
|
import '../../chunk-TGP5M5R7.mjs';
|
|
24
24
|
import '../../chunk-U2BKYEC7.mjs';
|
|
25
|
-
import '../../chunk-
|
|
25
|
+
import '../../chunk-DGRM4EZZ.mjs';
|
|
26
26
|
import '../../chunk-QHPYS427.mjs';
|
|
27
27
|
import '../../chunk-QYRNLLLI.mjs';
|
|
28
28
|
import '../../chunk-ZXNQEW43.mjs';
|
package/dist/react-native/chunks/{analysis-change-HGW3L6NY.mjs → analysis-change-DXPZRGWF.mjs}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { requireAnalysisRebuildMemory } from './chunk-
|
|
1
|
+
import { requireAnalysisRebuildMemory } from './chunk-JYL35A4A.mjs';
|
|
2
2
|
import { targetOfPending, configUnderAnalysis } from './chunk-TLYD7IJ2.mjs';
|
|
3
|
-
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createPartitionIndex, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue, ARRAY_POSITION_GAP } from './chunk-
|
|
3
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createPartitionIndex, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue, ARRAY_POSITION_GAP } from './chunk-IDFMNRIC.mjs';
|
|
4
4
|
import './chunk-LQAKO5QV.mjs';
|
|
5
5
|
import './chunk-NNE4VDMM.mjs';
|
|
6
6
|
import './chunk-L4M7QJNP.mjs';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-
|
|
1
|
+
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-DFQJ4IJB.mjs';
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor } from './chunk-HJBJBH26.mjs';
|
|
2
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-VGUWIZFQ.mjs';
|
|
2
3
|
import { gunzip, gzip } from './chunk-LRGWK367.mjs';
|
|
3
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, encodeSegmentManifest, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, readListedFile, requireListedChecksum, vectorFileKey, DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, graphPartKey, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-
|
|
4
|
-
import { restoreVectorFields } from './chunk-
|
|
5
|
-
import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, openChecksummedEnvelope, isCompositePartition, crc32, reconstructSchemaFromMetadata, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, prepareDocumentVectors } from './chunk-
|
|
4
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, encodeSegmentManifest, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, readListedFile, requireListedChecksum, vectorFileKey, DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, graphPartKey, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-FS7BYE23.mjs';
|
|
5
|
+
import { restoreVectorFields } from './chunk-CLE3LZ77.mjs';
|
|
6
|
+
import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, openChecksummedEnvelope, isCompositePartition, crc32, reconstructSchemaFromMetadata, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, prepareDocumentVectors } from './chunk-IDFMNRIC.mjs';
|
|
6
7
|
import { flattenSchema, requireValidFieldTypes, extractVectorFieldsFromSchema, partialFieldsOf, isQuantizationMode, isStorageMode } from './chunk-PU55IOG4.mjs';
|
|
7
8
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-TJMSKBEM.mjs';
|
|
8
9
|
import { VECTOR_PATTERN } from './chunk-DKJTZ625.mjs';
|
|
@@ -12,6 +13,21 @@ import { NarsilError, ErrorCodes } from './chunk-YH7W556M.mjs';
|
|
|
12
13
|
import { decode, encode } from '@msgpack/msgpack';
|
|
13
14
|
|
|
14
15
|
// src/engine/durability-metadata.ts
|
|
16
|
+
var VECTOR_TYPE = /^vector\[(\d+)\]$/;
|
|
17
|
+
function vectorFieldsOf(schema, promotion) {
|
|
18
|
+
const fields = {};
|
|
19
|
+
for (const [fieldPath, type] of Object.entries(schema)) {
|
|
20
|
+
const match = VECTOR_TYPE.exec(type);
|
|
21
|
+
if (match === null) continue;
|
|
22
|
+
const dimension = Number(match[1]);
|
|
23
|
+
fields[fieldPath] = {
|
|
24
|
+
dimension,
|
|
25
|
+
metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
|
|
26
|
+
quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
return Object.keys(fields).length === 0 ? null : fields;
|
|
30
|
+
}
|
|
15
31
|
function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
16
32
|
const metadata = {
|
|
17
33
|
indexName,
|
|
@@ -27,6 +43,10 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
27
43
|
engineVersion: VERSION
|
|
28
44
|
};
|
|
29
45
|
metadata.documentCount = documentCount ?? config.documentCount;
|
|
46
|
+
const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
|
|
47
|
+
if (vectorFields !== null) {
|
|
48
|
+
metadata.vectorFields = vectorFields;
|
|
49
|
+
}
|
|
30
50
|
if (config.embedding) {
|
|
31
51
|
metadata.embedding = config.embedding;
|
|
32
52
|
}
|
|
@@ -1362,7 +1382,7 @@ function holdsEveryHead(ref) {
|
|
|
1362
1382
|
async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
|
|
1363
1383
|
const { pathOf } = store;
|
|
1364
1384
|
if (pathOf === void 0 || vectorIndex.storage !== "disk" || !holdsEveryHead(ref)) return false;
|
|
1365
|
-
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-
|
|
1385
|
+
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-KFGEWYEL.mjs');
|
|
1366
1386
|
await Promise.all([
|
|
1367
1387
|
...ref.files.map((file) => requireListedSize(store, file.key, { bytes: file.bytes, what: "vector file" })),
|
|
1368
1388
|
...ref.graphParts.map((part) => requireListedSize(store, part.key, { bytes: part.bytes, what: "graph part" }))
|
|
@@ -1383,8 +1403,8 @@ async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
|
|
|
1383
1403
|
resolvePartition
|
|
1384
1404
|
});
|
|
1385
1405
|
if (field === null) return false;
|
|
1386
|
-
vectorIndex.openColdField(field, async (beginRestore, stillWanted,
|
|
1387
|
-
await waitBeforeColdLoad(
|
|
1406
|
+
vectorIndex.openColdField(field, async (beginRestore, stillWanted, loadingWanted) => {
|
|
1407
|
+
await waitBeforeColdLoad(loadingWanted);
|
|
1388
1408
|
if (!stillWanted()) return;
|
|
1389
1409
|
const graph = await readGraphParts(store, ref);
|
|
1390
1410
|
if (!stillWanted()) return;
|
|
@@ -1724,18 +1744,19 @@ function frameRecord(entry) {
|
|
|
1724
1744
|
}
|
|
1725
1745
|
function checkSegmentHeader(data) {
|
|
1726
1746
|
if (data.length < SEGMENT_HEADER_SIZE) {
|
|
1727
|
-
return { ok: false, reason: "segment shorter than the 8-byte header" };
|
|
1747
|
+
return { ok: false, reason: "segment shorter than the 8-byte header", code: ErrorCodes.PERSISTENCE_WAL_CORRUPT };
|
|
1728
1748
|
}
|
|
1729
1749
|
for (let i = 0; i < WAL_MAGIC.length; i += 1) {
|
|
1730
1750
|
if (data[i] !== WAL_MAGIC[i]) {
|
|
1731
|
-
return { ok: false, reason: "missing NRSW magic" };
|
|
1751
|
+
return { ok: false, reason: "missing NRSW magic", code: ErrorCodes.PERSISTENCE_WAL_CORRUPT };
|
|
1732
1752
|
}
|
|
1733
1753
|
}
|
|
1734
1754
|
const formatVersion = data[4];
|
|
1735
1755
|
if (formatVersion > WAL_FORMAT_VERSION) {
|
|
1736
1756
|
return {
|
|
1737
1757
|
ok: false,
|
|
1738
|
-
reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION}
|
|
1758
|
+
reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION}, so a newer release wrote this segment`,
|
|
1759
|
+
code: ErrorCodes.ENVELOPE_VERSION_MISMATCH
|
|
1739
1760
|
};
|
|
1740
1761
|
}
|
|
1741
1762
|
return { ok: true };
|
|
@@ -1754,7 +1775,9 @@ function corrupt(message, details) {
|
|
|
1754
1775
|
function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
1755
1776
|
const header = checkSegmentHeader(data);
|
|
1756
1777
|
if (!header.ok) {
|
|
1757
|
-
|
|
1778
|
+
throw new NarsilError(header.code, `WAL segment header invalid in durable region: ${header.reason}`, {
|
|
1779
|
+
reason: header.reason
|
|
1780
|
+
});
|
|
1758
1781
|
}
|
|
1759
1782
|
if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
|
|
1760
1783
|
corrupt("Commit marker durable byte length is outside the segment", {
|
|
@@ -1902,14 +1925,10 @@ async function* readSegmentsInOrder(directory, wal) {
|
|
|
1902
1925
|
if (startSeqNo < marker.state.activeSegmentSeqNo) {
|
|
1903
1926
|
const header = checkSegmentHeader(bytes);
|
|
1904
1927
|
if (!header.ok) {
|
|
1905
|
-
throw new NarsilError(
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
key,
|
|
1910
|
-
reason: header.reason
|
|
1911
|
-
}
|
|
1912
|
-
);
|
|
1928
|
+
throw new NarsilError(header.code, `Sealed WAL segment header invalid: ${header.reason}`, {
|
|
1929
|
+
key,
|
|
1930
|
+
reason: header.reason
|
|
1931
|
+
});
|
|
1913
1932
|
}
|
|
1914
1933
|
const durableEntries2 = readDurableRegion(bytes, bytes.length);
|
|
1915
1934
|
for (const entry of durableEntries2) highestRead = Math.max(highestRead, entry.seqNo);
|
|
@@ -2123,7 +2142,7 @@ async function compactPartitionSegments(input) {
|
|
|
2123
2142
|
if (paged) {
|
|
2124
2143
|
const { directory } = input.store;
|
|
2125
2144
|
if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
|
|
2126
|
-
const { mergePagedWindow } = await import('./module-
|
|
2145
|
+
const { mergePagedWindow } = await import('./module-FHJ2GOBI.mjs');
|
|
2127
2146
|
return replacing(
|
|
2128
2147
|
await mergePagedWindow({
|
|
2129
2148
|
store: input.store,
|
|
@@ -2211,7 +2230,7 @@ function pagedSettingsOf(context) {
|
|
|
2211
2230
|
async function tombstonesOfDelta(context, older, selections, deleted) {
|
|
2212
2231
|
const { directory } = context.input.store;
|
|
2213
2232
|
if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
|
|
2214
|
-
const { pagedTombstonesFor } = await import('./module-
|
|
2233
|
+
const { pagedTombstonesFor } = await import('./module-FHJ2GOBI.mjs');
|
|
2215
2234
|
return pagedTombstonesFor(
|
|
2216
2235
|
directory,
|
|
2217
2236
|
older,
|
|
@@ -2412,7 +2431,7 @@ function removeFromPartition(manager, partitionId, docIds) {
|
|
|
2412
2431
|
}
|
|
2413
2432
|
}
|
|
2414
2433
|
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
2415
|
-
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-
|
|
2434
|
+
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-FHJ2GOBI.mjs');
|
|
2416
2435
|
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
2417
2436
|
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
2418
2437
|
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
@@ -2426,7 +2445,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
2426
2445
|
const { directory } = store;
|
|
2427
2446
|
for (const ref of partition.segments) {
|
|
2428
2447
|
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
2429
|
-
const { isPagedSegmentFile } = await import('./module-
|
|
2448
|
+
const { isPagedSegmentFile } = await import('./module-FHJ2GOBI.mjs');
|
|
2430
2449
|
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
2431
2450
|
await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
2432
2451
|
await yieldToEventLoop();
|
|
@@ -2482,7 +2501,7 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
2482
2501
|
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
2483
2502
|
);
|
|
2484
2503
|
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
2485
|
-
const { startReadingOpeningSectionsAhead } = await import('./module-
|
|
2504
|
+
const { startReadingOpeningSectionsAhead } = await import('./module-FHJ2GOBI.mjs');
|
|
2486
2505
|
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
2487
2506
|
}
|
|
2488
2507
|
for (const partition of manifest.partitions) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-IDFMNRIC.mjs';
|
|
2
2
|
import { getNestedValue, requireValidFieldTypes } from './chunk-PU55IOG4.mjs';
|
|
3
3
|
import { VECTOR_PATTERN } from './chunk-DKJTZ625.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-YH7W556M.mjs';
|
|
@@ -608,7 +608,11 @@ function decodeKeywordSegment(payload, shared) {
|
|
|
608
608
|
}
|
|
609
609
|
const raw = requireRecord(decoded, "payload");
|
|
610
610
|
if (raw.v !== KEYWORD_SEGMENT_PAYLOAD_VERSION) {
|
|
611
|
-
|
|
611
|
+
throw new NarsilError(
|
|
612
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
613
|
+
`This keyword segment payload was written in layout ${String(raw.v)}, and this build reads layout ${KEYWORD_SEGMENT_PAYLOAD_VERSION} alone`,
|
|
614
|
+
{ version: raw.v }
|
|
615
|
+
);
|
|
612
616
|
}
|
|
613
617
|
const schema = decodeSchema(raw.schema);
|
|
614
618
|
const tombstones = requireStrings(raw.tombstones, "tombstones");
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { encodeDocumentJson } from './chunk-R4OPUZQ5.mjs';
|
|
2
2
|
import { createVectorFileReader } from './chunk-UFNCLEQM.mjs';
|
|
3
|
-
import { loadFailed, sectionNamed, PAGED_SEGMENT_ENVELOPE_VERSION, writeSegmentsOf, MessagePackChunks, writeSurfaceForms, writeFieldLengths, writeValueColumns, writeBooleanColumns, writeEnumColumns, writeGeopointColumns, writePatternColumns, writePartialWords, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, REWRITE_SPILL_BYTES, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, LIST_VIEWS_HELD_PER_SEGMENT, TOMBSTONES_BETWEEN_YIELDS, readSegmentFile, decodeSections, READS_AHEAD_AT_ONCE, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, LARGEST_CACHED_ENTRY_SHARE, MAX_THREAD_FILE_NUMBER, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, SECTION_WRITE_BUFFER_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, READ_AHEAD_CHUNK_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, THREAD_READ_CACHE_BYTES, MERGE_READ_CACHE_BYTES } from './chunk-
|
|
4
|
-
import { requireOffsets, requireUtf8Entries, requireEntriesAscend, decodeFieldLengths, decodePartialWords, decodePatternColumns, decodeGeopointColumns, decodeEnumColumns, decodeBooleanColumns, decodeValueColumns, setNestedValue, littleEndianBytes, uint32sFrom, uint64Bytes, decodeSurfaceForms } from './chunk-
|
|
5
|
-
import { HEADER_SIZE, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, isCompositePartition, mergeFieldLengths, mergeSurfaceForms, encodeSurfaceTable, mergePartialWords, buildPartialWordTableFrom, mergeFieldIndexes, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, decodeSurfaceTable, readHeader, mergeDocFrequencies, createSharedFrozenSegment, shareSegmentArrays, selectionKeeps, fieldIndexRemapOf, freezeSegmentShared, createFrozenSegment, scanPostingBlocks, holdsADiscardedField, endOfValue } from './chunk-
|
|
3
|
+
import { loadFailed, sectionNamed, PAGED_SEGMENT_ENVELOPE_VERSION, writeSegmentsOf, MessagePackChunks, writeSurfaceForms, writeFieldLengths, writeValueColumns, writeBooleanColumns, writeEnumColumns, writeGeopointColumns, writePatternColumns, writePartialWords, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, REWRITE_SPILL_BYTES, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, LIST_VIEWS_HELD_PER_SEGMENT, TOMBSTONES_BETWEEN_YIELDS, readSegmentFile, decodeSections, READS_AHEAD_AT_ONCE, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, LARGEST_CACHED_ENTRY_SHARE, MAX_THREAD_FILE_NUMBER, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, SECTION_WRITE_BUFFER_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, READ_AHEAD_CHUNK_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, THREAD_READ_CACHE_BYTES, MERGE_READ_CACHE_BYTES } from './chunk-FS7BYE23.mjs';
|
|
4
|
+
import { requireOffsets, requireUtf8Entries, requireEntriesAscend, decodeFieldLengths, decodePartialWords, decodePatternColumns, decodeGeopointColumns, decodeEnumColumns, decodeBooleanColumns, decodeValueColumns, setNestedValue, littleEndianBytes, uint32sFrom, uint64Bytes, decodeSurfaceForms } from './chunk-CLE3LZ77.mjs';
|
|
5
|
+
import { HEADER_SIZE, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, isCompositePartition, mergeFieldLengths, mergeSurfaceForms, encodeSurfaceTable, mergePartialWords, buildPartialWordTableFrom, mergeFieldIndexes, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, decodeSurfaceTable, readHeader, mergeDocFrequencies, createSharedFrozenSegment, shareSegmentArrays, selectionKeeps, fieldIndexRemapOf, freezeSegmentShared, createFrozenSegment, scanPostingBlocks, holdsADiscardedField, endOfValue } from './chunk-IDFMNRIC.mjs';
|
|
6
6
|
import { flattenSchema, parseVectorDimension, encodeDocument } from './chunk-PU55IOG4.mjs';
|
|
7
7
|
import { VECTOR_PATTERN } from './chunk-DKJTZ625.mjs';
|
|
8
8
|
import { compareCodePoints } from './chunk-PPEPTUVV.mjs';
|
|
@@ -1306,7 +1306,7 @@ async function coldFieldOf(checkpoint, field) {
|
|
|
1306
1306
|
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
1307
1307
|
if (dimension === null) return null;
|
|
1308
1308
|
const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
|
|
1309
|
-
const { openColdVectorField } = await import('./cold-module-
|
|
1309
|
+
const { openColdVectorField } = await import('./cold-module-KFGEWYEL.mjs');
|
|
1310
1310
|
return openColdVectorField({
|
|
1311
1311
|
fieldPath: field.fieldPath,
|
|
1312
1312
|
dimension,
|
|
@@ -4,14 +4,14 @@ function holdsChangeWork(entry) {
|
|
|
4
4
|
}
|
|
5
5
|
async function abandonChangeOf(core, indexName) {
|
|
6
6
|
if (!holdsChangeWork(core.indexRegistry.get(indexName))) return;
|
|
7
|
-
const { abandonAnalysisChange } = await import('./analysis-change-
|
|
7
|
+
const { abandonAnalysisChange } = await import('./analysis-change-DXPZRGWF.mjs');
|
|
8
8
|
await abandonAnalysisChange(core, indexName);
|
|
9
9
|
}
|
|
10
10
|
async function resumeChangeOf(core, indexName) {
|
|
11
11
|
if ((core.indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
12
12
|
if (core.mutationCtx.capacityScope === "partition") return;
|
|
13
13
|
if (core.executor.getManager(indexName) === void 0) return;
|
|
14
|
-
const { resumeAnalysisChange } = await import('./analysis-change-
|
|
14
|
+
const { resumeAnalysisChange } = await import('./analysis-change-DXPZRGWF.mjs');
|
|
15
15
|
void resumeAnalysisChange(core, indexName).catch(() => void 0);
|
|
16
16
|
}
|
|
17
17
|
async function resumeEveryChange(core) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { littleEndianBytes, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, KEYWORD_SEGMENT_PAYLOAD_VERSION, uint64Bytes } from './chunk-
|
|
2
|
-
import { crc32, openChecksummedEnvelope, packChecksummedEnvelope, selectionKeeps, HEADER_SIZE, segmentArraysOf, mergeSegmentContents } from './chunk-
|
|
1
|
+
import { littleEndianBytes, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, KEYWORD_SEGMENT_PAYLOAD_VERSION, uint64Bytes } from './chunk-CLE3LZ77.mjs';
|
|
2
|
+
import { crc32, openChecksummedEnvelope, packChecksummedEnvelope, selectionKeeps, HEADER_SIZE, segmentArraysOf, mergeSegmentContents } from './chunk-IDFMNRIC.mjs';
|
|
3
3
|
import { fnv1a } from './chunk-L4M7QJNP.mjs';
|
|
4
4
|
import { messagePackUpperBoundBytes } from './chunk-PU55IOG4.mjs';
|
|
5
5
|
import { MAX_FILE_BYTES } from './chunk-TJMSKBEM.mjs';
|
|
@@ -107,7 +107,11 @@ function decodePagedSegmentHead(head, payloadLength, what) {
|
|
|
107
107
|
if (raw === null || typeof raw !== "object") loadFailed(`The head of ${what} is not a map`);
|
|
108
108
|
const entry = raw;
|
|
109
109
|
if (entry.v !== PAGED_SEGMENT_HEAD_VERSION) {
|
|
110
|
-
|
|
110
|
+
throw new NarsilError(
|
|
111
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
112
|
+
`The head of ${what} was written in layout ${String(entry.v)}, and this build reads layout ${PAGED_SEGMENT_HEAD_VERSION} alone`,
|
|
113
|
+
{ version: entry.v }
|
|
114
|
+
);
|
|
111
115
|
}
|
|
112
116
|
if (!isStringRecord(entry.schema)) loadFailed(`The head of ${what} holds no schema`);
|
|
113
117
|
if (!isStringList(entry.tombstones)) loadFailed(`The head of ${what} holds no tombstone list`);
|
|
@@ -910,7 +914,7 @@ async function writePagedGroup(state, group) {
|
|
|
910
914
|
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
911
915
|
const id = state.nextSegmentId;
|
|
912
916
|
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
913
|
-
const { writePagedSegment } = await import('./module-
|
|
917
|
+
const { writePagedSegment } = await import('./module-FHJ2GOBI.mjs');
|
|
914
918
|
const written = await writePagedSegment({
|
|
915
919
|
directory,
|
|
916
920
|
key,
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// src/vector/vector-index/constants.ts
|
|
2
|
+
var DEFAULT_PROMOTION_THRESHOLD = 1024;
|
|
3
|
+
var DEFAULT_FILTER_THRESHOLD = 0.03;
|
|
4
|
+
var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
|
|
5
|
+
var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
|
|
6
|
+
var WORKER_COPY_MIN_VECTORS = 1024;
|
|
7
|
+
var BUILD_CHUNK_SIZE = 100;
|
|
8
|
+
var BUILD_STEPS_BETWEEN_CLOCK_CHECKS = 256;
|
|
9
|
+
var SEARCH_SCAN_CHUNK_VECTORS = 4096;
|
|
10
|
+
var SEARCH_REFILL_RESERVE = 64;
|
|
11
|
+
var SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES = 1;
|
|
12
|
+
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
13
|
+
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
14
|
+
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
15
|
+
var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
|
|
16
|
+
var REBUILD_REMOVED_RATIO = 0.2;
|
|
17
|
+
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
18
|
+
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
19
|
+
var RECALIBRATION_CHUNK_VECTORS = 512;
|
|
20
|
+
var VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE = 0.2;
|
|
21
|
+
var VECTOR_FILE_FIXED_OVERHEAD_BYTES = 65536;
|
|
22
|
+
var VECTOR_FILE_STRING_HEADER_BYTES = 5;
|
|
23
|
+
var VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT = 9;
|
|
24
|
+
var OSQ4_MIN_DIMENSION = 384;
|
|
25
|
+
var DEFAULT_VECTOR_METRIC = "cosine";
|
|
26
|
+
var DISK_ADOPTION_YIELD_INTERVAL = 4096;
|
|
27
|
+
|
|
28
|
+
// src/vector/vector-index/default-quantization.ts
|
|
29
|
+
function defaultQuantizationFor(dimension) {
|
|
30
|
+
if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
|
|
31
|
+
return "osq8";
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, BUILD_TURN_HANDOVER_AFTER_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, REBUILD_REMOVED_RATIO, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT, VECTOR_FILE_FIXED_OVERHEAD_BYTES, VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE, VECTOR_FILE_STRING_HEADER_BYTES, WORKER_COPY_MIN_VECTORS, defaultQuantizationFor };
|
|
@@ -13518,7 +13518,7 @@ async function attachSegments(entry, action) {
|
|
|
13518
13518
|
growPartitionsTo(entry.manager, segment.partitionId);
|
|
13519
13519
|
const surfaceForms = segment.survivingSurfaceForms;
|
|
13520
13520
|
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
13521
|
-
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-
|
|
13521
|
+
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-FHJ2GOBI.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
13522
13522
|
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
13523
13523
|
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
13524
13524
|
}
|
|
@@ -13573,7 +13573,7 @@ function runSegmentAction(entry, action) {
|
|
|
13573
13573
|
case "attachSegments":
|
|
13574
13574
|
return attachSegments(entry, action);
|
|
13575
13575
|
case "adoptDiskSegments":
|
|
13576
|
-
return import('./module-
|
|
13576
|
+
return import('./module-FHJ2GOBI.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
|
|
13577
13577
|
case "compactSegments": {
|
|
13578
13578
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
13579
13579
|
return mergeFrozenSegments(
|
|
@@ -13621,7 +13621,7 @@ function requireLoadedVectorCode() {
|
|
|
13621
13621
|
function loadVectorCode() {
|
|
13622
13622
|
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
13623
13623
|
if (loading2 === null) {
|
|
13624
|
-
loading2 = import('./module-
|
|
13624
|
+
loading2 = import('./module-UJ5VUARS.mjs').then(
|
|
13625
13625
|
(code) => {
|
|
13626
13626
|
loaded2 = code;
|
|
13627
13627
|
return code;
|
|
@@ -14037,7 +14037,7 @@ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
|
14037
14037
|
// src/partitioning/rewrite-spill-of.ts
|
|
14038
14038
|
async function rewriteSpillOf(storage, manager, target) {
|
|
14039
14039
|
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
14040
|
-
const { openRewriteSpill } = await import('./module-
|
|
14040
|
+
const { openRewriteSpill } = await import('./module-FHJ2GOBI.mjs');
|
|
14041
14041
|
return openRewriteSpill(storage, manager, target);
|
|
14042
14042
|
}
|
|
14043
14043
|
|
|
@@ -14730,7 +14730,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
14730
14730
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
14731
14731
|
if (existing !== void 0) {
|
|
14732
14732
|
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
14733
|
-
const { adoptSavedAnalysis } = await import('./analysis-change-
|
|
14733
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-DXPZRGWF.mjs');
|
|
14734
14734
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
14735
14735
|
}
|
|
14736
14736
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
@@ -17364,7 +17364,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
17364
17364
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
17365
17365
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
17366
17366
|
}
|
|
17367
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
17367
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-QKJB2QBT.mjs');
|
|
17368
17368
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
17369
17369
|
const executor = createDirectExecutor({
|
|
17370
17370
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -17549,7 +17549,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
17549
17549
|
requireManager,
|
|
17550
17550
|
resumeChange: async (indexName) => {
|
|
17551
17551
|
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
17552
|
-
await (await import('./analysis-change-entry-
|
|
17552
|
+
await (await import('./analysis-change-entry-CXNRW324.mjs')).resumeChangeOf(engineCore, indexName);
|
|
17553
17553
|
},
|
|
17554
17554
|
onClose: hooks?.onIndexClose,
|
|
17555
17555
|
onOpen: (indexName) => {
|
|
@@ -17636,7 +17636,7 @@ async function startEngineCore(core, config) {
|
|
|
17636
17636
|
checkHeapAfterRecovery(core);
|
|
17637
17637
|
}
|
|
17638
17638
|
if (core.invalidation) await core.invalidation.start();
|
|
17639
|
-
if (core.durability) await (await import('./analysis-change-entry-
|
|
17639
|
+
if (core.durability) await (await import('./analysis-change-entry-CXNRW324.mjs')).resumeEveryChange(core);
|
|
17640
17640
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
17641
17641
|
} catch (err) {
|
|
17642
17642
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -18315,7 +18315,7 @@ async function dropEngineIndex(core, name) {
|
|
|
18315
18315
|
const entry = core.requireIndex(name);
|
|
18316
18316
|
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
18317
18317
|
const abandonChange = async () => {
|
|
18318
|
-
await (await import('./analysis-change-entry-
|
|
18318
|
+
await (await import('./analysis-change-entry-CXNRW324.mjs')).abandonChangeOf(core, name);
|
|
18319
18319
|
};
|
|
18320
18320
|
if (changeRecorded()) await abandonChange();
|
|
18321
18321
|
await core.indexState.drop(name, async () => {
|
|
@@ -20171,14 +20171,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
20171
20171
|
return {
|
|
20172
20172
|
snapshot(indexName) {
|
|
20173
20173
|
return withOpenIndex2(indexName, async () => {
|
|
20174
|
-
const { createSnapshot } = await import('./snapshot-
|
|
20174
|
+
const { createSnapshot } = await import('./snapshot-KVBD4LRC.mjs');
|
|
20175
20175
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
20176
20176
|
});
|
|
20177
20177
|
},
|
|
20178
20178
|
async restore(indexName, data) {
|
|
20179
20179
|
core.guardShutdown();
|
|
20180
20180
|
validateIndexName(indexName);
|
|
20181
|
-
const restored = import('./restore-queue-
|
|
20181
|
+
const restored = import('./restore-queue-H5VCN453.mjs').then(
|
|
20182
20182
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
20183
20183
|
);
|
|
20184
20184
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -20187,8 +20187,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
20187
20187
|
exportDocuments(indexName) {
|
|
20188
20188
|
return withOpenIndex2(indexName, async () => {
|
|
20189
20189
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
20190
|
-
import('./snapshot-
|
|
20191
|
-
import('./write-
|
|
20190
|
+
import('./snapshot-KVBD4LRC.mjs'),
|
|
20191
|
+
import('./write-ES5A3WUA.mjs')
|
|
20192
20192
|
]);
|
|
20193
20193
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
20194
20194
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -20196,7 +20196,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
20196
20196
|
},
|
|
20197
20197
|
importDocumentExport(indexName, data) {
|
|
20198
20198
|
core.guardShutdown();
|
|
20199
|
-
const imported = import('./import-
|
|
20199
|
+
const imported = import('./import-XFBCCENV.mjs').then(
|
|
20200
20200
|
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
20201
20201
|
);
|
|
20202
20202
|
return emitWriteEventAfter(core, indexName, imported);
|
|
@@ -22209,7 +22209,7 @@ function createNarsilFromCore(core, config) {
|
|
|
22209
22209
|
const entry = requireIndex(indexName);
|
|
22210
22210
|
await withOpenIndexWrite(indexName, async () => {
|
|
22211
22211
|
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
22212
|
-
return (await import('./analysis-change-
|
|
22212
|
+
return (await import('./analysis-change-DXPZRGWF.mjs')).resumeAnalysisChange(core, indexName);
|
|
22213
22213
|
});
|
|
22214
22214
|
},
|
|
22215
22215
|
async changeAnalysis(indexName, change) {
|
|
@@ -22217,7 +22217,7 @@ function createNarsilFromCore(core, config) {
|
|
|
22217
22217
|
requireIndex(indexName);
|
|
22218
22218
|
const changed = withOpenIndex2(
|
|
22219
22219
|
indexName,
|
|
22220
|
-
async () => (await import('./analysis-change-
|
|
22220
|
+
async () => (await import('./analysis-change-DXPZRGWF.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
22221
22221
|
);
|
|
22222
22222
|
await emitWriteEventAfter(core, indexName, changed);
|
|
22223
22223
|
},
|
|
@@ -22255,7 +22255,7 @@ function createNarsilFromCore(core, config) {
|
|
|
22255
22255
|
}
|
|
22256
22256
|
rebalancesStarting.set(indexName, started);
|
|
22257
22257
|
const rebalanced = withOpenIndex2(indexName, async () => {
|
|
22258
|
-
const { executeRebalance } = await import('./analysis-change-
|
|
22258
|
+
const { executeRebalance } = await import('./analysis-change-DXPZRGWF.mjs');
|
|
22259
22259
|
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
22260
22260
|
releaseWrites();
|
|
22261
22261
|
return running;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readHeapStatistics, readResidentMemory } from './chunk-
|
|
1
|
+
import { readHeapStatistics, readResidentMemory } from './chunk-IDFMNRIC.mjs';
|
|
2
2
|
import { collectAndReturnPages } from './chunk-LQAKO5QV.mjs';
|
|
3
3
|
import { extractVectorFieldsFromSchema } from './chunk-PU55IOG4.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-YH7W556M.mjs';
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { REBUILD_REMOVED_RATIO, BUILD_TURN_HANDOVER_AFTER_MS, DRIFT_CHECK_GROWTH_FACTOR, VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE, VECTOR_FILE_FIXED_OVERHEAD_BYTES, VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT, VECTOR_FILE_STRING_HEADER_BYTES } from './chunk-HJBJBH26.mjs';
|
|
2
|
+
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-KOCTCUXW.mjs';
|
|
3
|
+
import { osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-VGUWIZFQ.mjs';
|
|
3
4
|
import { turnsAreBudgeted, backgroundTurnIsSpent, yieldBackgroundTurn } from './chunk-LQAKO5QV.mjs';
|
|
4
5
|
import { MAX_DOC_ID_LENGTH } from './chunk-AM7RSBPF.mjs';
|
|
5
6
|
import { MAX_FILE_BYTES } from './chunk-TJMSKBEM.mjs';
|
|
@@ -72,32 +73,6 @@ function ordinalFilterForSubsets(subsets, space) {
|
|
|
72
73
|
return filter;
|
|
73
74
|
}
|
|
74
75
|
|
|
75
|
-
// src/vector/vector-index/constants.ts
|
|
76
|
-
var DEFAULT_PROMOTION_THRESHOLD = 1024;
|
|
77
|
-
var DEFAULT_FILTER_THRESHOLD = 0.03;
|
|
78
|
-
var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
|
|
79
|
-
var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
|
|
80
|
-
var WORKER_COPY_MIN_VECTORS = 1024;
|
|
81
|
-
var BUILD_CHUNK_SIZE = 100;
|
|
82
|
-
var BUILD_STEPS_BETWEEN_CLOCK_CHECKS = 256;
|
|
83
|
-
var SEARCH_SCAN_CHUNK_VECTORS = 4096;
|
|
84
|
-
var SEARCH_REFILL_RESERVE = 64;
|
|
85
|
-
var SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES = 1;
|
|
86
|
-
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
87
|
-
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
88
|
-
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
89
|
-
var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
|
|
90
|
-
var REBUILD_REMOVED_RATIO = 0.2;
|
|
91
|
-
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
92
|
-
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
93
|
-
var RECALIBRATION_CHUNK_VECTORS = 512;
|
|
94
|
-
var VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE = 0.2;
|
|
95
|
-
var VECTOR_FILE_FIXED_OVERHEAD_BYTES = 65536;
|
|
96
|
-
var VECTOR_FILE_STRING_HEADER_BYTES = 5;
|
|
97
|
-
var VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT = 9;
|
|
98
|
-
var OSQ4_MIN_DIMENSION = 384;
|
|
99
|
-
var DISK_ADOPTION_YIELD_INTERVAL = 4096;
|
|
100
|
-
|
|
101
76
|
// src/vector/vector-index/shared.ts
|
|
102
77
|
var VECTOR_WORKER_COPIES_ALLOWED = { enabled: true };
|
|
103
78
|
function liveSize(state) {
|
|
@@ -676,4 +651,4 @@ function recordCheckpoint(state, plan, written) {
|
|
|
676
651
|
state.savedSignature = plan.signature;
|
|
677
652
|
}
|
|
678
653
|
|
|
679
|
-
export {
|
|
654
|
+
export { VECTOR_WORKER_COPIES_ALLOWED, adoptGraph, adoptLoadedCalibration, allLiveDocIds, assignStorePartitions, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, handOverBuildTurns, liveOrdinals, liveSize, noBuildHistory, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldBuildTurn };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { osqCodeBytes, osqStagedQueryBytes, osqRecordBytes, unpackLevels, readTrailer, stageQueryLevels, writeRecord } from './chunk-
|
|
1
|
+
import { osqCodeBytes, osqStagedQueryBytes, osqRecordBytes, unpackLevels, readTrailer, stageQueryLevels, writeRecord } from './chunk-VGUWIZFQ.mjs';
|
|
2
2
|
import { markSimdUnavailable, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_PLACEMENT_QUERY_BITS, turnsAreBudgeted, holdWrites, yieldBackgroundTurn, OSQ_LAMBDA, threadSlotGone, backgroundTurnIsSpent } from './chunk-LQAKO5QV.mjs';
|
|
3
3
|
import { NATIVE_THREAD_SEARCH_SLOTS, VECTOR_SCRATCH_SLOTS, VECTOR_BLOCK_BYTES_ON_A_PHONE, MAX_DOC_ID_TABLE_BYTES_ON_A_PHONE, MAX_VECTOR_ORDINALS_ON_A_PHONE, WASM_PAGE_BYTES, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_BLOCK_INITIAL_SLOTS, MAX_WASM_PAGES, NATIVE_ATTACH_RETRY_MS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_DEADLINE_MS } from './chunk-L4M7QJNP.mjs';
|
|
4
4
|
import { DEFAULT_EF_SEARCH, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, CONNECT_RESTARTS_BEFORE_ONE_PASS, FARTHEST_REJECTED_LINKS_KEPT, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, NEAREST_LINK_FALLBACK_HOSTS } from './chunk-TJMSKBEM.mjs';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { chunksOf, restoredConfigFields, restoredEmbedding } from './chunk-QC4ZRBSX.mjs';
|
|
2
2
|
import { untagDocumentLine } from './chunk-BRNAVOKP.mjs';
|
|
3
|
-
import { requireConfigurationShape } from './chunk-
|
|
3
|
+
import { requireConfigurationShape } from './chunk-IDFMNRIC.mjs';
|
|
4
4
|
import { turnsAreBudgeted, turnIsSpent, yieldTurn } from './chunk-LQAKO5QV.mjs';
|
|
5
5
|
import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-AM7RSBPF.mjs';
|
|
6
6
|
import { flattenSchema } from './chunk-PU55IOG4.mjs';
|