@delali/narsil 0.3.1-next.140 → 0.3.1-next.143
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, encodeSegmentManifest, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, readListedFile, requireListedChecksum, vectorFileKey, DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, graphPartKey, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-MVM33ZUL.mjs';
|
|
2
|
+
import { restoreVectorFields } from './chunk-TCLEOZ4P.mjs';
|
|
3
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-LVY2LI35.mjs';
|
|
4
|
+
import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-5DB2VDGI.mjs';
|
|
5
|
+
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-D33A5VMU.mjs';
|
|
5
6
|
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
6
7
|
import { decode, encode } from '@msgpack/msgpack';
|
|
7
8
|
|
|
@@ -57,6 +58,9 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
57
58
|
if (config.trackPositions !== void 0) {
|
|
58
59
|
metadata.trackPositions = config.trackPositions;
|
|
59
60
|
}
|
|
61
|
+
if (config.textStorage !== void 0) {
|
|
62
|
+
metadata.textStorage = config.textStorage;
|
|
63
|
+
}
|
|
60
64
|
if (config.strict !== void 0) {
|
|
61
65
|
metadata.strict = config.strict;
|
|
62
66
|
}
|
|
@@ -107,6 +111,7 @@ function metadataConfigOf(entry) {
|
|
|
107
111
|
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
108
112
|
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
109
113
|
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
114
|
+
...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
|
|
110
115
|
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
111
116
|
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
112
117
|
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
@@ -150,26 +155,6 @@ function indexMetadataOf(indexName, entry, partitionCount) {
|
|
|
150
155
|
return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
|
|
151
156
|
}
|
|
152
157
|
|
|
153
|
-
// src/persistence/durability/constants.ts
|
|
154
|
-
var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
|
|
155
|
-
var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
|
|
156
|
-
var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
|
|
157
|
-
var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
|
|
158
|
-
var DIRECTORY_LOCK_ATTEMPTS = 3;
|
|
159
|
-
var DIRECTORY_LOCK_SETTLE_MS = 50;
|
|
160
|
-
var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
|
|
161
|
-
var DEFAULT_COMPACTION_THRESHOLD = 12;
|
|
162
|
-
var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
|
|
163
|
-
var SEGMENT_FILE_FRAMING_SLACK_BYTES = 65536;
|
|
164
|
-
var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
|
|
165
|
-
var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
|
|
166
|
-
var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
|
|
167
|
-
var LOG_RECORDS_PER_CHECKPOINT = 1e5;
|
|
168
|
-
var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
|
|
169
|
-
var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
|
|
170
|
-
var CHECKPOINT_WORKER_IDLE_MS = 5e3;
|
|
171
|
-
var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
|
|
172
|
-
|
|
173
158
|
// src/serialization/stored-vector-values.ts
|
|
174
159
|
function dropStoredVectorValues(documents, schema) {
|
|
175
160
|
const vectorFieldPaths = [];
|
|
@@ -403,6 +388,9 @@ function metadataToWire(meta) {
|
|
|
403
388
|
if (meta.trackPositions !== void 0) {
|
|
404
389
|
wire.track_positions = meta.trackPositions;
|
|
405
390
|
}
|
|
391
|
+
if (meta.textStorage !== void 0) {
|
|
392
|
+
wire.text_storage = meta.textStorage;
|
|
393
|
+
}
|
|
406
394
|
if (meta.strict !== void 0) {
|
|
407
395
|
wire.strict = meta.strict;
|
|
408
396
|
}
|
|
@@ -514,6 +502,14 @@ function wireToMetadata(raw) {
|
|
|
514
502
|
if (typeof raw.track_positions === "boolean") {
|
|
515
503
|
meta.trackPositions = raw.track_positions;
|
|
516
504
|
}
|
|
505
|
+
if (raw.text_storage !== void 0) {
|
|
506
|
+
if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
|
|
507
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
|
|
508
|
+
textStorage: raw.text_storage
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
meta.textStorage = raw.text_storage;
|
|
512
|
+
}
|
|
517
513
|
if (typeof raw.strict === "boolean") {
|
|
518
514
|
meta.strict = raw.strict;
|
|
519
515
|
}
|
|
@@ -664,7 +660,7 @@ async function packEnvelope(payloadBytes, options) {
|
|
|
664
660
|
const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
|
|
665
661
|
const [major, minor, patch] = engineVersionParts();
|
|
666
662
|
const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
|
|
667
|
-
const
|
|
663
|
+
const header = {
|
|
668
664
|
magic: "NRSL",
|
|
669
665
|
envelopeFormatVersion,
|
|
670
666
|
engineVersionMajor: major,
|
|
@@ -675,64 +671,64 @@ async function packEnvelope(payloadBytes, options) {
|
|
|
675
671
|
checksum,
|
|
676
672
|
reserved: new Uint8Array(14)
|
|
677
673
|
};
|
|
678
|
-
const headerBytes = writeHeader(
|
|
674
|
+
const headerBytes = writeHeader(header);
|
|
679
675
|
const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
|
|
680
676
|
envelope.set(headerBytes, 0);
|
|
681
677
|
envelope.set(finalPayload, HEADER_SIZE);
|
|
682
678
|
return envelope;
|
|
683
679
|
}
|
|
684
680
|
async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
|
|
685
|
-
const
|
|
686
|
-
if (
|
|
681
|
+
const header = readHeader(data);
|
|
682
|
+
if (header.envelopeFormatVersion > maxAcceptedVersion) {
|
|
687
683
|
throw new NarsilError(
|
|
688
684
|
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
689
|
-
`This data holds Narsil envelope format v${
|
|
685
|
+
`This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
|
|
690
686
|
);
|
|
691
687
|
}
|
|
692
|
-
if (
|
|
688
|
+
if (header.flags.encryptionEnabled) {
|
|
693
689
|
throw new NarsilError(
|
|
694
690
|
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
695
691
|
"Encrypted envelopes are not supported in this version of Narsil"
|
|
696
692
|
);
|
|
697
693
|
}
|
|
698
|
-
const compAlgo =
|
|
699
|
-
if (
|
|
694
|
+
const compAlgo = header.flags.compressionAlgorithm;
|
|
695
|
+
if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
|
|
700
696
|
throw new NarsilError(
|
|
701
697
|
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
702
698
|
`Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
|
|
703
699
|
);
|
|
704
700
|
}
|
|
705
701
|
const availablePayload = data.length - HEADER_SIZE;
|
|
706
|
-
if (availablePayload <
|
|
702
|
+
if (availablePayload < header.payloadLength) {
|
|
707
703
|
throw new NarsilError(
|
|
708
704
|
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
709
|
-
`Envelope data is truncated: header declares ${
|
|
705
|
+
`Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
|
|
710
706
|
);
|
|
711
707
|
}
|
|
712
|
-
let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE +
|
|
713
|
-
if (
|
|
708
|
+
let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
|
|
709
|
+
if (header.flags.checksumPresent) {
|
|
714
710
|
const computed = crc32(payloadBytes);
|
|
715
|
-
if (computed !==
|
|
711
|
+
if (computed !== header.checksum) {
|
|
716
712
|
throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
|
|
717
|
-
expected:
|
|
713
|
+
expected: header.checksum,
|
|
718
714
|
computed
|
|
719
715
|
});
|
|
720
716
|
}
|
|
721
717
|
}
|
|
722
|
-
if (
|
|
718
|
+
if (header.flags.compressionEnabled) {
|
|
723
719
|
payloadBytes = await decompressGzip(payloadBytes);
|
|
724
720
|
}
|
|
725
|
-
return { header
|
|
721
|
+
return { header, payloadBytes };
|
|
726
722
|
}
|
|
727
723
|
async function unpackEnvelopeBytes(data) {
|
|
728
724
|
return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
|
|
729
725
|
}
|
|
730
726
|
async function unpackIndexSnapshotEnvelope(data) {
|
|
731
|
-
const { header
|
|
732
|
-
if (
|
|
727
|
+
const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
|
|
728
|
+
if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
|
|
733
729
|
throw new NarsilError(
|
|
734
730
|
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
735
|
-
`Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${
|
|
731
|
+
`Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
|
|
736
732
|
);
|
|
737
733
|
}
|
|
738
734
|
return payloadBytes;
|
|
@@ -742,127 +738,9 @@ async function writeMetadataEnvelope(meta, options = {}) {
|
|
|
742
738
|
return packEnvelope(payloadBytes, options);
|
|
743
739
|
}
|
|
744
740
|
async function readMetadataEnvelope(data) {
|
|
745
|
-
const { header
|
|
741
|
+
const { header, payloadBytes } = await unpackEnvelope(data);
|
|
746
742
|
const metadata = deserializeMetadata(payloadBytes);
|
|
747
|
-
return { header
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
// src/persistence/durability/segment/layout.ts
|
|
751
|
-
var SEGMENT_ID_WIDTH = 16;
|
|
752
|
-
function manifestKey(indexName) {
|
|
753
|
-
return `${indexName}/manifest`;
|
|
754
|
-
}
|
|
755
|
-
function snapshotBundleKey(indexName) {
|
|
756
|
-
return `${indexName}/snapshot`;
|
|
757
|
-
}
|
|
758
|
-
function segmentPrefix(indexName, partitionId) {
|
|
759
|
-
return `${indexName}/segments/${partitionId}/`;
|
|
760
|
-
}
|
|
761
|
-
function segmentKey(indexName, partitionId, segmentId) {
|
|
762
|
-
return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
|
|
763
|
-
}
|
|
764
|
-
function segmentsPrefix(indexName) {
|
|
765
|
-
return `${indexName}/segments/`;
|
|
766
|
-
}
|
|
767
|
-
function vectorFileKey(indexName, fieldPath, fileId) {
|
|
768
|
-
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
|
|
769
|
-
}
|
|
770
|
-
function requireGeneration(generation) {
|
|
771
|
-
if (Number.isSafeInteger(generation) && generation > 0) return;
|
|
772
|
-
throw new NarsilError(
|
|
773
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
774
|
-
`Vector graph generation ${generation} must be a positive integer`,
|
|
775
|
-
{ generation }
|
|
776
|
-
);
|
|
777
|
-
}
|
|
778
|
-
function graphPartKey(indexName, fieldPath, generation, part) {
|
|
779
|
-
requireGeneration(generation);
|
|
780
|
-
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
|
|
781
|
-
}
|
|
782
|
-
function formatSegmentId(segmentId) {
|
|
783
|
-
if (!Number.isInteger(segmentId) || segmentId < 0) {
|
|
784
|
-
throw new NarsilError(
|
|
785
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
786
|
-
`Segment id ${segmentId} must be a non-negative integer`,
|
|
787
|
-
{ segmentId }
|
|
788
|
-
);
|
|
789
|
-
}
|
|
790
|
-
return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
|
|
791
|
-
}
|
|
792
|
-
var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
|
|
793
|
-
function encodeFieldPath(fieldPath) {
|
|
794
|
-
if (!FIELD_PATH_PATTERN.test(fieldPath)) {
|
|
795
|
-
throw new NarsilError(
|
|
796
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
797
|
-
`Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
|
|
798
|
-
{ fieldPath }
|
|
799
|
-
);
|
|
800
|
-
}
|
|
801
|
-
const fingerprint = fnv1a(fieldPath).toString(36);
|
|
802
|
-
return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
|
|
803
|
-
}
|
|
804
|
-
var SEGMENT_MANIFEST_VERSION = 7;
|
|
805
|
-
var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
|
|
806
|
-
var MANIFEST_ENVELOPE_VERSION = 2;
|
|
807
|
-
var MAX_SEGMENTS_PER_PARTITION = 65536;
|
|
808
|
-
var MAX_SEGMENT_ID = 281474976710655;
|
|
809
|
-
function fileRefOnTheWire(ref) {
|
|
810
|
-
return { key: ref.key, bytes: ref.bytes, checksum: ref.checksum };
|
|
811
|
-
}
|
|
812
|
-
function encodeSegmentManifest(manifest) {
|
|
813
|
-
const payload = encode({
|
|
814
|
-
version: SEGMENT_MANIFEST_VERSION,
|
|
815
|
-
schema: manifest.schema,
|
|
816
|
-
language: manifest.language,
|
|
817
|
-
...manifest.analysisGeneration ? { analysisGeneration: manifest.analysisGeneration } : {},
|
|
818
|
-
checkpoint: manifest.checkpoint.map((c) => ({
|
|
819
|
-
partitionId: c.partitionId,
|
|
820
|
-
lastSeqNo: c.lastSeqNo,
|
|
821
|
-
primaryTerm: c.primaryTerm,
|
|
822
|
-
...c.termHistory === void 0 ? {} : {
|
|
823
|
-
termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
|
|
824
|
-
}
|
|
825
|
-
})),
|
|
826
|
-
partitions: manifest.partitions.map((p) => ({
|
|
827
|
-
partitionId: p.partitionId,
|
|
828
|
-
nextSegmentId: p.nextSegmentId,
|
|
829
|
-
segments: p.segments.map((s) => ({
|
|
830
|
-
id: s.id,
|
|
831
|
-
key: s.key,
|
|
832
|
-
docCount: s.docCount,
|
|
833
|
-
tombstoneCount: s.tombstoneCount,
|
|
834
|
-
bytes: s.bytes,
|
|
835
|
-
checksum: s.checksum
|
|
836
|
-
}))
|
|
837
|
-
})),
|
|
838
|
-
vectors: manifest.vectors.map((v) => ({
|
|
839
|
-
fieldPath: v.fieldPath,
|
|
840
|
-
nextFileId: v.nextFileId,
|
|
841
|
-
files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
|
|
842
|
-
graphGeneration: v.graphGeneration,
|
|
843
|
-
graphParts: v.graphParts.map(fileRefOnTheWire)
|
|
844
|
-
}))
|
|
845
|
-
});
|
|
846
|
-
return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
|
|
847
|
-
}
|
|
848
|
-
function manifestReferencedKeys(manifest) {
|
|
849
|
-
const keys = /* @__PURE__ */ new Set();
|
|
850
|
-
for (const partition of manifest.partitions) {
|
|
851
|
-
for (const segment of partition.segments) {
|
|
852
|
-
keys.add(segment.key);
|
|
853
|
-
}
|
|
854
|
-
}
|
|
855
|
-
for (const vector of manifest.vectors) {
|
|
856
|
-
for (const file of vector.files) keys.add(file.key);
|
|
857
|
-
for (const part of vector.graphParts) keys.add(part.key);
|
|
858
|
-
}
|
|
859
|
-
return keys;
|
|
860
|
-
}
|
|
861
|
-
function manifestFileRefs(manifest) {
|
|
862
|
-
const refs = [];
|
|
863
|
-
for (const partition of manifest.partitions) refs.push(...partition.segments);
|
|
864
|
-
for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
|
|
865
|
-
return refs;
|
|
743
|
+
return { header, metadata };
|
|
866
744
|
}
|
|
867
745
|
|
|
868
746
|
// src/persistence/durability/segment/manifest-fields.ts
|
|
@@ -893,7 +771,20 @@ function readFileRef(raw, expectedKey, what) {
|
|
|
893
771
|
invalidManifest(`gives the ${what} "${expectedKey}" an invalid length`, { key: expectedKey, bytes });
|
|
894
772
|
}
|
|
895
773
|
if (!isUint32(raw.checksum)) invalidManifest(`gives the ${what} "${expectedKey}" an invalid checksum`);
|
|
896
|
-
return { key: expectedKey, bytes, checksum: raw.checksum };
|
|
774
|
+
if (raw.headOffset === void 0) return { key: expectedKey, bytes, checksum: raw.checksum };
|
|
775
|
+
const { headOffset, headLength, headChecksum } = raw;
|
|
776
|
+
if (!isSequenceNumber(headOffset) || !isUint32(headLength) || !isUint32(headChecksum)) {
|
|
777
|
+
invalidManifest(`gives the ${what} "${expectedKey}" an invalid head`, { key: expectedKey });
|
|
778
|
+
}
|
|
779
|
+
if (headOffset + headLength > bytes - HEADER_SIZE) {
|
|
780
|
+
invalidManifest(`ends the head of the ${what} "${expectedKey}" outside its payload`, {
|
|
781
|
+
key: expectedKey,
|
|
782
|
+
headOffset,
|
|
783
|
+
headLength,
|
|
784
|
+
bytes
|
|
785
|
+
});
|
|
786
|
+
}
|
|
787
|
+
return { key: expectedKey, bytes, checksum: raw.checksum, headOffset, headLength, headChecksum };
|
|
897
788
|
}
|
|
898
789
|
|
|
899
790
|
// src/persistence/durability/segment/manifest-vectors-decode.ts
|
|
@@ -1151,83 +1042,292 @@ function contentsOfCaptured(captured) {
|
|
|
1151
1042
|
return contents;
|
|
1152
1043
|
}
|
|
1153
1044
|
|
|
1154
|
-
// src/
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1045
|
+
// src/serialization/crc32-combine.ts
|
|
1046
|
+
var IEEE_POLYNOMIAL = 3988292384;
|
|
1047
|
+
var CRC_BITS = 32;
|
|
1048
|
+
var zeroByteOperators = [];
|
|
1049
|
+
function timesVector(operator, vector) {
|
|
1050
|
+
let sum = 0;
|
|
1051
|
+
let rest = vector >>> 0;
|
|
1052
|
+
for (let row = 0; rest !== 0; row++, rest >>>= 1) {
|
|
1053
|
+
if (rest & 1) sum ^= operator[row];
|
|
1054
|
+
}
|
|
1055
|
+
return sum >>> 0;
|
|
1056
|
+
}
|
|
1057
|
+
function squared(operator) {
|
|
1058
|
+
const result = new Uint32Array(CRC_BITS);
|
|
1059
|
+
for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
|
|
1060
|
+
return result;
|
|
1061
|
+
}
|
|
1062
|
+
function firstOperator() {
|
|
1063
|
+
let oneZeroBit = new Uint32Array(CRC_BITS);
|
|
1064
|
+
oneZeroBit[0] = IEEE_POLYNOMIAL;
|
|
1065
|
+
for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
|
|
1066
|
+
for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
|
|
1067
|
+
return oneZeroBit;
|
|
1068
|
+
}
|
|
1069
|
+
function operatorForZeroBytesPower(power) {
|
|
1070
|
+
if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
|
|
1071
|
+
while (zeroByteOperators.length <= power) {
|
|
1072
|
+
zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
|
|
1073
|
+
}
|
|
1074
|
+
return zeroByteOperators[power];
|
|
1075
|
+
}
|
|
1076
|
+
function crc32Combine(first, second, secondByteLength) {
|
|
1077
|
+
let crc = first >>> 0;
|
|
1078
|
+
let rest = secondByteLength;
|
|
1079
|
+
for (let power = 0; rest > 0; power++) {
|
|
1080
|
+
if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
|
|
1081
|
+
rest = Math.floor(rest / 2);
|
|
1082
|
+
}
|
|
1083
|
+
return (crc ^ second) >>> 0;
|
|
1084
|
+
}
|
|
1085
|
+
var BIN8 = 196;
|
|
1086
|
+
var BIN16 = 197;
|
|
1087
|
+
var BIN32 = 198;
|
|
1088
|
+
var ARRAY16 = 220;
|
|
1089
|
+
var ARRAY32 = 221;
|
|
1090
|
+
var FIXMAP = 128;
|
|
1091
|
+
var FIXARRAY = 144;
|
|
1092
|
+
var FIXED_LIST_MAX = 15;
|
|
1093
|
+
var FIXSTR = 160;
|
|
1094
|
+
var FIXSTR_MAX_LENGTH = 31;
|
|
1095
|
+
var ASCII_MAX = 127;
|
|
1096
|
+
var UTF8_MAX_BYTES_PER_UNIT = 3;
|
|
1097
|
+
var STRING_HEADER_MAX_BYTES = 5;
|
|
1098
|
+
var BIN8_MAX_BYTES = 255;
|
|
1099
|
+
var UINT16_MAX = 65535;
|
|
1100
|
+
function binHeader(byteLength) {
|
|
1101
|
+
if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
|
|
1102
|
+
if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
1103
|
+
const header = new Uint8Array(5);
|
|
1104
|
+
header[0] = BIN32;
|
|
1105
|
+
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
1106
|
+
return header;
|
|
1107
|
+
}
|
|
1108
|
+
function arrayHeader(count) {
|
|
1109
|
+
if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
|
|
1110
|
+
if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
|
|
1111
|
+
const header = new Uint8Array(5);
|
|
1112
|
+
header[0] = ARRAY32;
|
|
1113
|
+
new DataView(header.buffer).setUint32(1, count, false);
|
|
1114
|
+
return header;
|
|
1115
|
+
}
|
|
1116
|
+
function createPayloadBuilder() {
|
|
1117
|
+
const chunks = [];
|
|
1172
1118
|
let offset = 0;
|
|
1173
|
-
|
|
1174
|
-
|
|
1119
|
+
function push(chunk) {
|
|
1120
|
+
chunks.push(chunk);
|
|
1175
1121
|
offset += chunk.byteLength;
|
|
1176
1122
|
}
|
|
1123
|
+
return {
|
|
1124
|
+
chunks,
|
|
1125
|
+
get offset() {
|
|
1126
|
+
return offset;
|
|
1127
|
+
},
|
|
1128
|
+
push,
|
|
1129
|
+
entry(key, value) {
|
|
1130
|
+
push(encode(key));
|
|
1131
|
+
push(encode(value));
|
|
1132
|
+
},
|
|
1133
|
+
rawEntry(key, encoded) {
|
|
1134
|
+
push(encode(key));
|
|
1135
|
+
const at = offset;
|
|
1136
|
+
push(encoded);
|
|
1137
|
+
return at;
|
|
1138
|
+
},
|
|
1139
|
+
bytesEntry(key, bytes) {
|
|
1140
|
+
push(encode(key));
|
|
1141
|
+
push(binHeader(bytes.byteLength));
|
|
1142
|
+
const at = offset;
|
|
1143
|
+
push(bytes);
|
|
1144
|
+
return at;
|
|
1145
|
+
}
|
|
1146
|
+
};
|
|
1147
|
+
}
|
|
1148
|
+
function pageChecksums(bytes, pageBytes) {
|
|
1149
|
+
const pages = Math.ceil(bytes.byteLength / pageBytes);
|
|
1150
|
+
const checksums = new Uint8Array(pages * 4);
|
|
1151
|
+
const view = new DataView(checksums.buffer);
|
|
1152
|
+
for (let page = 0; page < pages; page++) {
|
|
1153
|
+
const from = page * pageBytes;
|
|
1154
|
+
view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
|
|
1155
|
+
}
|
|
1156
|
+
return checksums;
|
|
1157
|
+
}
|
|
1158
|
+
function sectionOf(name, offset, bytes, pageBytes) {
|
|
1159
|
+
return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
|
|
1160
|
+
}
|
|
1161
|
+
function uint32Bytes(values) {
|
|
1162
|
+
const bytes = new Uint8Array(values.length * 4);
|
|
1163
|
+
const view = new DataView(bytes.buffer);
|
|
1164
|
+
for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
|
|
1177
1165
|
return bytes;
|
|
1178
1166
|
}
|
|
1179
|
-
function
|
|
1180
|
-
if (
|
|
1181
|
-
|
|
1182
|
-
|
|
1167
|
+
function writtenAsShortAscii(docId, list, at) {
|
|
1168
|
+
if (docId.length > FIXSTR_MAX_LENGTH) return false;
|
|
1169
|
+
for (let unit = 0; unit < docId.length; unit++) {
|
|
1170
|
+
const code = docId.charCodeAt(unit);
|
|
1171
|
+
if (code > ASCII_MAX) return false;
|
|
1172
|
+
list[at + 1 + unit] = code;
|
|
1173
|
+
}
|
|
1174
|
+
list[at] = FIXSTR | docId.length;
|
|
1175
|
+
return true;
|
|
1183
1176
|
}
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1177
|
+
function encodedIdList(docIds) {
|
|
1178
|
+
const header = arrayHeader(docIds.length);
|
|
1179
|
+
let capacity = header.byteLength;
|
|
1180
|
+
for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
|
|
1181
|
+
const list = new Uint8Array(capacity);
|
|
1182
|
+
list.set(header);
|
|
1183
|
+
const offsets = new Uint32Array(docIds.length + 1);
|
|
1184
|
+
let cursor = header.byteLength;
|
|
1185
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
1186
|
+
offsets[i] = cursor;
|
|
1187
|
+
if (writtenAsShortAscii(docIds[i], list, cursor)) {
|
|
1188
|
+
cursor += docIds[i].length + 1;
|
|
1189
|
+
continue;
|
|
1190
|
+
}
|
|
1191
|
+
const encoded = encode(docIds[i]);
|
|
1192
|
+
list.set(encoded, cursor);
|
|
1193
|
+
cursor += encoded.byteLength;
|
|
1189
1194
|
}
|
|
1195
|
+
offsets[docIds.length] = cursor;
|
|
1196
|
+
return { list: list.subarray(0, cursor), offsets };
|
|
1190
1197
|
}
|
|
1191
|
-
function
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
|
|
1196
|
-
list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
|
|
1197
|
-
};
|
|
1198
|
+
function positionsInIdOrder(docIds) {
|
|
1199
|
+
const positions = Array.from(docIds.keys());
|
|
1200
|
+
positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
|
|
1201
|
+
return Uint32Array.from(positions);
|
|
1198
1202
|
}
|
|
1199
|
-
|
|
1200
|
-
const
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
const bytes = await store.read(key);
|
|
1204
|
-
if (bytes === null) missingFile(expected.what, key);
|
|
1205
|
-
requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
|
|
1203
|
+
function magnitudeBytes(magnitudes) {
|
|
1204
|
+
const bytes = new Uint8Array(magnitudes.length * 8);
|
|
1205
|
+
const view = new DataView(bytes.buffer);
|
|
1206
|
+
for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
|
|
1206
1207
|
return bytes;
|
|
1207
1208
|
}
|
|
1208
|
-
function
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1209
|
+
function checksumAcrossPages(checksums, byteLength, pageBytes) {
|
|
1210
|
+
const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
|
|
1211
|
+
let whole = 0;
|
|
1212
|
+
for (let page = 0; page * 4 < checksums.byteLength; page++) {
|
|
1213
|
+
const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
|
|
1214
|
+
whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
|
|
1215
|
+
}
|
|
1216
|
+
return whole;
|
|
1217
|
+
}
|
|
1218
|
+
function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
|
|
1219
|
+
let headLength = 0;
|
|
1220
|
+
let head = new Uint8Array(0);
|
|
1221
|
+
let vectorsSection = {
|
|
1222
|
+
name: "vectors",
|
|
1223
|
+
offset: 0,
|
|
1224
|
+
length: vectors.byteLength,
|
|
1225
|
+
pageBytes: SEGMENT_PAGE_BYTES,
|
|
1226
|
+
checksums
|
|
1227
|
+
};
|
|
1228
|
+
for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
|
|
1229
|
+
vectorsSection = {
|
|
1230
|
+
...vectorsSection,
|
|
1231
|
+
offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
|
|
1232
|
+
};
|
|
1233
|
+
head = encode({ sections: [...sections, vectorsSection] });
|
|
1234
|
+
if (head.byteLength === headLength) break;
|
|
1235
|
+
headLength = head.byteLength;
|
|
1236
|
+
}
|
|
1237
|
+
return { head, vectorsSection };
|
|
1238
|
+
}
|
|
1239
|
+
function pagedVectorFileChunks(payload, heldMagnitudes) {
|
|
1240
|
+
const builder = createPayloadBuilder();
|
|
1241
|
+
const { codes } = payload;
|
|
1242
|
+
builder.push(Uint8Array.of(FIXMAP | 10));
|
|
1243
|
+
builder.entry("v", payload.v);
|
|
1244
|
+
builder.entry("fieldName", payload.fieldName);
|
|
1245
|
+
builder.entry("dimension", payload.dimension);
|
|
1246
|
+
const { list, offsets } = encodedIdList(payload.docIds);
|
|
1247
|
+
const docIdsAt = builder.rawEntry("docIds", list);
|
|
1248
|
+
const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
|
|
1249
|
+
builder.push(encode("codes"));
|
|
1250
|
+
if (codes === null) {
|
|
1251
|
+
builder.push(encode(null));
|
|
1252
|
+
} else {
|
|
1253
|
+
builder.push(Uint8Array.of(FIXMAP | 3));
|
|
1254
|
+
builder.entry("bits", codes.bits);
|
|
1255
|
+
builder.entry("centroid", codes.centroid);
|
|
1256
|
+
const recordsAt = builder.bytesEntry("records", codes.records);
|
|
1257
|
+
sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
|
|
1258
|
+
}
|
|
1259
|
+
const magnitudes = magnitudeBytes(heldMagnitudes);
|
|
1260
|
+
sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
|
|
1261
|
+
const idOffsets = uint32Bytes(offsets);
|
|
1262
|
+
sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
|
|
1263
|
+
const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
|
|
1264
|
+
sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
|
|
1265
|
+
const headKey = encode("head");
|
|
1266
|
+
const vectorsKey = encode("vectors");
|
|
1267
|
+
const vectorsHeader = binHeader(payload.vectors.byteLength);
|
|
1268
|
+
const beforeHead = builder.offset + headKey.byteLength;
|
|
1269
|
+
const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
|
|
1270
|
+
const { head } = encodedHeadAndVectorsPlace(
|
|
1271
|
+
sections,
|
|
1272
|
+
(headLength) => beforeHead + binHeader(headLength).byteLength,
|
|
1273
|
+
payload.vectors,
|
|
1274
|
+
vectorChecksums,
|
|
1275
|
+
vectorsKey.byteLength + vectorsHeader.byteLength
|
|
1219
1276
|
);
|
|
1277
|
+
const headOffset = builder.bytesEntry("head", head);
|
|
1278
|
+
builder.push(vectorsKey);
|
|
1279
|
+
builder.push(vectorsHeader);
|
|
1280
|
+
const beforeVectors = [...builder.chunks];
|
|
1281
|
+
builder.push(payload.vectors);
|
|
1282
|
+
return {
|
|
1283
|
+
chunks: builder.chunks,
|
|
1284
|
+
headOffset,
|
|
1285
|
+
headLength: head.byteLength,
|
|
1286
|
+
headChecksum: crc32(head),
|
|
1287
|
+
beforeVectors,
|
|
1288
|
+
vectors: {
|
|
1289
|
+
byteLength: payload.vectors.byteLength,
|
|
1290
|
+
checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
|
|
1291
|
+
}
|
|
1292
|
+
};
|
|
1220
1293
|
}
|
|
1221
|
-
function
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1294
|
+
function nodeOffsetsOf(graph) {
|
|
1295
|
+
const offsets = new Uint32Array(graph.levels.length + 1);
|
|
1296
|
+
const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
|
|
1297
|
+
let cursor = 0;
|
|
1298
|
+
for (let at = 0; at < graph.levels.length; at++) {
|
|
1299
|
+
offsets[at] = cursor;
|
|
1300
|
+
for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
|
|
1301
|
+
}
|
|
1302
|
+
offsets[graph.levels.length] = cursor;
|
|
1303
|
+
return uint32Bytes(offsets);
|
|
1304
|
+
}
|
|
1305
|
+
function pagedGraphPartChunks(part) {
|
|
1306
|
+
const builder = createPayloadBuilder();
|
|
1307
|
+
builder.push(Uint8Array.of(FIXMAP | 4));
|
|
1308
|
+
builder.entry("v", part.v);
|
|
1309
|
+
builder.entry("fieldName", part.fieldName);
|
|
1310
|
+
builder.push(encode("graphs"));
|
|
1311
|
+
builder.push(arrayHeader(part.graphs.length));
|
|
1312
|
+
const layouts = [];
|
|
1313
|
+
for (const graph of part.graphs) {
|
|
1314
|
+
const { levels, neighbours, ...header } = graph;
|
|
1315
|
+
const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
|
|
1316
|
+
builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
|
|
1317
|
+
for (const [key, value] of headerEntries) builder.entry(key, value);
|
|
1318
|
+
const levelsAt = builder.bytesEntry("levels", levels);
|
|
1319
|
+
const neighboursAt = builder.bytesEntry("neighbours", neighbours);
|
|
1320
|
+
layouts.push({
|
|
1321
|
+
...header,
|
|
1322
|
+
levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
|
|
1323
|
+
nodeOffsets: nodeOffsetsOf(graph),
|
|
1324
|
+
neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
|
|
1325
|
+
});
|
|
1326
|
+
}
|
|
1327
|
+
const head = encode({ graphs: layouts });
|
|
1328
|
+
const headOffset = builder.bytesEntry("head", head);
|
|
1329
|
+
return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
|
|
1228
1330
|
}
|
|
1229
|
-
|
|
1230
|
-
// src/persistence/durability/segment/vector-load.ts
|
|
1231
1331
|
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
1232
1332
|
var GRAPH_PART_ENVELOPE_VERSION = 2;
|
|
1233
1333
|
function liveVectorsOf(ref) {
|
|
@@ -1237,13 +1337,13 @@ function liveVectorsOf(ref) {
|
|
|
1237
1337
|
}
|
|
1238
1338
|
async function openListedFile(store, ref, what, allowsCompression) {
|
|
1239
1339
|
const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what });
|
|
1240
|
-
const { header
|
|
1340
|
+
const { header, payload } = await openChecksummedEnvelope(bytes, {
|
|
1241
1341
|
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
|
|
1242
1342
|
what: `${what} "${ref.key}"`,
|
|
1243
1343
|
allowsCompression
|
|
1244
1344
|
});
|
|
1245
|
-
requireListedChecksum(what, ref.key,
|
|
1246
|
-
return { payload, payloadLength:
|
|
1345
|
+
requireListedChecksum(what, ref.key, header.checksum, ref.checksum);
|
|
1346
|
+
return { payload, payloadLength: header.payloadLength };
|
|
1247
1347
|
}
|
|
1248
1348
|
async function readGraphParts(store, ref) {
|
|
1249
1349
|
if (ref.graphParts.length === 0) return null;
|
|
@@ -1281,7 +1381,46 @@ async function readVectorFile(store, file, fieldPath) {
|
|
|
1281
1381
|
}
|
|
1282
1382
|
return { payload, payloadLength };
|
|
1283
1383
|
}
|
|
1284
|
-
|
|
1384
|
+
function holdsEveryHead(ref) {
|
|
1385
|
+
if (ref.graphParts.length === 0 || ref.files.length === 0) return false;
|
|
1386
|
+
return [...ref.files, ...ref.graphParts].every((file) => file.headOffset !== void 0);
|
|
1387
|
+
}
|
|
1388
|
+
async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
|
|
1389
|
+
const { pathOf } = store;
|
|
1390
|
+
if (pathOf === void 0 || vectorIndex.storage !== "disk" || !holdsEveryHead(ref)) return false;
|
|
1391
|
+
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-MJMKNUYJ.mjs');
|
|
1392
|
+
await Promise.all([
|
|
1393
|
+
...ref.files.map((file) => requireListedSize(store, file.key, { bytes: file.bytes, what: "vector file" })),
|
|
1394
|
+
...ref.graphParts.map((part) => requireListedSize(store, part.key, { bytes: part.bytes, what: "graph part" }))
|
|
1395
|
+
]);
|
|
1396
|
+
const files = await Promise.all(
|
|
1397
|
+
ref.files.map(async (file) => ({ ref: file, path: await pathOf.call(store, file.key) }))
|
|
1398
|
+
);
|
|
1399
|
+
const graphParts = await Promise.all(
|
|
1400
|
+
ref.graphParts.map(async (part) => ({ ref: part, path: await pathOf.call(store, part.key) }))
|
|
1401
|
+
);
|
|
1402
|
+
const field = openColdVectorField({
|
|
1403
|
+
fieldPath: ref.fieldPath,
|
|
1404
|
+
dimension: vectorIndex.dimension,
|
|
1405
|
+
metric: vectorIndex.metric,
|
|
1406
|
+
codeBits: osqBitsOf(vectorIndex.quantization),
|
|
1407
|
+
files,
|
|
1408
|
+
graphParts,
|
|
1409
|
+
resolvePartition
|
|
1410
|
+
});
|
|
1411
|
+
if (field === null) return false;
|
|
1412
|
+
vectorIndex.openColdField(field, async (beginRestore, stillWanted, firstSearchAnswered) => {
|
|
1413
|
+
await waitBeforeColdLoad(firstSearchAnswered);
|
|
1414
|
+
if (!stillWanted()) return;
|
|
1415
|
+
const graph = await readGraphParts(store, ref);
|
|
1416
|
+
if (!stillWanted()) return;
|
|
1417
|
+
const restore = beginRestore({ liveVectors: liveVectorsOf(ref), holdsGraph: true, graph });
|
|
1418
|
+
await loadFilesByParts(files, restore, ref.fieldPath, vectorIndex.dimension, stillWanted);
|
|
1419
|
+
});
|
|
1420
|
+
return true;
|
|
1421
|
+
}
|
|
1422
|
+
async function loadVectorField(store, ref, vectorIndex, resolvePartition = null) {
|
|
1423
|
+
if (resolvePartition !== null && await loadFieldOnDemand(store, ref, vectorIndex, resolvePartition)) return;
|
|
1285
1424
|
const restore = vectorIndex.restoreCheckpoint({
|
|
1286
1425
|
liveVectors: liveVectorsOf(ref),
|
|
1287
1426
|
holdsGraph: ref.graphParts.length > 0,
|
|
@@ -1304,24 +1443,24 @@ async function loadVectorField(store, ref, vectorIndex) {
|
|
|
1304
1443
|
}
|
|
1305
1444
|
var EMPTY_BIN = new Uint8Array(0);
|
|
1306
1445
|
var EMPTY_BIN_BYTES = 2;
|
|
1307
|
-
var
|
|
1308
|
-
var
|
|
1309
|
-
var
|
|
1310
|
-
var
|
|
1446
|
+
var BIN82 = 196;
|
|
1447
|
+
var BIN162 = 197;
|
|
1448
|
+
var BIN322 = 198;
|
|
1449
|
+
var BIN8_MAX_BYTES2 = 255;
|
|
1311
1450
|
var BIN16_MAX_BYTES = 65535;
|
|
1312
|
-
function
|
|
1313
|
-
if (byteLength <=
|
|
1314
|
-
if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(
|
|
1315
|
-
const
|
|
1316
|
-
|
|
1317
|
-
new DataView(
|
|
1318
|
-
return
|
|
1451
|
+
function binHeader2(byteLength) {
|
|
1452
|
+
if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
|
|
1453
|
+
if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
|
|
1454
|
+
const header = new Uint8Array(5);
|
|
1455
|
+
header[0] = BIN322;
|
|
1456
|
+
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
1457
|
+
return header;
|
|
1319
1458
|
}
|
|
1320
1459
|
function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
|
|
1321
1460
|
const head = encode(withEmptyBytesInTheirPlace);
|
|
1322
|
-
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] ===
|
|
1461
|
+
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
|
|
1323
1462
|
if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
|
|
1324
|
-
return [head.subarray(0, head.length - EMPTY_BIN_BYTES),
|
|
1463
|
+
return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
|
|
1325
1464
|
}
|
|
1326
1465
|
function vectorPartChunks(part) {
|
|
1327
1466
|
return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
|
|
@@ -1337,20 +1476,27 @@ function keptFileRef(input, prior, kept) {
|
|
|
1337
1476
|
{ key: kept.key }
|
|
1338
1477
|
);
|
|
1339
1478
|
}
|
|
1340
|
-
return { ...kept, bytes: listed.bytes, checksum: listed.checksum };
|
|
1479
|
+
return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
|
|
1480
|
+
}
|
|
1481
|
+
function headFieldsOf(ref) {
|
|
1482
|
+
return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
|
|
1483
|
+
}
|
|
1484
|
+
function headRefOf(paged) {
|
|
1485
|
+
return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
|
|
1341
1486
|
}
|
|
1342
1487
|
async function writeGraphParts(input, fieldPath, plan, generation) {
|
|
1343
1488
|
const parts = plan.graphParts ?? [];
|
|
1344
1489
|
const refs = [];
|
|
1345
1490
|
for (let part = 0; part < parts.length; part++) {
|
|
1346
1491
|
const key = graphPartKey(input.indexName, fieldPath, generation, part);
|
|
1492
|
+
const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
|
|
1347
1493
|
const packed = await packChecksummedEnvelope(
|
|
1348
|
-
[encode(parts[part])],
|
|
1494
|
+
paged === null ? [encode(parts[part])] : paged.chunks,
|
|
1349
1495
|
GRAPH_PART_ENVELOPE_VERSION,
|
|
1350
1496
|
`graph part "${key}"`
|
|
1351
1497
|
);
|
|
1352
1498
|
await input.store.write(key, packed.chunks);
|
|
1353
|
-
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum });
|
|
1499
|
+
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
|
|
1354
1500
|
}
|
|
1355
1501
|
return refs;
|
|
1356
1502
|
}
|
|
@@ -1364,17 +1510,28 @@ async function writeField(input, fieldPath, plan, outcome) {
|
|
|
1364
1510
|
const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
|
|
1365
1511
|
const written = [];
|
|
1366
1512
|
for (let index = 0; index < plan.newFiles; index++) {
|
|
1367
|
-
const { payload, ordinals, dead } = plan.readNewFile(index);
|
|
1513
|
+
const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
|
|
1368
1514
|
const id = nextFileId;
|
|
1369
1515
|
nextFileId += 1;
|
|
1370
1516
|
const key = vectorFileKey(input.indexName, fieldPath, id);
|
|
1371
|
-
const
|
|
1372
|
-
|
|
1517
|
+
const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
|
|
1518
|
+
const what = `vector file "${key}"`;
|
|
1519
|
+
const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
|
|
1520
|
+
paged.chunks,
|
|
1373
1521
|
VECTOR_FILE_ENVELOPE_VERSION,
|
|
1374
|
-
|
|
1522
|
+
what,
|
|
1523
|
+
crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
|
|
1375
1524
|
);
|
|
1376
1525
|
await input.store.write(key, packed.chunks);
|
|
1377
|
-
files.push({
|
|
1526
|
+
files.push({
|
|
1527
|
+
id,
|
|
1528
|
+
key,
|
|
1529
|
+
count: payload.docIds.length,
|
|
1530
|
+
dead,
|
|
1531
|
+
bytes: packed.byteLength,
|
|
1532
|
+
checksum: packed.checksum,
|
|
1533
|
+
...headRefOf(paged)
|
|
1534
|
+
});
|
|
1378
1535
|
written.push({ id, key, ordinals });
|
|
1379
1536
|
outcome.layouts.push({
|
|
1380
1537
|
fieldPath,
|
|
@@ -1577,10 +1734,10 @@ var RECORD_LENGTH_SIZE = 4;
|
|
|
1577
1734
|
var FRAME_CRC_SIZE = 4;
|
|
1578
1735
|
var MAX_WAL_RECORD_BYTES = 268435456;
|
|
1579
1736
|
function writeSegmentHeader() {
|
|
1580
|
-
const
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
return
|
|
1737
|
+
const header = new Uint8Array(SEGMENT_HEADER_SIZE);
|
|
1738
|
+
header.set(WAL_MAGIC, 0);
|
|
1739
|
+
header[4] = WAL_FORMAT_VERSION & 255;
|
|
1740
|
+
return header;
|
|
1584
1741
|
}
|
|
1585
1742
|
function frameRecord(entry) {
|
|
1586
1743
|
const payload = encode(entry);
|
|
@@ -1621,9 +1778,9 @@ function corrupt(message, details) {
|
|
|
1621
1778
|
throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
|
|
1622
1779
|
}
|
|
1623
1780
|
function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
1624
|
-
const
|
|
1625
|
-
if (!
|
|
1626
|
-
corrupt(`WAL segment header invalid in durable region: ${
|
|
1781
|
+
const header = checkSegmentHeader(data);
|
|
1782
|
+
if (!header.ok) {
|
|
1783
|
+
corrupt(`WAL segment header invalid in durable region: ${header.reason}`, { reason: header.reason });
|
|
1627
1784
|
}
|
|
1628
1785
|
if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
|
|
1629
1786
|
corrupt("Commit marker durable byte length is outside the segment", {
|
|
@@ -1722,6 +1879,9 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
|
|
|
1722
1879
|
function walPrefix(indexName, partitionId) {
|
|
1723
1880
|
return `${indexName}/wal/${partitionId}/`;
|
|
1724
1881
|
}
|
|
1882
|
+
function nextSegmentKey(indexName, partitionId) {
|
|
1883
|
+
return `${walPrefix(indexName, partitionId)}next`;
|
|
1884
|
+
}
|
|
1725
1885
|
function segmentStartSeqNo(key, prefix) {
|
|
1726
1886
|
if (!key.startsWith(prefix)) {
|
|
1727
1887
|
return null;
|
|
@@ -1766,14 +1926,14 @@ async function* readSegmentsInOrder(directory, wal) {
|
|
|
1766
1926
|
continue;
|
|
1767
1927
|
}
|
|
1768
1928
|
if (startSeqNo < marker.state.activeSegmentSeqNo) {
|
|
1769
|
-
const
|
|
1770
|
-
if (!
|
|
1929
|
+
const header = checkSegmentHeader(bytes);
|
|
1930
|
+
if (!header.ok) {
|
|
1771
1931
|
throw new NarsilError(
|
|
1772
1932
|
ErrorCodes.PERSISTENCE_WAL_CORRUPT,
|
|
1773
|
-
`Sealed WAL segment header invalid: ${
|
|
1933
|
+
`Sealed WAL segment header invalid: ${header.reason}`,
|
|
1774
1934
|
{
|
|
1775
1935
|
key,
|
|
1776
|
-
reason:
|
|
1936
|
+
reason: header.reason
|
|
1777
1937
|
}
|
|
1778
1938
|
);
|
|
1779
1939
|
}
|
|
@@ -1903,569 +2063,23 @@ async function buildSegmentFromEntries(input) {
|
|
|
1903
2063
|
if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
|
|
1904
2064
|
return built;
|
|
1905
2065
|
}
|
|
1906
|
-
var FIXMAP = 128;
|
|
1907
|
-
var MAP16 = 222;
|
|
1908
|
-
var MAP32 = 223;
|
|
1909
|
-
var FIXARRAY = 144;
|
|
1910
|
-
var ARRAY16 = 220;
|
|
1911
|
-
var ARRAY32 = 221;
|
|
1912
|
-
var BIN82 = 196;
|
|
1913
|
-
var BIN162 = 197;
|
|
1914
|
-
var BIN322 = 198;
|
|
1915
|
-
var FIX_COUNT_LIMIT = 16;
|
|
1916
|
-
var UINT8_LIMIT = 256;
|
|
1917
|
-
var UINT16_LIMIT = 65536;
|
|
1918
|
-
function header(fixed, sixteen, thirtyTwo, count) {
|
|
1919
|
-
if (count < FIX_COUNT_LIMIT) return Uint8Array.of(fixed | count);
|
|
1920
|
-
if (count < UINT16_LIMIT) return Uint8Array.of(sixteen, count >> 8, count & 255);
|
|
1921
|
-
const bytes = new Uint8Array(5);
|
|
1922
|
-
bytes[0] = thirtyTwo;
|
|
1923
|
-
new DataView(bytes.buffer).setUint32(1, count, false);
|
|
1924
|
-
return bytes;
|
|
1925
|
-
}
|
|
1926
|
-
function binHeader2(byteLength) {
|
|
1927
|
-
if (byteLength < UINT8_LIMIT) return Uint8Array.of(BIN82, byteLength);
|
|
1928
|
-
if (byteLength < UINT16_LIMIT) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
|
|
1929
|
-
const bytes = new Uint8Array(5);
|
|
1930
|
-
bytes[0] = BIN322;
|
|
1931
|
-
new DataView(bytes.buffer).setUint32(1, byteLength, false);
|
|
1932
|
-
return bytes;
|
|
1933
|
-
}
|
|
1934
|
-
var MessagePackChunks = class {
|
|
1935
|
-
chunks = [];
|
|
1936
|
-
byteLength = 0;
|
|
1937
|
-
push(chunk) {
|
|
1938
|
-
if (chunk.byteLength === 0) return;
|
|
1939
|
-
this.chunks.push(chunk);
|
|
1940
|
-
this.byteLength += chunk.byteLength;
|
|
1941
|
-
}
|
|
1942
|
-
map(entries) {
|
|
1943
|
-
this.push(header(FIXMAP, MAP16, MAP32, entries));
|
|
1944
|
-
return this;
|
|
1945
|
-
}
|
|
1946
|
-
array(items) {
|
|
1947
|
-
this.push(header(FIXARRAY, ARRAY16, ARRAY32, items));
|
|
1948
|
-
return this;
|
|
1949
|
-
}
|
|
1950
|
-
value(value) {
|
|
1951
|
-
this.push(encode(value));
|
|
1952
|
-
return this;
|
|
1953
|
-
}
|
|
1954
|
-
key(name) {
|
|
1955
|
-
return this.value(name);
|
|
1956
|
-
}
|
|
1957
|
-
binary(bytes) {
|
|
1958
|
-
this.push(binHeader2(bytes.byteLength));
|
|
1959
|
-
this.push(bytes);
|
|
1960
|
-
return this;
|
|
1961
|
-
}
|
|
1962
|
-
binaryOrNil(bytes) {
|
|
1963
|
-
return bytes === null ? this.value(null) : this.binary(bytes);
|
|
1964
|
-
}
|
|
1965
|
-
};
|
|
1966
|
-
|
|
1967
|
-
// src/serialization/keyword-segment/token-order.ts
|
|
1968
|
-
function slotsAreInTokenOrder(payloadSlots) {
|
|
1969
|
-
for (let at = 0; at < payloadSlots.length; at++) {
|
|
1970
|
-
if (payloadSlots[at] !== at) return false;
|
|
1971
|
-
}
|
|
1972
|
-
return true;
|
|
1973
|
-
}
|
|
1974
|
-
function reorderedPositions(arrays, postingOffsets) {
|
|
1975
|
-
const { positionOffsets, positionValues, tokenTable } = arrays;
|
|
1976
|
-
if (positionOffsets === null || positionValues === null) return { positionOffsets: null, positionValues: null };
|
|
1977
|
-
const postings = postingOffsets[postingOffsets.length - 1];
|
|
1978
|
-
const offsets = new Uint32Array(postings + 1);
|
|
1979
|
-
const values = new Uint32Array(positionValues.length);
|
|
1980
|
-
let cursor = 0;
|
|
1981
|
-
for (let token = 0; token < tokenTable.payloadSlots.length; token++) {
|
|
1982
|
-
const slot = tokenTable.payloadSlots[token];
|
|
1983
|
-
for (let from = arrays.postingOffsets[slot]; from < arrays.postingOffsets[slot + 1]; from++) {
|
|
1984
|
-
const to = postingOffsets[token] + (from - arrays.postingOffsets[slot]);
|
|
1985
|
-
offsets[to] = cursor;
|
|
1986
|
-
const start = positionOffsets[from];
|
|
1987
|
-
const end = positionOffsets[from + 1];
|
|
1988
|
-
values.set(positionValues.subarray(start, end), cursor);
|
|
1989
|
-
cursor += end - start;
|
|
1990
|
-
}
|
|
1991
|
-
}
|
|
1992
|
-
offsets[postings] = cursor;
|
|
1993
|
-
return { positionOffsets: offsets, positionValues: values };
|
|
1994
|
-
}
|
|
1995
|
-
function postingsInTokenOrder(arrays) {
|
|
1996
|
-
const { tokenTable } = arrays;
|
|
1997
|
-
if (slotsAreInTokenOrder(tokenTable.payloadSlots)) return arrays;
|
|
1998
|
-
const tokens = tokenTable.payloadSlots.length;
|
|
1999
|
-
const postingOffsets = new Uint32Array(tokens + 1);
|
|
2000
|
-
for (let token = 0; token < tokens; token++) {
|
|
2001
|
-
const slot = tokenTable.payloadSlots[token];
|
|
2002
|
-
postingOffsets[token + 1] = postingOffsets[token] + arrays.postingOffsets[slot + 1] - arrays.postingOffsets[slot];
|
|
2003
|
-
}
|
|
2004
|
-
const postings = postingOffsets[tokens];
|
|
2005
|
-
const postingDocIds = new Uint32Array(postings);
|
|
2006
|
-
const postingFrequencies = new Uint16Array(postings);
|
|
2007
|
-
const postingFieldIndices = new Uint16Array(postings);
|
|
2008
|
-
for (let token = 0; token < tokens; token++) {
|
|
2009
|
-
const slot = tokenTable.payloadSlots[token];
|
|
2010
|
-
const start = arrays.postingOffsets[slot];
|
|
2011
|
-
const end = arrays.postingOffsets[slot + 1];
|
|
2012
|
-
postingDocIds.set(arrays.postingDocIds.subarray(start, end), postingOffsets[token]);
|
|
2013
|
-
postingFrequencies.set(arrays.postingFrequencies.subarray(start, end), postingOffsets[token]);
|
|
2014
|
-
postingFieldIndices.set(arrays.postingFieldIndices.subarray(start, end), postingOffsets[token]);
|
|
2015
|
-
}
|
|
2016
|
-
return {
|
|
2017
|
-
postingOffsets,
|
|
2018
|
-
postingDocIds,
|
|
2019
|
-
postingFrequencies,
|
|
2020
|
-
postingFieldIndices,
|
|
2021
|
-
...reorderedPositions(arrays, postingOffsets)
|
|
2022
|
-
};
|
|
2023
|
-
}
|
|
2024
|
-
|
|
2025
|
-
// src/serialization/keyword-segment/encode.ts
|
|
2026
|
-
var TOP_LEVEL_ENTRIES = 28;
|
|
2027
|
-
function writeFieldLengths(out, arrays) {
|
|
2028
|
-
out.array(arrays.fieldLengthNames.length);
|
|
2029
|
-
for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
|
|
2030
|
-
out.map(2).key("field").value(arrays.fieldLengthNames[f]);
|
|
2031
|
-
out.key("lengths").binary(littleEndianBytes(arrays.fieldLengthColumns[f]));
|
|
2032
|
-
}
|
|
2033
|
-
}
|
|
2034
|
-
function writeValueColumns(out, columns) {
|
|
2035
|
-
out.array(columns.length);
|
|
2036
|
-
for (const column of columns) {
|
|
2037
|
-
out.map(3).key("field").value(column.fieldPath);
|
|
2038
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
2039
|
-
out.key("values").binary(littleEndianBytes(column.values));
|
|
2040
|
-
}
|
|
2041
|
-
}
|
|
2042
|
-
function writeBooleanColumns(out, columns) {
|
|
2043
|
-
out.array(columns.length);
|
|
2044
|
-
for (const column of columns) {
|
|
2045
|
-
out.map(3).key("field").value(column.fieldPath);
|
|
2046
|
-
out.key("trueDocuments").binary(littleEndianBytes(column.trueDocs));
|
|
2047
|
-
out.key("falseDocuments").binary(littleEndianBytes(column.falseDocs));
|
|
2048
|
-
}
|
|
2049
|
-
}
|
|
2050
|
-
function writeEnumColumns(out, columns) {
|
|
2051
|
-
out.array(columns.length);
|
|
2052
|
-
for (const column of columns) {
|
|
2053
|
-
out.map(4).key("field").value(column.fieldPath);
|
|
2054
|
-
out.key("values").value(column.values);
|
|
2055
|
-
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
2056
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
2057
|
-
}
|
|
2058
|
-
}
|
|
2059
|
-
function writeGeopointColumns(out, columns) {
|
|
2060
|
-
out.array(columns.length);
|
|
2061
|
-
for (const column of columns) {
|
|
2062
|
-
out.map(4).key("field").value(column.fieldPath);
|
|
2063
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
2064
|
-
out.key("latitudes").binary(littleEndianBytes(column.latitudes));
|
|
2065
|
-
out.key("longitudes").binary(littleEndianBytes(column.longitudes));
|
|
2066
|
-
}
|
|
2067
|
-
}
|
|
2068
|
-
function writePatternColumns(out, columns) {
|
|
2069
|
-
out.array(columns.length);
|
|
2070
|
-
for (const column of columns) {
|
|
2071
|
-
out.map(5).key("field").value(column.fieldPath);
|
|
2072
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
2073
|
-
out.key("runs").binary(littleEndianBytes(column.runs));
|
|
2074
|
-
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
2075
|
-
out.key("postings").binary(littleEndianBytes(column.postings));
|
|
2076
|
-
}
|
|
2077
|
-
}
|
|
2078
|
-
function writeSurfaceForms(out, arrays) {
|
|
2079
|
-
const table = arrays.surfaceTable;
|
|
2080
|
-
if (table === null || table.counts.length === 0) {
|
|
2081
|
-
out.value(null);
|
|
2082
|
-
return;
|
|
2083
|
-
}
|
|
2084
|
-
out.map(3).key("text").binary(table.blob);
|
|
2085
|
-
out.key("offsets").binary(littleEndianBytes(table.offsets));
|
|
2086
|
-
out.key("counts").binary(littleEndianBytes(table.counts));
|
|
2087
|
-
}
|
|
2088
|
-
function writePartialWords(out, arrays) {
|
|
2089
|
-
const table = arrays.partialWords;
|
|
2090
|
-
if (table === null) {
|
|
2091
|
-
out.value(null);
|
|
2092
|
-
return;
|
|
2093
|
-
}
|
|
2094
|
-
out.map(8).key("fields").value(table.fields);
|
|
2095
|
-
out.key("text").binary(table.spellings);
|
|
2096
|
-
out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
|
|
2097
|
-
out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
|
|
2098
|
-
out.key("counts").binary(littleEndianBytes(table.spellingCounts));
|
|
2099
|
-
out.key("runs").binary(littleEndianBytes(table.runs));
|
|
2100
|
-
out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
|
|
2101
|
-
out.key("runEntries").binary(littleEndianBytes(table.runEntries));
|
|
2102
|
-
}
|
|
2103
|
-
function encodeKeywordSegment(contents) {
|
|
2104
|
-
const { arrays } = contents;
|
|
2105
|
-
const postings = postingsInTokenOrder(arrays);
|
|
2106
|
-
const out = new MessagePackChunks();
|
|
2107
|
-
const { writeRecords } = contents;
|
|
2108
|
-
out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
|
|
2109
|
-
out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
|
|
2110
|
-
out.key("schema").value(contents.schema);
|
|
2111
|
-
out.key("tombstones").value(contents.tombstones);
|
|
2112
|
-
out.key("documentCount").value(arrays.documentCount);
|
|
2113
|
-
out.key("ids").binary(arrays.idTable.blob);
|
|
2114
|
-
out.key("idOffsets").binary(littleEndianBytes(arrays.idTable.offsets));
|
|
2115
|
-
out.key("idOrder").binary(littleEndianBytes(arrays.idTable.sortedOrdinals));
|
|
2116
|
-
out.key("documents").binary(arrays.documentTable.blob);
|
|
2117
|
-
out.key("documentOffsets").binary(littleEndianBytes(arrays.documentTable.offsets));
|
|
2118
|
-
out.key("fieldNames").value(arrays.fieldNames);
|
|
2119
|
-
out.key("tokens").binary(arrays.tokenTable.blob);
|
|
2120
|
-
out.key("tokenOffsets").binary(littleEndianBytes(arrays.tokenTable.offsets));
|
|
2121
|
-
out.key("documentFrequencies").binary(littleEndianBytes(arrays.tokenTable.documentFrequencies));
|
|
2122
|
-
out.key("postingOffsets").binary(littleEndianBytes(postings.postingOffsets));
|
|
2123
|
-
out.key("postingDocuments").binary(littleEndianBytes(postings.postingDocIds));
|
|
2124
|
-
out.key("postingFrequencies").binary(littleEndianBytes(postings.postingFrequencies));
|
|
2125
|
-
out.key("postingFields").binary(littleEndianBytes(postings.postingFieldIndices));
|
|
2126
|
-
out.key("positionOffsets").binaryOrNil(postings.positionOffsets === null ? null : littleEndianBytes(postings.positionOffsets));
|
|
2127
|
-
out.key("positions").binaryOrNil(postings.positionValues === null ? null : littleEndianBytes(postings.positionValues));
|
|
2128
|
-
out.key("fieldLengths");
|
|
2129
|
-
writeFieldLengths(out, arrays);
|
|
2130
|
-
out.key("numeric");
|
|
2131
|
-
writeValueColumns(out, arrays.numeric);
|
|
2132
|
-
out.key("sizes");
|
|
2133
|
-
writeValueColumns(out, arrays.sizes);
|
|
2134
|
-
out.key("boolean");
|
|
2135
|
-
writeBooleanColumns(out, arrays.boolean);
|
|
2136
|
-
out.key("enum");
|
|
2137
|
-
writeEnumColumns(out, arrays.enums);
|
|
2138
|
-
out.key("geopoint");
|
|
2139
|
-
writeGeopointColumns(out, arrays.geo);
|
|
2140
|
-
out.key("pattern");
|
|
2141
|
-
writePatternColumns(out, arrays.pattern);
|
|
2142
|
-
out.key("surfaceForms");
|
|
2143
|
-
writeSurfaceForms(out, arrays);
|
|
2144
|
-
out.key("partialWords");
|
|
2145
|
-
writePartialWords(out, arrays);
|
|
2146
|
-
if (writeRecords !== void 0) {
|
|
2147
|
-
out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
|
|
2148
|
-
out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
|
|
2149
|
-
}
|
|
2150
|
-
return out;
|
|
2151
|
-
}
|
|
2152
|
-
|
|
2153
|
-
// src/persistence/durability/segment/segment-file.ts
|
|
2154
|
-
function sharesMemoryWithWorkers() {
|
|
2155
|
-
return typeof SharedArrayBuffer === "function";
|
|
2156
|
-
}
|
|
2157
|
-
async function readSegmentFile(store, ref) {
|
|
2158
|
-
const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "segment" });
|
|
2159
|
-
const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
|
|
2160
|
-
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
2161
|
-
what: `segment "${ref.key}"`,
|
|
2162
|
-
allowsCompression: true
|
|
2163
|
-
});
|
|
2164
|
-
requireListedChecksum("segment", ref.key, header2.checksum, ref.checksum);
|
|
2165
|
-
return decodeKeywordSegment(payload, sharesMemoryWithWorkers());
|
|
2166
|
-
}
|
|
2167
|
-
var ORDINAL_BYTES = 4;
|
|
2168
|
-
var POSTING_BYTES = 8;
|
|
2169
|
-
var VALUE_ENTRY_BYTES = 12;
|
|
2170
|
-
var GEOPOINT_ENTRY_BYTES = 20;
|
|
2171
|
-
var TOKEN_ENTRY_BYTES = 12;
|
|
2172
|
-
var STRING_HEADER_BYTES = 5;
|
|
2173
|
-
var encoder = new TextEncoder();
|
|
2174
|
-
function documentBytes(segment, ordinal) {
|
|
2175
|
-
const table = segment.arrays.documentTable;
|
|
2176
|
-
if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
|
|
2177
|
-
return messagePackUpperBoundBytes(segment.documentSource.docAt(ordinal));
|
|
2178
|
-
}
|
|
2179
|
-
function addPerEntry(costs, docIds, bytes) {
|
|
2180
|
-
for (let at = 0; at < docIds.length; at++) costs[docIds[at]] += bytes;
|
|
2181
|
-
}
|
|
2182
|
-
function chargePostings(segment, costs, chargesTokens) {
|
|
2183
|
-
const arrays = segment.arrays;
|
|
2184
|
-
const positions = arrays.positionOffsets;
|
|
2185
|
-
for (let p = 0; p < arrays.postingDocIds.length; p++) {
|
|
2186
|
-
let bytes = POSTING_BYTES;
|
|
2187
|
-
if (positions !== null) bytes += ORDINAL_BYTES * (1 + positions[p + 1] - positions[p]);
|
|
2188
|
-
costs[arrays.postingDocIds[p]] += bytes;
|
|
2189
|
-
}
|
|
2190
|
-
if (!chargesTokens) return;
|
|
2191
|
-
const table = arrays.tokenTable;
|
|
2192
|
-
for (let at = 0; at < table.size; at++) {
|
|
2193
|
-
const slot = table.payloadSlot(at);
|
|
2194
|
-
const bytes = table.byteLengthAt(at) + TOKEN_ENTRY_BYTES;
|
|
2195
|
-
for (let p = arrays.postingOffsets[slot]; p < arrays.postingOffsets[slot + 1]; p++) {
|
|
2196
|
-
costs[arrays.postingDocIds[p]] += bytes;
|
|
2197
|
-
}
|
|
2198
|
-
}
|
|
2199
|
-
}
|
|
2200
|
-
function chargeColumns(segment, costs) {
|
|
2201
|
-
const arrays = segment.arrays;
|
|
2202
|
-
for (const column of [...arrays.numeric, ...arrays.sizes]) addPerEntry(costs, column.docIds, VALUE_ENTRY_BYTES);
|
|
2203
|
-
for (const column of arrays.boolean) {
|
|
2204
|
-
addPerEntry(costs, column.trueDocs, ORDINAL_BYTES);
|
|
2205
|
-
addPerEntry(costs, column.falseDocs, ORDINAL_BYTES);
|
|
2206
|
-
}
|
|
2207
|
-
for (const column of arrays.enums) addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
2208
|
-
for (const column of arrays.geo) addPerEntry(costs, column.docIds, GEOPOINT_ENTRY_BYTES);
|
|
2209
|
-
for (const column of arrays.pattern) {
|
|
2210
|
-
addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
2211
|
-
addPerEntry(costs, column.postings, ORDINAL_BYTES);
|
|
2212
|
-
}
|
|
2213
|
-
}
|
|
2214
|
-
function stringListBytes(values) {
|
|
2215
|
-
let bytes = STRING_HEADER_BYTES;
|
|
2216
|
-
for (const value of values) bytes += STRING_HEADER_BYTES + encoder.encode(value).byteLength;
|
|
2217
|
-
return bytes;
|
|
2218
|
-
}
|
|
2219
|
-
function fixedBytesOf(segment, chargesTokens) {
|
|
2220
|
-
const arrays = segment.arrays;
|
|
2221
|
-
let bytes = stringListBytes(arrays.fieldNames);
|
|
2222
|
-
if (!chargesTokens) bytes += arrays.tokenTable.bytes;
|
|
2223
|
-
for (const column of arrays.enums) bytes += stringListBytes(column.values) + column.offsets.byteLength;
|
|
2224
|
-
for (const column of arrays.pattern) bytes += column.runs.byteLength + column.offsets.byteLength;
|
|
2225
|
-
const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
|
|
2226
|
-
if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
|
|
2227
|
-
else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
|
|
2228
|
-
return bytes + segment.partialWords.bytes();
|
|
2229
|
-
}
|
|
2230
|
-
function keptDocumentsOf(selection, chargesTokens) {
|
|
2231
|
-
const { segment } = selection;
|
|
2232
|
-
const count = segment.arrays.documentCount;
|
|
2233
|
-
const costs = new Float64Array(count);
|
|
2234
|
-
const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
|
|
2235
|
-
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
2236
|
-
costs[ordinal] = perDocument + encoder.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
|
|
2237
|
-
}
|
|
2238
|
-
chargePostings(segment, costs, chargesTokens);
|
|
2239
|
-
chargeColumns(segment, costs);
|
|
2240
|
-
const ordinals = [];
|
|
2241
|
-
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
2242
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
2243
|
-
costs[ordinal] += documentBytes(segment, ordinal);
|
|
2244
|
-
ordinals.push(ordinal);
|
|
2245
|
-
}
|
|
2246
|
-
return { segment, ordinals, costs };
|
|
2247
|
-
}
|
|
2248
|
-
function selectionOf(segment, ordinals) {
|
|
2249
|
-
const marked = new Uint8Array(segment.arrays.documentCount);
|
|
2250
|
-
for (const ordinal of ordinals) marked[ordinal] = 1;
|
|
2251
|
-
return { segment, keep: (ordinal) => marked[ordinal] === 1 };
|
|
2252
|
-
}
|
|
2253
|
-
function tooLargeForOneFile(cost, budget) {
|
|
2254
|
-
throw new NarsilError(
|
|
2255
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2256
|
-
`A document needs up to ${Math.ceil(cost)} bytes of a segment, more than the ${Math.max(0, Math.floor(budget))} that one file leaves it`,
|
|
2257
|
-
{ bytes: Math.ceil(cost), budget: Math.floor(budget), maximum: MAX_FILE_BYTES }
|
|
2258
|
-
);
|
|
2259
|
-
}
|
|
2260
|
-
function splitIntoSegmentFiles(selections, schema, tombstones) {
|
|
2261
|
-
let tokenBytes = 0;
|
|
2262
|
-
for (const { segment } of selections) tokenBytes += segment.arrays.tokenTable.bytes;
|
|
2263
|
-
const chargesTokens = tokenBytes > MAX_FILE_BYTES * SEGMENT_TOKEN_TABLE_SHARE_OF_FILE;
|
|
2264
|
-
let fixed = HEADER_SIZE + SEGMENT_FILE_FRAMING_SLACK_BYTES + encode(schema).byteLength + encode(tombstones).byteLength;
|
|
2265
|
-
for (const { segment } of selections) fixed += fixedBytesOf(segment, chargesTokens);
|
|
2266
|
-
const budget = MAX_FILE_BYTES - fixed;
|
|
2267
|
-
const groups = [];
|
|
2268
|
-
let current = [];
|
|
2269
|
-
let currentBytes = 0;
|
|
2270
|
-
let currentCount = 0;
|
|
2271
|
-
const flush = () => {
|
|
2272
|
-
if (currentCount === 0) return;
|
|
2273
|
-
groups.push({
|
|
2274
|
-
selections: current.map((entry) => selectionOf(entry.segment, entry.ordinals)),
|
|
2275
|
-
documentCount: currentCount
|
|
2276
|
-
});
|
|
2277
|
-
current = [];
|
|
2278
|
-
currentBytes = 0;
|
|
2279
|
-
currentCount = 0;
|
|
2280
|
-
};
|
|
2281
|
-
for (const selection of selections) {
|
|
2282
|
-
const kept = keptDocumentsOf(selection, chargesTokens);
|
|
2283
|
-
let entry = null;
|
|
2284
|
-
for (const ordinal of kept.ordinals) {
|
|
2285
|
-
const cost = kept.costs[ordinal];
|
|
2286
|
-
if (cost > budget) tooLargeForOneFile(cost, budget);
|
|
2287
|
-
if (currentBytes + cost > budget) {
|
|
2288
|
-
flush();
|
|
2289
|
-
entry = null;
|
|
2290
|
-
}
|
|
2291
|
-
if (entry === null) {
|
|
2292
|
-
entry = { segment: kept.segment, ordinals: [] };
|
|
2293
|
-
current.push(entry);
|
|
2294
|
-
}
|
|
2295
|
-
entry.ordinals.push(ordinal);
|
|
2296
|
-
currentBytes += cost;
|
|
2297
|
-
currentCount += 1;
|
|
2298
|
-
}
|
|
2299
|
-
}
|
|
2300
|
-
flush();
|
|
2301
|
-
return groups;
|
|
2302
|
-
}
|
|
2303
|
-
function halvesOf(group) {
|
|
2304
|
-
const flat = [];
|
|
2305
|
-
for (const selection of group.selections) {
|
|
2306
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
2307
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
2308
|
-
flat.push({ segment: selection.segment, ordinal });
|
|
2309
|
-
}
|
|
2310
|
-
}
|
|
2311
|
-
const middle = Math.ceil(flat.length / 2);
|
|
2312
|
-
const groupOf = (entries) => {
|
|
2313
|
-
const bySegment = /* @__PURE__ */ new Map();
|
|
2314
|
-
for (const { segment, ordinal } of entries) {
|
|
2315
|
-
let ordinals = bySegment.get(segment);
|
|
2316
|
-
if (ordinals === void 0) {
|
|
2317
|
-
ordinals = [];
|
|
2318
|
-
bySegment.set(segment, ordinals);
|
|
2319
|
-
}
|
|
2320
|
-
ordinals.push(ordinal);
|
|
2321
|
-
}
|
|
2322
|
-
return {
|
|
2323
|
-
selections: [...bySegment].map(([segment, ordinals]) => selectionOf(segment, ordinals)),
|
|
2324
|
-
documentCount: entries.length
|
|
2325
|
-
};
|
|
2326
|
-
};
|
|
2327
|
-
return [groupOf(flat.slice(0, middle)), groupOf(flat.slice(middle))];
|
|
2328
|
-
}
|
|
2329
|
-
|
|
2330
|
-
// src/persistence/durability/segment/segment-write.ts
|
|
2331
|
-
function emptyPayload() {
|
|
2332
|
-
return {
|
|
2333
|
-
documentCount: 0,
|
|
2334
|
-
docIds: [],
|
|
2335
|
-
fieldNames: [],
|
|
2336
|
-
tokens: [],
|
|
2337
|
-
postingOffsets: new Uint32Array(1),
|
|
2338
|
-
postingDocIds: new Uint32Array(0),
|
|
2339
|
-
postingFrequencies: new Uint16Array(0),
|
|
2340
|
-
postingFieldIndices: new Uint16Array(0),
|
|
2341
|
-
positionOffsets: null,
|
|
2342
|
-
positionValues: null,
|
|
2343
|
-
fieldLengthNames: [],
|
|
2344
|
-
fieldLengthColumns: [],
|
|
2345
|
-
totalFieldLengths: {},
|
|
2346
|
-
docFrequencies: {},
|
|
2347
|
-
surfaceForms: null,
|
|
2348
|
-
partialSpellings: null,
|
|
2349
|
-
numeric: [],
|
|
2350
|
-
boolean: [],
|
|
2351
|
-
enums: [],
|
|
2352
|
-
geo: [],
|
|
2353
|
-
pattern: [],
|
|
2354
|
-
sizes: []
|
|
2355
|
-
};
|
|
2356
|
-
}
|
|
2357
|
-
function noSpellingsOf(analysis) {
|
|
2358
|
-
if (analysis === null || analysis.fields.length === 0) return null;
|
|
2359
|
-
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
2360
|
-
}
|
|
2361
|
-
function writeRecordsOfGroup(group, records, count) {
|
|
2362
|
-
const seqNos = new Float64Array(count);
|
|
2363
|
-
const primaryTerms = new Float64Array(count);
|
|
2364
|
-
let at = 0;
|
|
2365
|
-
for (const selection of group?.selections ?? []) {
|
|
2366
|
-
const source = records.get(selection.segment);
|
|
2367
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
2368
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
2369
|
-
if (source !== void 0) {
|
|
2370
|
-
seqNos[at] = source.seqNos[ordinal];
|
|
2371
|
-
primaryTerms[at] = source.primaryTerms[ordinal];
|
|
2372
|
-
}
|
|
2373
|
-
at += 1;
|
|
2374
|
-
}
|
|
2375
|
-
}
|
|
2376
|
-
return { seqNos, primaryTerms };
|
|
2377
|
-
}
|
|
2378
|
-
function withWriteRecords(contents, group, request) {
|
|
2379
|
-
if (request.writeRecords === void 0) return contents;
|
|
2380
|
-
const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
|
|
2381
|
-
return { ...contents, writeRecords };
|
|
2382
|
-
}
|
|
2383
|
-
function contentsOfGroup(group, request, tombstones) {
|
|
2384
|
-
const { schema, partialWordAnalysis, textAnalysis } = request;
|
|
2385
|
-
if (group === null) {
|
|
2386
|
-
const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
|
|
2387
|
-
return {
|
|
2388
|
-
schema,
|
|
2389
|
-
tombstones,
|
|
2390
|
-
arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
|
|
2391
|
-
};
|
|
2392
|
-
}
|
|
2393
|
-
const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
|
|
2394
|
-
return {
|
|
2395
|
-
schema,
|
|
2396
|
-
tombstones,
|
|
2397
|
-
arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
|
|
2398
|
-
};
|
|
2399
|
-
}
|
|
2400
|
-
function requireRoomForAnotherSegment(state) {
|
|
2401
|
-
if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
|
|
2402
|
-
throw new NarsilError(
|
|
2403
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2404
|
-
`Partition ${state.request.partitionId} would hold more than ${MAX_SEGMENTS_PER_PARTITION} segments`,
|
|
2405
|
-
{ partitionId: state.request.partitionId, maximum: MAX_SEGMENTS_PER_PARTITION }
|
|
2406
|
-
);
|
|
2407
|
-
}
|
|
2408
|
-
async function writeGroup(state, group) {
|
|
2409
|
-
const { request } = state;
|
|
2410
|
-
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
2411
|
-
const contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
|
|
2412
|
-
const encoded = encodeKeywordSegment(contents);
|
|
2413
|
-
if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
|
|
2414
|
-
const [first, second] = halvesOf(group);
|
|
2415
|
-
await writeGroup(state, first);
|
|
2416
|
-
await writeGroup(state, second);
|
|
2417
|
-
return;
|
|
2418
|
-
}
|
|
2419
|
-
requireRoomForAnotherSegment(state);
|
|
2420
|
-
const id = state.nextSegmentId;
|
|
2421
|
-
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
2422
|
-
const packed = await packChecksummedEnvelope(encoded.chunks, KEYWORD_SEGMENT_ENVELOPE_VERSION, `segment "${key}"`);
|
|
2423
|
-
await request.store.write(key, packed.chunks);
|
|
2424
|
-
state.segments.push({
|
|
2425
|
-
id,
|
|
2426
|
-
key,
|
|
2427
|
-
bytes: packed.byteLength,
|
|
2428
|
-
checksum: packed.checksum,
|
|
2429
|
-
docCount: contents.arrays.documentCount,
|
|
2430
|
-
tombstoneCount: tombstones.length
|
|
2431
|
-
});
|
|
2432
|
-
state.nextSegmentId = id + 1;
|
|
2433
|
-
state.tombstonesWritten = true;
|
|
2434
|
-
}
|
|
2435
|
-
function keptDocumentCount(selections) {
|
|
2436
|
-
let count = 0;
|
|
2437
|
-
for (const selection of selections) {
|
|
2438
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
2439
|
-
if (selectionKeeps(selection, ordinal)) count += 1;
|
|
2440
|
-
}
|
|
2441
|
-
}
|
|
2442
|
-
return count;
|
|
2443
|
-
}
|
|
2444
|
-
async function writeSegmentsOf(request) {
|
|
2445
|
-
const state = { request, segments: [], nextSegmentId: request.nextSegmentId, tombstonesWritten: false };
|
|
2446
|
-
const groups = request.mergesIntoOneFile === true ? [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }] : splitIntoSegmentFiles(request.selections, request.schema, request.tombstones);
|
|
2447
|
-
for (const group of groups) {
|
|
2448
|
-
if (group.documentCount > 0) await writeGroup(state, group);
|
|
2449
|
-
}
|
|
2450
|
-
if (!state.tombstonesWritten && request.tombstones.length > 0) await writeGroup(state, null);
|
|
2451
|
-
return { segments: state.segments, nextSegmentId: state.nextSegmentId };
|
|
2452
|
-
}
|
|
2453
2066
|
|
|
2454
2067
|
// src/persistence/durability/segment/compaction.ts
|
|
2455
|
-
function windowFits(segments, start, windowLength) {
|
|
2068
|
+
function windowFits(segments, start, windowLength, paged) {
|
|
2456
2069
|
let documents = 0;
|
|
2457
2070
|
let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
|
|
2458
2071
|
for (let at = start; at < start + windowLength; at++) {
|
|
2459
2072
|
documents += segments[at].docCount;
|
|
2460
2073
|
bytes += segments[at].bytes;
|
|
2461
2074
|
}
|
|
2075
|
+
if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
|
|
2462
2076
|
return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
|
|
2463
2077
|
}
|
|
2464
|
-
function smallestFittingWindow(segments, windowLength) {
|
|
2078
|
+
function smallestFittingWindow(segments, windowLength, paged) {
|
|
2465
2079
|
let best = null;
|
|
2466
2080
|
let bestBytes = Number.POSITIVE_INFINITY;
|
|
2467
2081
|
for (let start = 0; start + windowLength <= segments.length; start++) {
|
|
2468
|
-
if (!windowFits(segments, start, windowLength)) continue;
|
|
2082
|
+
if (!windowFits(segments, start, windowLength, paged)) continue;
|
|
2469
2083
|
let bytes = 0;
|
|
2470
2084
|
for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
|
|
2471
2085
|
if (bytes < bestBytes) {
|
|
@@ -2475,9 +2089,9 @@ function smallestFittingWindow(segments, windowLength) {
|
|
|
2475
2089
|
}
|
|
2476
2090
|
return best;
|
|
2477
2091
|
}
|
|
2478
|
-
function chooseWindow(segments, wantedLength) {
|
|
2092
|
+
function chooseWindow(segments, wantedLength, paged) {
|
|
2479
2093
|
for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
|
|
2480
|
-
const window = smallestFittingWindow(segments, windowLength);
|
|
2094
|
+
const window = smallestFittingWindow(segments, windowLength, paged);
|
|
2481
2095
|
if (window !== null) return window;
|
|
2482
2096
|
}
|
|
2483
2097
|
return null;
|
|
@@ -2518,11 +2132,39 @@ async function compactPartitionSegments(input) {
|
|
|
2518
2132
|
if (segments.length <= compactionThreshold) {
|
|
2519
2133
|
return { segments, nextSegmentId: input.nextSegmentId };
|
|
2520
2134
|
}
|
|
2521
|
-
const
|
|
2135
|
+
const paged = input.paged === true;
|
|
2136
|
+
const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
|
|
2522
2137
|
if (window === null) {
|
|
2523
2138
|
return { segments, nextSegmentId: input.nextSegmentId };
|
|
2524
2139
|
}
|
|
2525
2140
|
const windowRefs = segments.slice(window.start, window.start + window.windowLength);
|
|
2141
|
+
const replacing = (written2) => ({
|
|
2142
|
+
segments: [
|
|
2143
|
+
...segments.slice(0, window.start),
|
|
2144
|
+
...written2.segments,
|
|
2145
|
+
...segments.slice(window.start + window.windowLength)
|
|
2146
|
+
],
|
|
2147
|
+
nextSegmentId: written2.nextSegmentId
|
|
2148
|
+
});
|
|
2149
|
+
if (paged) {
|
|
2150
|
+
const { directory } = input.store;
|
|
2151
|
+
if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
|
|
2152
|
+
const { mergePagedWindow } = await import('./module-BUFNZMEU.mjs');
|
|
2153
|
+
return replacing(
|
|
2154
|
+
await mergePagedWindow({
|
|
2155
|
+
store: input.store,
|
|
2156
|
+
directory,
|
|
2157
|
+
indexName: input.indexName,
|
|
2158
|
+
partitionId: input.partitionId,
|
|
2159
|
+
schema: input.schema,
|
|
2160
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
2161
|
+
textAnalysis: input.textAnalysis,
|
|
2162
|
+
nextSegmentId: input.nextSegmentId,
|
|
2163
|
+
refs: windowRefs,
|
|
2164
|
+
dropsEveryTombstone: window.start === 0
|
|
2165
|
+
})
|
|
2166
|
+
);
|
|
2167
|
+
}
|
|
2526
2168
|
const contents = [];
|
|
2527
2169
|
for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
|
|
2528
2170
|
const frozen = contents.map(frozenOf);
|
|
@@ -2541,14 +2183,7 @@ async function compactPartitionSegments(input) {
|
|
|
2541
2183
|
mergesIntoOneFile: true,
|
|
2542
2184
|
...writeRecords === void 0 ? {} : { writeRecords }
|
|
2543
2185
|
});
|
|
2544
|
-
return
|
|
2545
|
-
segments: [
|
|
2546
|
-
...segments.slice(0, window.start),
|
|
2547
|
-
...written.segments,
|
|
2548
|
-
...segments.slice(window.start + window.windowLength)
|
|
2549
|
-
],
|
|
2550
|
-
nextSegmentId: written.nextSegmentId
|
|
2551
|
-
};
|
|
2186
|
+
return replacing(written);
|
|
2552
2187
|
}
|
|
2553
2188
|
|
|
2554
2189
|
// src/persistence/durability/segment/write.ts
|
|
@@ -2595,6 +2230,21 @@ function wholeContentsOf(input, partitionId) {
|
|
|
2595
2230
|
const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
|
|
2596
2231
|
return captured === void 0 ? void 0 : contentsOfCaptured(captured);
|
|
2597
2232
|
}
|
|
2233
|
+
function pagedSettingsOf(context) {
|
|
2234
|
+
if (context.input.metadata.textStorage !== "disk") return {};
|
|
2235
|
+
return { paged: true, keepsWriteRecords: context.input.keepsWriteRecords === true };
|
|
2236
|
+
}
|
|
2237
|
+
async function tombstonesOfDelta(context, older, selections, deleted) {
|
|
2238
|
+
const { directory } = context.input.store;
|
|
2239
|
+
if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
|
|
2240
|
+
const { pagedTombstonesFor } = await import('./module-BUFNZMEU.mjs');
|
|
2241
|
+
return pagedTombstonesFor(
|
|
2242
|
+
directory,
|
|
2243
|
+
older,
|
|
2244
|
+
selections.map((selection) => selection.segment),
|
|
2245
|
+
deleted
|
|
2246
|
+
);
|
|
2247
|
+
}
|
|
2598
2248
|
async function writeWholePartition(context, partitionId, priorPartition, whole) {
|
|
2599
2249
|
const written = await writeSegmentsOf({
|
|
2600
2250
|
store: context.input.store,
|
|
@@ -2606,7 +2256,8 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
|
|
|
2606
2256
|
textAnalysis: context.textAnalysis,
|
|
2607
2257
|
selections: whole.selections,
|
|
2608
2258
|
tombstones: [],
|
|
2609
|
-
...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords }
|
|
2259
|
+
...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords },
|
|
2260
|
+
...pagedSettingsOf(context)
|
|
2610
2261
|
});
|
|
2611
2262
|
return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
|
|
2612
2263
|
}
|
|
@@ -2622,6 +2273,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
2622
2273
|
let segments = priorPartition ? [...priorPartition.segments] : [];
|
|
2623
2274
|
let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
|
|
2624
2275
|
if (built !== null) {
|
|
2276
|
+
const tombstones = await tombstonesOfDelta(context, segments, built.selections, built.tombstones);
|
|
2625
2277
|
const written = await writeSegmentsOf({
|
|
2626
2278
|
store: context.input.store,
|
|
2627
2279
|
indexName: context.indexName,
|
|
@@ -2631,8 +2283,9 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
2631
2283
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
2632
2284
|
textAnalysis: context.textAnalysis,
|
|
2633
2285
|
selections: built.selections,
|
|
2634
|
-
tombstones
|
|
2635
|
-
...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords }
|
|
2286
|
+
tombstones,
|
|
2287
|
+
...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords },
|
|
2288
|
+
...pagedSettingsOf(context)
|
|
2636
2289
|
});
|
|
2637
2290
|
segments.push(...written.segments);
|
|
2638
2291
|
nextSegmentId = written.nextSegmentId;
|
|
@@ -2653,7 +2306,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
2653
2306
|
textAnalysis: context.textAnalysis,
|
|
2654
2307
|
segments,
|
|
2655
2308
|
nextSegmentId,
|
|
2656
|
-
compactionThreshold: context.compactionThreshold
|
|
2309
|
+
compactionThreshold: context.compactionThreshold,
|
|
2310
|
+
paged: context.input.metadata.textStorage === "disk"
|
|
2657
2311
|
});
|
|
2658
2312
|
segments = compacted.segments;
|
|
2659
2313
|
nextSegmentId = compacted.nextSegmentId;
|
|
@@ -2716,6 +2370,7 @@ async function writeCheckpointSegments(input) {
|
|
|
2716
2370
|
async function commitCheckpointManifest(store, metadata, written, vectors) {
|
|
2717
2371
|
const indexName = metadata.indexName;
|
|
2718
2372
|
const manifest = {
|
|
2373
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
2719
2374
|
schema: metadata.schema,
|
|
2720
2375
|
language: metadata.language,
|
|
2721
2376
|
...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
|
|
@@ -2769,8 +2424,41 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
|
2769
2424
|
function yieldToEventLoop() {
|
|
2770
2425
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
2771
2426
|
}
|
|
2427
|
+
function removeFromPartition(manager, partitionId, docIds) {
|
|
2428
|
+
const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
|
|
2429
|
+
composite.beginBatch();
|
|
2430
|
+
try {
|
|
2431
|
+
for (const docId of docIds) {
|
|
2432
|
+
if (!composite.has(docId)) continue;
|
|
2433
|
+
composite.remove(docId, manager.config.schema, manager.language, insertOptions);
|
|
2434
|
+
docPartitionMap.delete(docId);
|
|
2435
|
+
}
|
|
2436
|
+
} finally {
|
|
2437
|
+
composite.endBatch();
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
2441
|
+
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-BUFNZMEU.mjs');
|
|
2442
|
+
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
2443
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
2444
|
+
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
2445
|
+
const segment = createPagedFrozenSegment(snapshot);
|
|
2446
|
+
removeFromPartition(manager, partitionId, snapshot.tombstones);
|
|
2447
|
+
options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
|
|
2448
|
+
if (snapshot.documentCount > 0) manager.attachFrozenSegment(partitionId, segment);
|
|
2449
|
+
else segment.paged?.close();
|
|
2450
|
+
}
|
|
2772
2451
|
async function loadPartitionSegments(store, partition, manager, options) {
|
|
2452
|
+
const { directory } = store;
|
|
2773
2453
|
for (const ref of partition.segments) {
|
|
2454
|
+
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
2455
|
+
const { isPagedSegmentFile } = await import('./module-BUFNZMEU.mjs');
|
|
2456
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
2457
|
+
await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
2458
|
+
await yieldToEventLoop();
|
|
2459
|
+
continue;
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2774
2462
|
const contents = await readSegmentFile(store, ref);
|
|
2775
2463
|
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
2776
2464
|
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
@@ -2815,6 +2503,14 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
2815
2503
|
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
2816
2504
|
}
|
|
2817
2505
|
growPartitionsTo(deps.manager, highestPartitionId);
|
|
2506
|
+
const { directory } = store;
|
|
2507
|
+
const opensVectorsOnDemand = manifest.vectors.some(
|
|
2508
|
+
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
2509
|
+
);
|
|
2510
|
+
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
2511
|
+
const { startReadingOpeningSectionsAhead } = await import('./module-BUFNZMEU.mjs');
|
|
2512
|
+
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
2513
|
+
}
|
|
2818
2514
|
for (const partition of manifest.partitions) {
|
|
2819
2515
|
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
2820
2516
|
throw new NarsilError(
|
|
@@ -2832,7 +2528,12 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
2832
2528
|
for (const vector of manifest.vectors) {
|
|
2833
2529
|
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
2834
2530
|
if (vectorIndex === void 0) continue;
|
|
2835
|
-
await loadVectorField(
|
|
2531
|
+
await loadVectorField(
|
|
2532
|
+
store,
|
|
2533
|
+
vector,
|
|
2534
|
+
vectorIndex,
|
|
2535
|
+
decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
|
|
2536
|
+
);
|
|
2836
2537
|
}
|
|
2837
2538
|
return manifest.checkpoint;
|
|
2838
2539
|
}
|
|
@@ -2941,4 +2642,4 @@ function deserializePayloadV2(data) {
|
|
|
2941
2642
|
};
|
|
2942
2643
|
}
|
|
2943
2644
|
|
|
2944
|
-
export {
|
|
2645
|
+
export { SEGMENT_HEADER_SIZE, SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, deserializePayloadV2, documentsOfPayloadV2, durableMetadataConfigOf, durableRecordMissing, encodeMarkerSlot, frameRecord, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadSegmentedCheckpoint, markerSlotOffset, metadataConfigOf, metadataFromConfig, nextMarkerSlot, nextSegmentKey, openManifestPayload, partitionContentsOf, readCommitMarker, readCurrentManifest, readGraphParts, readMetadataEnvelope, readSegmentManifest, readVectorFile, readWalSegments, reclaimUnlistedKeys, removeCheckpointGarbage, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeCheckpointSegments, writeLiveVectors, writeMetadataEnvelope, writeSegmentHeader };
|