@delali/narsil 0.3.1-next.185 → 0.3.1-next.193
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/{chunk-4ZICWS4V.mjs → chunk-QAOVAYAG.mjs} +1 -1
- package/dist/browser/{chunk-CPSHS46N.mjs → chunk-UQG4HUGQ.mjs} +82 -50
- package/dist/browser/{chunk-2Q45HI5H.mjs → chunk-YINZFPLT.mjs} +7 -0
- package/dist/browser/{module-XVLHJ7T7.mjs → module-ZZTNG5JZ.mjs} +26 -3
- package/dist/browser/{optional-writes-ZTBHYQMN.mjs → optional-writes-J5BHKEQL.mjs} +21 -8
- package/dist/browser/{storage-leaf-4XMCITJ6.mjs → storage-leaf-N3JMQ6CF.mjs} +14569 -13747
- package/dist/{capabilities-DAHXphRn.d.ts → capabilities-CLnQHIqV.d.ts} +51 -9
- package/dist/{chunk-KLVMK3R5.mjs → chunk-2CKDCC65.mjs} +1 -1
- package/dist/{chunk-CODGHJVW.mjs → chunk-2HVQYOOD.mjs} +40 -12
- package/dist/{chunk-M5I64AQW.mjs → chunk-3N5GUXBI.mjs} +33 -33
- package/dist/{chunk-DU23J5JG.mjs → chunk-5CIHQIA4.mjs} +6 -6
- package/dist/{react-native/chunks/chunk-ZUFOYZ5P.mjs → chunk-62X472AC.mjs} +1 -1
- package/dist/{chunk-QJGDB3WR.mjs → chunk-6GHX35ZM.mjs} +2 -2
- package/dist/{chunk-G3XRHAJA.mjs → chunk-6MC6BCXK.mjs} +3 -3
- package/dist/{chunk-X5NHWOXT.mjs → chunk-7P5YWDRQ.mjs} +2 -2
- package/dist/{chunk-MHBG33R5.mjs → chunk-7X4IIREW.mjs} +76 -46
- package/dist/{chunk-F4OOBPKA.mjs → chunk-AOYQTX6W.mjs} +1 -1
- package/dist/{chunk-D6FYUTFN.mjs → chunk-ASZLNVWW.mjs} +1 -1
- package/dist/{chunk-LR6JJVML.mjs → chunk-D2EJUM4A.mjs} +764 -699
- package/dist/{chunk-FETZYCH7.mjs → chunk-F6F4G322.mjs} +26 -1
- package/dist/{chunk-2YWWHMVC.mjs → chunk-FGOGT273.mjs} +2 -2
- package/dist/{chunk-5BUEBIXR.mjs → chunk-GLXLSPLO.mjs} +18 -5
- package/dist/{chunk-IE3Q62QD.mjs → chunk-GR5GMFJJ.mjs} +1 -1
- package/dist/{chunk-Y7QUFIGV.mjs → chunk-GXED6R3Q.mjs} +2 -2
- package/dist/{chunk-BVZWWMOU.mjs → chunk-HHYQMGPW.mjs} +2 -2
- package/dist/{chunk-SL53DHVE.mjs → chunk-HSA3H3VW.mjs} +121 -68
- package/dist/{chunk-YM2HI5N5.mjs → chunk-IL5DRCNU.mjs} +2 -2
- package/dist/{chunk-JDG6CCXM.mjs → chunk-JENWA4WX.mjs} +2 -2
- package/dist/{chunk-2GONI5HL.mjs → chunk-JXINPXRS.mjs} +5 -4
- package/dist/chunk-K5GNK5GE.mjs +602 -0
- package/dist/{chunk-WLFOLAQI.mjs → chunk-KBU3ZFJN.mjs} +2 -2
- package/dist/{chunk-EDEE7ETQ.mjs → chunk-KOA6T2IC.mjs} +3 -3
- package/dist/{chunk-N3M4NPML.mjs → chunk-M2Z5WVWW.mjs} +1 -1
- package/dist/{chunk-EWGIF6FQ.mjs → chunk-NJJN4FMM.mjs} +1 -1
- package/dist/{chunk-C7YQHCSO.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/{chunk-VYSWVQCE.mjs → chunk-PAPQGC6I.mjs} +2 -2
- package/dist/{chunk-I2WJCRIH.mjs → chunk-R2PJKSNL.mjs} +1 -1
- package/dist/{chunk-6SZQETSC.mjs → chunk-RORNUNXW.mjs} +5 -2
- package/dist/{chunk-FBQYC5OX.mjs → chunk-SB5HCIIO.mjs} +16 -13
- package/dist/{chunk-CYCVPTTR.mjs → chunk-SCZI7COG.mjs} +2 -2
- package/dist/{chunk-O337MDRJ.mjs → chunk-SHES4VP5.mjs} +38 -13
- package/dist/{chunk-H4QYQ5DE.mjs → chunk-SIIDIGKI.mjs} +2 -2
- package/dist/{chunk-A3OAFUZI.mjs → chunk-SM2SYVIX.mjs} +2 -2
- package/dist/{chunk-II6XF6M4.mjs → chunk-SMLUZGSI.mjs} +9 -8
- package/dist/{chunk-2KCCQK4R.mjs → chunk-TSE2VWPX.mjs} +2 -2
- package/dist/{chunk-WSIU6YXQ.mjs → chunk-YPEJ6ODL.mjs} +1732 -743
- package/dist/{chunk-7WXGUTWK.mjs → chunk-Z6SIJH5O.mjs} +17 -6
- package/dist/{chunk-Q3ZZH5Z7.mjs → chunk-ZKOQR7EA.mjs} +7 -0
- package/dist/{chunk-CKEOTSF5.mjs → chunk-ZS2YUB2L.mjs} +6 -3
- package/dist/{chunk-FCQMZDRI.mjs → chunk-ZVZYJIIP.mjs} +3 -3
- package/dist/{client-DpqtT2XG.d.ts → client-uB0kp2yT.d.ts} +88 -5
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +3 -1265
- package/dist/{config-D9WR3MhO.d.ts → config-BvHQ-C7U.d.ts} +3 -3
- package/dist/{config-C687-reW.d.ts → config-PqgNY1H7.d.ts} +1 -1
- package/dist/{core-leaf-NEUM7PU3.mjs → core-leaf-TAR3GJVE.mjs} +3 -3
- package/dist/{direct-executor-37IH3ISS.mjs → direct-executor-WSZALYCV.mjs} +8 -8
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +35 -15
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.mjs +16 -6
- package/dist/distribution/transport/grpc.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +70 -9
- package/dist/distribution.mjs +1695 -680
- package/dist/{durability-config-NWU2G2SJ.mjs → durability-config-VWS2K24E.mjs} +1 -1
- package/dist/{react-native/chunks/embed-4LBSQD7U.mjs → embed-4PHHKGBK.mjs} +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-De6JzTW8.d.ts → engine-qjAxJSKk.d.ts} +206 -7
- package/dist/{index-CBUl4z3E.d.ts → index-BaQIAnk0.d.ts} +137 -111
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +8 -8
- package/dist/index.react-native.mjs +9 -9
- package/dist/{long-query-thread-DIPJBMV7.mjs → long-query-thread-N2DNPEA3.mjs} +21 -21
- package/dist/{module-Z3CL4GJ4.mjs → module-UQFIJQMG.mjs} +33 -13
- package/dist/{msgpack-leaf-7KBYOVTF.mjs → msgpack-leaf-H5ZYW7S7.mjs} +4 -4
- package/dist/{optional-reads-ENKMVQ4Q.mjs → optional-reads-5HFMF2QZ.mjs} +2 -2
- package/dist/{optional-writes-YIVJMQ4D.mjs → optional-writes-CGDBJBHQ.mjs} +31 -18
- package/dist/{paged-partition-archive-IZ6U7DVN.mjs → paged-partition-archive-UK44CVXG.mjs} +20 -20
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +8 -8
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +18 -18
- package/dist/{phrase-module-ZY3TLTQA.mjs → phrase-module-EFESWOCR.mjs} +2 -2
- package/dist/{query-actions-7EOKT3WD.mjs → query-actions-7BL3YZDW.mjs} +8 -8
- package/dist/query-suggestions/merge-worker.mjs +11 -11
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +9 -9
- package/dist/react-native/chunks/{chunk-TQ26BKBH.mjs → chunk-43R4GCM7.mjs} +7 -0
- package/dist/react-native/chunks/{chunk-CBU4M3EU.mjs → chunk-63DZ26X5.mjs} +277 -75
- package/dist/react-native/chunks/{chunk-BN6FUB7N.mjs → chunk-AN4HKLB4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-T5HF3FW3.mjs → chunk-BNFGA7TY.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-PVMEHBQO.mjs → chunk-CW2ZFXSE.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-NLEFMPY3.mjs → chunk-G434H5WG.mjs} +15 -4
- package/dist/react-native/chunks/{chunk-SK3DPAI4.mjs → chunk-HTBY4GON.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-ZQPSXN6C.mjs → chunk-IP3QLETQ.mjs} +5 -5
- package/dist/react-native/chunks/{chunk-7TPO4FLE.mjs → chunk-LOKGE4OQ.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-ZDDJVR7X.mjs → chunk-MRH36RN3.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-YOTP2FPE.mjs → chunk-UO44MA5M.mjs} +1 -1
- package/dist/{chunk-4JBTALUC.mjs → react-native/chunks/chunk-UYAZLGEM.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-WFQ6YQZA.mjs → chunk-UYKIV2KT.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-IEY4DARA.mjs → chunk-XHYOQ4K4.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-OZ3XEM6D.mjs → chunk-ZUJCUPVS.mjs} +1 -1
- package/dist/react-native/chunks/{core-leaf-TPBRRWVN.mjs → core-leaf-5PVST5PR.mjs} +4 -4
- package/dist/react-native/chunks/{durability-config-RVTWSXPH.mjs → durability-config-FCSD4MXY.mjs} +1 -1
- package/dist/{embed-2QMYCD2Q.mjs → react-native/chunks/embed-HA2I7RCF.mjs} +1 -1
- package/dist/react-native/chunks/{gzip.react-native-DLO4WW27.mjs → gzip.react-native-CVM4NQG2.mjs} +2 -2
- package/dist/react-native/chunks/{module-COLNJNML.mjs → module-QRCYB7QV.mjs} +32 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-74HYI4DO.mjs → optional-reads-AFPO2FEB.mjs} +3 -3
- package/dist/react-native/chunks/{optional-writes-QETIM6BI.mjs → optional-writes-GC6JWMAF.mjs} +31 -18
- package/dist/react-native/chunks/{phrase-module-UC4YGEBB.mjs → phrase-module-IGYBK7XH.mjs} +3 -3
- package/dist/react-native/chunks/{resolve-vector-text-FU7D4QP5.mjs → resolve-vector-text-GE2CRJKI.mjs} +1 -1
- package/dist/react-native/chunks/{restore-queue-3E2MJIJS.mjs → restore-queue-2ACONS4W.mjs} +12 -12
- package/dist/react-native/chunks/{segment-codec-leaf-IE4DXUZV.mjs → segment-codec-leaf-SSIWXVUG.mjs} +10 -10
- package/dist/react-native/chunks/{segment-merge-leaf-62LQ5WZB.mjs → segment-merge-leaf-I2WDFE6O.mjs} +11 -11
- package/dist/react-native/chunks/{storage-leaf-LNOLKWZB.mjs → storage-leaf-YXXMG4VR.mjs} +14727 -13915
- package/dist/react-native/chunks/{store-leaf-Y7CBOPFZ.mjs → store-leaf-FS7A4PZA.mjs} +11 -11
- package/dist/react-native/chunks/{suggestions-leaf-PSUGL6MI.mjs → suggestions-leaf-PSFCT5EX.mjs} +11 -11
- package/dist/react-native/chunks/{worker-copies-MW5O5U3D.mjs → worker-copies-QRMISMEL.mjs} +10 -10
- package/dist/react-native/partial-words.mjs +9 -9
- package/dist/react-native/pattern.mjs +5 -5
- package/dist/react-native/query-suggestions.mjs +9 -9
- package/dist/react-native/sync.mjs +433 -51
- package/dist/react-native/synonyms.mjs +9 -9
- package/dist/react.d.ts +8 -8
- package/dist/react.mjs +7 -0
- package/dist/{rerank-KTV5L7MZ.mjs → rerank-4GFC26FD.mjs} +8 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{resolve-vector-text-SD6BEMKZ.mjs → resolve-vector-text-OD4J3WHF.mjs} +2 -2
- package/dist/{restore-queue-ES7TDJ5V.mjs → restore-queue-E4EIUYVV.mjs} +11 -11
- package/dist/{segment-codec-leaf-DO75VKEV.mjs → segment-codec-leaf-MP6DILVE.mjs} +9 -9
- package/dist/{segment-merge-leaf-JGND6LG5.mjs → segment-merge-leaf-ED2NMUC5.mjs} +9 -9
- package/dist/server.d.ts +13 -9
- package/dist/server.mjs +875 -348
- package/dist/{storage-leaf-ZI3H2WJY.mjs → storage-leaf-HQFO37W7.mjs} +5234 -5835
- package/dist/{store-leaf-YIH36Y5J.mjs → store-leaf-PL2G7RM5.mjs} +10 -10
- package/dist/{suggestions-leaf-FP5HSTFP.mjs → suggestions-leaf-I3MEHACP.mjs} +12 -12
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +16 -16
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync/indexeddb.mjs +3 -3
- package/dist/sync.browser.mjs +35 -1108
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +430 -51
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +8 -8
- package/dist/{tasks-X2IJQc8J.d.ts → tasks-C65V7lIr.d.ts} +3 -3
- package/dist/thread-LTFI3RK5.mjs +31 -0
- package/dist/{types-D0YasjoX.d.ts → types-CpCg-Gq-.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-WYPRCNB7.mjs → worker-copies-P7YWXEP5.mjs} +11 -11
- package/dist/workers/entry.mjs +14 -14
- package/openapi.json +144 -0
- package/package.json +5 -7
- package/dist/chunk-URVH6BLO.mjs +0 -217
- package/dist/react-native/chunks/msgpack-leaf-GTKLF6GJ.mjs +0 -7
- package/dist/thread-NQRNW3IH.mjs +0 -31
- /package/dist/{chunk-CV3Q2EKO.mjs → chunk-S5QMWECL.mjs} +0 -0
|
@@ -1,22 +1,545 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { PAGED_SEGMENT_ENVELOPE_VERSION, loadFailed, checkEverySectionPage, openPagedFile, sectionNamed, createSectionReader, closePagedFile, readWholeSectionChecked, readCacheOfThisThread, createReadCache, MERGE_READ_CACHE_BYTES, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, requireNativePage, ID_SAMPLE_INTERVAL, SMALLEST_BLOCK_POSTINGS, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, resumePagedFile, suspendPagedFile, cachedFileBytes, LIST_VIEWS_HELD_PER_SEGMENT, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, CACHED_POSITIONS, POSITION_WIDTH_BYTES, CACHED_LIST, POSTING_WIDTH_BYTES, readFileRangeLater, CACHED_DOCUMENT_BLOCK, failNativePage, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, POSTING_WRITE_ROWS, POSTING_WRITE_POSITIONS, SECTION_WRITE_BUFFER_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-3N5GUXBI.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-5FLRXF73.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
4
|
-
import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-
|
|
4
|
+
import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-TSE2VWPX.mjs';
|
|
5
5
|
import { DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_COMPACTION_THRESHOLD, SEGMENT_FILE_FRAMING_SLACK_BYTES, REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-MSIN4NM3.mjs';
|
|
6
|
-
import { encode } from './chunk-
|
|
7
|
-
import { restoreVectorFields } from './chunk-
|
|
8
|
-
import { requireWithinFileLimit, HEADER_SIZE, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, readHeader, packChecksummedEnvelope } from './chunk-
|
|
9
|
-
import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment,
|
|
10
|
-
import { encodeDocument } from './chunk-
|
|
6
|
+
import { encode } from './chunk-S5QMWECL.mjs';
|
|
7
|
+
import { restoreVectorFields } from './chunk-R2PJKSNL.mjs';
|
|
8
|
+
import { requireWithinFileLimit, HEADER_SIZE, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, readHeader, packChecksummedEnvelope } from './chunk-KOA6T2IC.mjs';
|
|
9
|
+
import { decodeStoredDocument, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-HSA3H3VW.mjs';
|
|
10
|
+
import { encodeDocument } from './chunk-2CKDCC65.mjs';
|
|
11
11
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
12
|
-
import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-
|
|
12
|
+
import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-7P5YWDRQ.mjs';
|
|
13
13
|
import { MAX_FILE_BYTES } from './chunk-4PY4JYMY.mjs';
|
|
14
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
14
|
+
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
15
15
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
16
16
|
import { deflateRawSync, inflateRawSync } from 'zlib';
|
|
17
17
|
|
|
18
|
+
// src/persistence/paged/verify.ts
|
|
19
|
+
async function orLoadFailure(key, read) {
|
|
20
|
+
try {
|
|
21
|
+
return await read();
|
|
22
|
+
} catch (error) {
|
|
23
|
+
if (error instanceof NarsilError) throw error;
|
|
24
|
+
return loadFailed(
|
|
25
|
+
`The segment "${key}" cannot be read: ${error instanceof Error ? error.message : String(error)}`,
|
|
26
|
+
{
|
|
27
|
+
key
|
|
28
|
+
}
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
async function envelopeVersionOf(directory, key) {
|
|
33
|
+
const { readRange } = directory;
|
|
34
|
+
if (readRange === void 0) return loadFailed(`The directory holding "${key}" reads no ranges`, { key });
|
|
35
|
+
const bytes = await orLoadFailure(key, () => readRange.call(directory, key, 4, 1));
|
|
36
|
+
return bytes[0];
|
|
37
|
+
}
|
|
38
|
+
async function isPagedSegmentFile(directory, key) {
|
|
39
|
+
return await envelopeVersionOf(directory, key) === PAGED_SEGMENT_ENVELOPE_VERSION;
|
|
40
|
+
}
|
|
41
|
+
async function requirePagedPayloadChecksum(directory, listing) {
|
|
42
|
+
const { readRange } = directory;
|
|
43
|
+
const { key, bytes } = listing;
|
|
44
|
+
if (readRange === void 0) return loadFailed(`The directory holding "${key}" reads no ranges`, { key });
|
|
45
|
+
const header2 = readHeader(await readRange.call(directory, key, 0, HEADER_SIZE));
|
|
46
|
+
if (HEADER_SIZE + header2.payloadLength !== bytes) {
|
|
47
|
+
loadFailed(`The segment "${key}" holds ${bytes} bytes where its header gives ${HEADER_SIZE + header2.payloadLength}`);
|
|
48
|
+
}
|
|
49
|
+
await requirePayloadChecksum(key, header2, (offset, length) => readRange.call(directory, key, offset, length));
|
|
50
|
+
}
|
|
51
|
+
async function requirePagedPathChecksum(path) {
|
|
52
|
+
const read = async (offset, length) => {
|
|
53
|
+
const bytes = new Uint8Array(length);
|
|
54
|
+
if (!await readFileRangeLater(path, offset, bytes))
|
|
55
|
+
loadFailed(`The segment "${path}" cannot be read at ${offset}`);
|
|
56
|
+
return bytes;
|
|
57
|
+
};
|
|
58
|
+
await requirePayloadChecksum(path, readHeader(await read(0, HEADER_SIZE)), read);
|
|
59
|
+
}
|
|
60
|
+
async function requirePayloadChecksum(key, header2, read) {
|
|
61
|
+
const bytes = HEADER_SIZE + header2.payloadLength;
|
|
62
|
+
let state = crc32Init();
|
|
63
|
+
for (let at = HEADER_SIZE; at < bytes; at += FILE_ASSEMBLY_CHUNK_BYTES) {
|
|
64
|
+
const size = Math.min(FILE_ASSEMBLY_CHUNK_BYTES, bytes - at);
|
|
65
|
+
state = crc32Update(state, await read(at, size));
|
|
66
|
+
}
|
|
67
|
+
const computed = crc32Final(state);
|
|
68
|
+
if (computed !== header2.checksum) {
|
|
69
|
+
throw new NarsilError(
|
|
70
|
+
ErrorCodes.PERSISTENCE_CRC_MISMATCH,
|
|
71
|
+
`The checksum of the segment "${key}" does not match its payload`,
|
|
72
|
+
{ key, expected: header2.checksum, computed }
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
async function pagedListingOf(directory, ref) {
|
|
77
|
+
const { byteLength } = directory;
|
|
78
|
+
const length = byteLength === void 0 ? ref.bytes : await byteLength.call(directory, ref.key);
|
|
79
|
+
if (length === null) loadFailed(`The segment "${ref.key}" that the manifest lists is missing`, { key: ref.key });
|
|
80
|
+
if (length !== ref.bytes) {
|
|
81
|
+
loadFailed(`The segment "${ref.key}" holds ${length} bytes where the manifest lists ${ref.bytes}`, {
|
|
82
|
+
key: ref.key,
|
|
83
|
+
bytes: length,
|
|
84
|
+
listed: ref.bytes
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
return { key: ref.key, path: await directory.pathOf(ref.key), bytes: ref.bytes, checksum: ref.checksum };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// src/persistence/paged/totals.ts
|
|
91
|
+
var VALUE_BYTES = Float64Array.BYTES_PER_ELEMENT;
|
|
92
|
+
function readPagedTotals(file, totals, first, output, fault) {
|
|
93
|
+
const section = totals.native;
|
|
94
|
+
if (section !== null && file.native !== null) {
|
|
95
|
+
const status = file.native.core.readPagedTotals(
|
|
96
|
+
section,
|
|
97
|
+
file.cache.pageCache(),
|
|
98
|
+
first,
|
|
99
|
+
output.length,
|
|
100
|
+
output,
|
|
101
|
+
fault
|
|
102
|
+
);
|
|
103
|
+
requireNativePage(status, file.path, "tokenTotals", fault[2]);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const bytes = totals.bytesAt(first * VALUE_BYTES, output.byteLength);
|
|
107
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
108
|
+
for (let at = 0; at < output.length; at++) {
|
|
109
|
+
const value = view.getFloat64(at * VALUE_BYTES, true);
|
|
110
|
+
if (!Number.isFinite(value) || value < 0) loadFailed(`The term total at ${first + at} of "${file.path}" is invalid`);
|
|
111
|
+
}
|
|
112
|
+
for (let at = 0; at < output.length; at++) output[at] = view.getFloat64(at * VALUE_BYTES, true);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// src/persistence/paged/lists.ts
|
|
116
|
+
var LITTLE_ENDIAN_PLATFORM = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
117
|
+
function swappedInPlace(bytes, width) {
|
|
118
|
+
for (let at = 0; at + width <= bytes.length; at += width) {
|
|
119
|
+
for (let low = 0, high = width - 1; low < high; low++, high--) {
|
|
120
|
+
const held = bytes[at + low];
|
|
121
|
+
bytes[at + low] = bytes[at + high];
|
|
122
|
+
bytes[at + high] = held;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
function fromLittleEndian(bytes, width) {
|
|
127
|
+
if (!LITTLE_ENDIAN_PLATFORM) swappedInPlace(bytes, width);
|
|
128
|
+
}
|
|
129
|
+
function requireListRules(list, snapshot, token) {
|
|
130
|
+
const { docIds, fieldNameIndices } = list;
|
|
131
|
+
const documents = snapshot.documentCount;
|
|
132
|
+
const fields = snapshot.fieldNames.length;
|
|
133
|
+
for (let row = 0; row < docIds.length; row++) {
|
|
134
|
+
if (docIds[row] >= documents) {
|
|
135
|
+
loadFailed(`Token ${token} of "${snapshot.key}" names document ${docIds[row]} of ${documents}`, { token, row });
|
|
136
|
+
}
|
|
137
|
+
if (fieldNameIndices[row] >= fields) {
|
|
138
|
+
loadFailed(`Token ${token} of "${snapshot.key}" names field ${fieldNameIndices[row]} of ${fields}`, {
|
|
139
|
+
token,
|
|
140
|
+
row
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
if (row === 0) continue;
|
|
144
|
+
const before = docIds[row - 1];
|
|
145
|
+
if (docIds[row] < before || docIds[row] === before && fieldNameIndices[row] <= fieldNameIndices[row - 1]) {
|
|
146
|
+
loadFailed(`The list of token ${token} of "${snapshot.key}" descends at posting ${row}`, { token, row });
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
function createListReader(file, snapshot, sectionReader) {
|
|
151
|
+
const { postingOffsets } = snapshot;
|
|
152
|
+
const postings = sectionReader("postings");
|
|
153
|
+
const positionOffsets = sectionReader("positionOffsets");
|
|
154
|
+
const positions = sectionReader("positions");
|
|
155
|
+
const totals = sectionReader("tokenTotals");
|
|
156
|
+
const { cache, fileNumber } = file;
|
|
157
|
+
const fault = new BigUint64Array(3);
|
|
158
|
+
const total = new Float64Array(1);
|
|
159
|
+
function listAt(token) {
|
|
160
|
+
const held = cache.get(fileNumber, CACHED_LIST, token);
|
|
161
|
+
if (held !== void 0) return held;
|
|
162
|
+
if (postings === null) return loadFailed(`The segment "${snapshot.key}" holds no postings`);
|
|
163
|
+
const start = postingOffsets[token];
|
|
164
|
+
const count = postingOffsets[token + 1] - start;
|
|
165
|
+
const buffer = new ArrayBuffer(count * POSTING_WIDTH_BYTES);
|
|
166
|
+
const bytes = new Uint8Array(buffer);
|
|
167
|
+
const descriptor = postings.native;
|
|
168
|
+
if (descriptor !== null && file.native !== null) {
|
|
169
|
+
const status = file.native.core.readPagedPostings(
|
|
170
|
+
descriptor,
|
|
171
|
+
cache.pageCache(),
|
|
172
|
+
start,
|
|
173
|
+
count,
|
|
174
|
+
snapshot.documentCount,
|
|
175
|
+
snapshot.fieldNames.length,
|
|
176
|
+
bytes,
|
|
177
|
+
fault
|
|
178
|
+
);
|
|
179
|
+
requireNativePage(status, file.path, "postings", fault[2]);
|
|
180
|
+
} else {
|
|
181
|
+
postings.read(start * POSTING_WIDTH_BYTES, bytes);
|
|
182
|
+
}
|
|
183
|
+
if (!LITTLE_ENDIAN_PLATFORM) {
|
|
184
|
+
swappedInPlace(bytes.subarray(0, count * 4), 4);
|
|
185
|
+
swappedInPlace(bytes.subarray(count * 4), 2);
|
|
186
|
+
}
|
|
187
|
+
const list = {
|
|
188
|
+
docIds: new Uint32Array(buffer, 0, count),
|
|
189
|
+
termFrequencies: new Uint16Array(buffer, count * 4, count),
|
|
190
|
+
fieldNameIndices: new Uint16Array(buffer, count * 6, count)
|
|
191
|
+
};
|
|
192
|
+
if (descriptor === null || file.native === null) requireListRules(list, snapshot, token);
|
|
193
|
+
cache.put(fileNumber, CACHED_LIST, token, list, buffer.byteLength);
|
|
194
|
+
return list;
|
|
195
|
+
}
|
|
196
|
+
function positionsAt(token) {
|
|
197
|
+
if (positionOffsets === null || positions === null) return null;
|
|
198
|
+
const held = cache.get(fileNumber, CACHED_POSITIONS, token);
|
|
199
|
+
if (held !== void 0) return held;
|
|
200
|
+
const start = postingOffsets[token];
|
|
201
|
+
const count = postingOffsets[token + 1] - start;
|
|
202
|
+
const offsetBytes = new Uint8Array((count + 1) * POSITION_WIDTH_BYTES);
|
|
203
|
+
positionOffsets.read(start * POSITION_WIDTH_BYTES, offsetBytes);
|
|
204
|
+
fromLittleEndian(offsetBytes, POSITION_WIDTH_BYTES);
|
|
205
|
+
const absolute = new Uint32Array(offsetBytes.buffer);
|
|
206
|
+
const first = absolute[0];
|
|
207
|
+
const end = absolute[count];
|
|
208
|
+
if (end < first || end * POSITION_WIDTH_BYTES > positions.length || start === 0 && first !== 0 || (start + count + 1) * POSITION_WIDTH_BYTES === positionOffsets.length && end * POSITION_WIDTH_BYTES !== positions.length)
|
|
209
|
+
loadFailed(`The positions of token ${token} of "${snapshot.key}" leave their section`, { token });
|
|
210
|
+
const offsets = new Uint32Array(count + 1);
|
|
211
|
+
const valueBytes = new Uint8Array((end - first) * POSITION_WIDTH_BYTES);
|
|
212
|
+
const loaded = { offsets, values: new Uint32Array(valueBytes.buffer) };
|
|
213
|
+
const offsetSection = positionOffsets.native;
|
|
214
|
+
const valueSection = positions.native;
|
|
215
|
+
if (offsetSection !== null && valueSection !== null && file.native !== null) {
|
|
216
|
+
const status = file.native.core.readPagedPositions(
|
|
217
|
+
offsetSection,
|
|
218
|
+
valueSection,
|
|
219
|
+
cache.pageCache(),
|
|
220
|
+
start,
|
|
221
|
+
count,
|
|
222
|
+
offsets,
|
|
223
|
+
loaded.values,
|
|
224
|
+
fault
|
|
225
|
+
);
|
|
226
|
+
requireNativePage(
|
|
227
|
+
status,
|
|
228
|
+
file.path,
|
|
229
|
+
fault[1] === offsetSection.offset ? "positionOffsets" : "positions",
|
|
230
|
+
fault[2]
|
|
231
|
+
);
|
|
232
|
+
} else {
|
|
233
|
+
for (let row = 0; row <= count; row++) {
|
|
234
|
+
if (absolute[row] < first || absolute[row] > end || row > 0 && absolute[row] < absolute[row - 1]) {
|
|
235
|
+
loadFailed(`The position offsets of token ${token} of "${snapshot.key}" leave their list`, { token, row });
|
|
236
|
+
}
|
|
237
|
+
offsets[row] = absolute[row] - first;
|
|
238
|
+
}
|
|
239
|
+
positions.read(first * POSITION_WIDTH_BYTES, valueBytes);
|
|
240
|
+
fromLittleEndian(valueBytes, POSITION_WIDTH_BYTES);
|
|
241
|
+
for (let row = 0; row < count; row++) {
|
|
242
|
+
for (let at = offsets[row] + 1; at < offsets[row + 1]; at++) {
|
|
243
|
+
if (loaded.values[at] < loaded.values[at - 1])
|
|
244
|
+
loadFailed(`The positions of token ${token} of "${snapshot.key}" descend`, { token, row });
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
cache.put(fileNumber, CACHED_POSITIONS, token, loaded, offsetBytes.byteLength + valueBytes.byteLength);
|
|
249
|
+
return loaded;
|
|
250
|
+
}
|
|
251
|
+
function totalAt(token) {
|
|
252
|
+
if (totals === null) return loadFailed(`The segment "${snapshot.key}" holds no token totals`);
|
|
253
|
+
readPagedTotals(file, totals, token, total, fault);
|
|
254
|
+
return total[0];
|
|
255
|
+
}
|
|
256
|
+
function listedIndexOf(token) {
|
|
257
|
+
const { blockLists } = snapshot;
|
|
258
|
+
let low = 0;
|
|
259
|
+
let high = blockLists.length;
|
|
260
|
+
while (low < high) {
|
|
261
|
+
const middle = low + high >>> 1;
|
|
262
|
+
if (blockLists[middle] < token) low = middle + 1;
|
|
263
|
+
else high = middle;
|
|
264
|
+
}
|
|
265
|
+
return low < blockLists.length && blockLists[low] === token ? low : -1;
|
|
266
|
+
}
|
|
267
|
+
function storedBoundsAt(token) {
|
|
268
|
+
const listed = listedIndexOf(token);
|
|
269
|
+
if (listed < 0) return null;
|
|
270
|
+
const first = snapshot.blockListOffsets[listed];
|
|
271
|
+
const blockCount = snapshot.blockListOffsets[listed + 1] - first;
|
|
272
|
+
const { blockSummaries } = snapshot;
|
|
273
|
+
const view = new DataView(blockSummaries.buffer, blockSummaries.byteOffset, blockSummaries.byteLength);
|
|
274
|
+
const bounds = {
|
|
275
|
+
structureRevision: 0,
|
|
276
|
+
blockCount,
|
|
277
|
+
entryEnd: new Int32Array(blockCount),
|
|
278
|
+
maxTermFrequency: new Int32Array(blockCount),
|
|
279
|
+
minFieldLength: new Float64Array(blockCount),
|
|
280
|
+
maxEntriesPerDocument: new Int32Array(blockCount),
|
|
281
|
+
documentCount: new Int32Array(blockCount)
|
|
282
|
+
};
|
|
283
|
+
const length = postingOffsets[token + 1] - postingOffsets[token];
|
|
284
|
+
for (let block = 0; block < blockCount; block++) {
|
|
285
|
+
const place = (first + block) * BLOCK_SUMMARY_BYTES;
|
|
286
|
+
bounds.entryEnd[block] = view.getUint32(place, true);
|
|
287
|
+
bounds.maxTermFrequency[block] = view.getUint16(place + 4, true);
|
|
288
|
+
bounds.minFieldLength[block] = view.getFloat64(place + 6, true);
|
|
289
|
+
bounds.maxEntriesPerDocument[block] = view.getUint32(place + 14, true);
|
|
290
|
+
bounds.documentCount[block] = view.getUint32(place + 18, true);
|
|
291
|
+
const before = block === 0 ? 0 : bounds.entryEnd[block - 1];
|
|
292
|
+
if (bounds.entryEnd[block] <= before || bounds.entryEnd[block] > length) {
|
|
293
|
+
loadFailed(`Block ${block} of token ${token} of "${snapshot.key}" ends outside its list`, { token, block });
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
if (blockCount === 0 || bounds.entryEnd[blockCount - 1] !== length) {
|
|
297
|
+
loadFailed(`The blocks of token ${token} of "${snapshot.key}" fail to cover its list`, { token });
|
|
298
|
+
}
|
|
299
|
+
return bounds;
|
|
300
|
+
}
|
|
301
|
+
return { listAt, positionsAt, totalAt, storedBoundsAt };
|
|
302
|
+
}
|
|
303
|
+
function createPagedPostingViews(snapshot, lists, tombstones, counts) {
|
|
304
|
+
const { postingOffsets, tracksPositions } = snapshot;
|
|
305
|
+
const views = /* @__PURE__ */ new Map();
|
|
306
|
+
function buildView(token) {
|
|
307
|
+
const length = postingOffsets[token + 1] - postingOffsets[token];
|
|
308
|
+
return {
|
|
309
|
+
length,
|
|
310
|
+
get docIds() {
|
|
311
|
+
return lists.listAt(token).docIds;
|
|
312
|
+
},
|
|
313
|
+
get termFrequencies() {
|
|
314
|
+
return lists.listAt(token).termFrequencies;
|
|
315
|
+
},
|
|
316
|
+
get fieldNameIndices() {
|
|
317
|
+
return lists.listAt(token).fieldNameIndices;
|
|
318
|
+
},
|
|
319
|
+
get positions() {
|
|
320
|
+
const loaded = lists.positionsAt(token);
|
|
321
|
+
if (loaded === null) return null;
|
|
322
|
+
const rows = new Array(length);
|
|
323
|
+
for (let row = 0; row < length; row++) {
|
|
324
|
+
rows[row] = [...loaded.values.subarray(loaded.offsets[row], loaded.offsets[row + 1])];
|
|
325
|
+
}
|
|
326
|
+
return rows;
|
|
327
|
+
},
|
|
328
|
+
holdsPositions: tracksPositions,
|
|
329
|
+
positionCountAt(row) {
|
|
330
|
+
const loaded = lists.positionsAt(token);
|
|
331
|
+
return loaded === null ? 0 : loaded.offsets[row + 1] - loaded.offsets[row];
|
|
332
|
+
},
|
|
333
|
+
get positionStore() {
|
|
334
|
+
return lists.positionsAt(token)?.values ?? null;
|
|
335
|
+
},
|
|
336
|
+
positionStartAt(row) {
|
|
337
|
+
return lists.positionsAt(token)?.offsets[row] ?? 0;
|
|
338
|
+
},
|
|
339
|
+
docIdSet: {
|
|
340
|
+
get size() {
|
|
341
|
+
return counts.documentFrequency(token);
|
|
342
|
+
}
|
|
343
|
+
},
|
|
344
|
+
deletedDocs: tombstones,
|
|
345
|
+
get totalTermFrequency() {
|
|
346
|
+
return lists.totalAt(token) - counts.removedOccurrences(token);
|
|
347
|
+
},
|
|
348
|
+
get structureRevision() {
|
|
349
|
+
return tombstones.revision;
|
|
350
|
+
},
|
|
351
|
+
ordered: true,
|
|
352
|
+
storedBlockBounds() {
|
|
353
|
+
return tombstones.size === 0 ? lists.storedBoundsAt(token) : null;
|
|
354
|
+
}
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
return {
|
|
358
|
+
viewAt(payloadSlot) {
|
|
359
|
+
let view = views.get(payloadSlot);
|
|
360
|
+
if (view === void 0) {
|
|
361
|
+
if (views.size >= LIST_VIEWS_HELD_PER_SEGMENT) views.clear();
|
|
362
|
+
view = buildView(payloadSlot);
|
|
363
|
+
views.set(payloadSlot, view);
|
|
364
|
+
}
|
|
365
|
+
return view;
|
|
366
|
+
},
|
|
367
|
+
viewReadOnce(payloadSlot) {
|
|
368
|
+
return buildView(payloadSlot);
|
|
369
|
+
}
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// src/persistence/paged/removals.ts
|
|
374
|
+
var VALUE_BYTES2 = 4;
|
|
375
|
+
function createRemovalReader(file, removalOffsets, removals, tokenCount) {
|
|
376
|
+
const fault = new BigUint64Array(3);
|
|
377
|
+
return (ordinal) => {
|
|
378
|
+
const bounds = removalOffsets.bytesAt(ordinal * VALUE_BYTES2, VALUE_BYTES2 * 2);
|
|
379
|
+
const boundsView = new DataView(bounds.buffer, bounds.byteOffset, bounds.byteLength);
|
|
380
|
+
const from = boundsView.getUint32(0, true);
|
|
381
|
+
const to = boundsView.getUint32(VALUE_BYTES2, true);
|
|
382
|
+
if (to < from || to * VALUE_BYTES2 > removals.length || from % 2 !== 0 || to % 2 !== 0 || ordinal === 0 && from !== 0 || (ordinal + 2) * VALUE_BYTES2 === removalOffsets.length && to * VALUE_BYTES2 !== removals.length) {
|
|
383
|
+
loadFailed(`The removal entry of document ${ordinal} of "${file.path}" leaves its list`, { ordinal });
|
|
384
|
+
}
|
|
385
|
+
const values = new Uint32Array(to - from);
|
|
386
|
+
const offsetSection = removalOffsets.native;
|
|
387
|
+
const valueSection = removals.native;
|
|
388
|
+
if (offsetSection !== null && valueSection !== null && file.native !== null) {
|
|
389
|
+
const status = file.native.core.readPagedRemovals(
|
|
390
|
+
offsetSection,
|
|
391
|
+
valueSection,
|
|
392
|
+
file.cache.pageCache(),
|
|
393
|
+
ordinal,
|
|
394
|
+
tokenCount,
|
|
395
|
+
values,
|
|
396
|
+
fault
|
|
397
|
+
);
|
|
398
|
+
if (status < 0)
|
|
399
|
+
requireNativePage(
|
|
400
|
+
status,
|
|
401
|
+
file.path,
|
|
402
|
+
fault[1] === offsetSection.offset ? "removalOffsets" : "removals",
|
|
403
|
+
fault[2]
|
|
404
|
+
);
|
|
405
|
+
} else {
|
|
406
|
+
const entry = removals.bytesAt(from * VALUE_BYTES2, values.byteLength);
|
|
407
|
+
const view = new DataView(entry.buffer, entry.byteOffset, entry.byteLength);
|
|
408
|
+
let previous = -1;
|
|
409
|
+
for (let at = 0; at < values.length; at += 2) {
|
|
410
|
+
const token = view.getUint32(at * VALUE_BYTES2, true);
|
|
411
|
+
if (token >= tokenCount || token <= previous) {
|
|
412
|
+
loadFailed(`The removal entry of document ${ordinal} of "${file.path}" holds invalid token ${token}`, {
|
|
413
|
+
ordinal
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
previous = token;
|
|
417
|
+
values[at] = token;
|
|
418
|
+
values[at + 1] = view.getUint32((at + 1) * VALUE_BYTES2, true);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
return values;
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
// src/persistence/paged/check-contents.ts
|
|
426
|
+
function requireIds(segment, snapshot) {
|
|
427
|
+
const ids = segment.arrays.idTable;
|
|
428
|
+
let previous = null;
|
|
429
|
+
for (let position = 0; position < snapshot.documentCount; position++) {
|
|
430
|
+
const id = ids.idAt(ids.sortedOrdinalAt(position));
|
|
431
|
+
if (previous !== null && compareCodePoints(previous, id) >= 0)
|
|
432
|
+
loadFailed(`The ID order of "${snapshot.key}" does not ascend at position ${position}`, { position });
|
|
433
|
+
if (position % ID_SAMPLE_INTERVAL === 0 && snapshot.idSampleIds[position / ID_SAMPLE_INTERVAL] !== id)
|
|
434
|
+
loadFailed(`The ID sample of "${snapshot.key}" differs at position ${position}`, { position });
|
|
435
|
+
previous = id;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
function requirePostings(lists, snapshot) {
|
|
439
|
+
const fields = snapshot.fieldNames.map((field) => {
|
|
440
|
+
const at = snapshot.fieldLengthNames.indexOf(field);
|
|
441
|
+
return at < 0 ? null : snapshot.fieldLengthColumns[at];
|
|
442
|
+
});
|
|
443
|
+
for (let token = 0; token < snapshot.tokenTable.documentFrequencies.length; token++) {
|
|
444
|
+
const list = lists.listAt(token);
|
|
445
|
+
lists.positionsAt(token);
|
|
446
|
+
const bounds = lists.storedBoundsAt(token);
|
|
447
|
+
if (list.docIds.length > SMALLEST_BLOCK_POSTINGS !== (bounds !== null))
|
|
448
|
+
loadFailed(`The block summaries of token ${token} of "${snapshot.key}" do not cover its postings`, { token });
|
|
449
|
+
let documents = 0;
|
|
450
|
+
let occurrences = 0;
|
|
451
|
+
let block = 0;
|
|
452
|
+
let start = 0;
|
|
453
|
+
let maxFrequency = 0;
|
|
454
|
+
let minLength = Number.POSITIVE_INFINITY;
|
|
455
|
+
let maxEntries = 0;
|
|
456
|
+
let blockDocuments = 0;
|
|
457
|
+
let row = 0;
|
|
458
|
+
while (row < list.docIds.length) {
|
|
459
|
+
const document = list.docIds[row];
|
|
460
|
+
let entries = 0;
|
|
461
|
+
do {
|
|
462
|
+
const frequency = list.termFrequencies[row];
|
|
463
|
+
occurrences += frequency;
|
|
464
|
+
maxFrequency = Math.max(maxFrequency, frequency);
|
|
465
|
+
minLength = Math.min(minLength, fields[list.fieldNameIndices[row]]?.[document] ?? 0);
|
|
466
|
+
row++;
|
|
467
|
+
entries++;
|
|
468
|
+
} while (row < list.docIds.length && list.docIds[row] === document);
|
|
469
|
+
documents++;
|
|
470
|
+
blockDocuments++;
|
|
471
|
+
maxEntries = Math.max(maxEntries, entries);
|
|
472
|
+
if (bounds === null || row - start < SMALLEST_BLOCK_POSTINGS && row < list.docIds.length) continue;
|
|
473
|
+
if (block >= bounds.blockCount || bounds.entryEnd[block] !== row || bounds.maxTermFrequency[block] !== maxFrequency || bounds.minFieldLength[block] !== minLength || bounds.maxEntriesPerDocument[block] !== maxEntries || bounds.documentCount[block] !== blockDocuments)
|
|
474
|
+
loadFailed(`Block ${block} of token ${token} of "${snapshot.key}" differs from its postings`, { token, block });
|
|
475
|
+
block++;
|
|
476
|
+
start = row;
|
|
477
|
+
maxFrequency = 0;
|
|
478
|
+
minLength = Number.POSITIVE_INFINITY;
|
|
479
|
+
maxEntries = 0;
|
|
480
|
+
blockDocuments = 0;
|
|
481
|
+
}
|
|
482
|
+
if (documents !== snapshot.tokenTable.documentFrequencies[token] || occurrences !== lists.totalAt(token))
|
|
483
|
+
loadFailed(`The counts of token ${token} of "${snapshot.key}" differ from its postings`, { token });
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
async function checkPagedContents(segment, cache) {
|
|
487
|
+
const { paged } = segment;
|
|
488
|
+
if (paged === null) return;
|
|
489
|
+
const { snapshot } = paged;
|
|
490
|
+
for (const section of snapshot.sections) await checkEverySectionPage(snapshot.path, section, () => true);
|
|
491
|
+
const file = openPagedFile(snapshot.path, cache);
|
|
492
|
+
try {
|
|
493
|
+
const sectionReader = (name) => {
|
|
494
|
+
const section = sectionNamed(snapshot.sections, name);
|
|
495
|
+
return section === void 0 ? null : createSectionReader(file, section);
|
|
496
|
+
};
|
|
497
|
+
const required = (name) => {
|
|
498
|
+
const section = sectionNamed(snapshot.sections, name);
|
|
499
|
+
if (section === void 0) loadFailed(`The segment "${snapshot.key}" holds no section "${name}"`);
|
|
500
|
+
return createSectionReader(file, section);
|
|
501
|
+
};
|
|
502
|
+
const removals = createRemovalReader(
|
|
503
|
+
file,
|
|
504
|
+
required("removalOffsets"),
|
|
505
|
+
required("removals"),
|
|
506
|
+
segment.arrays.tokenTable.size
|
|
507
|
+
);
|
|
508
|
+
for (let ordinal = 0; ordinal < snapshot.documentCount; ordinal++) {
|
|
509
|
+
decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields);
|
|
510
|
+
paged.writeRecordAt(ordinal);
|
|
511
|
+
removals(ordinal);
|
|
512
|
+
}
|
|
513
|
+
if (snapshot.documentCount === 0) {
|
|
514
|
+
for (const name of ["idOffsets", "removalOffsets"]) {
|
|
515
|
+
const offsets = required(name).bytesAt(0, 4);
|
|
516
|
+
if (new DataView(offsets.buffer, offsets.byteOffset, 4).getUint32(0, true) !== 0)
|
|
517
|
+
loadFailed(`Section "${name}" of "${snapshot.key}" does not start at zero`);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
if (snapshot.tracksPositions && snapshot.postingOffsets[snapshot.postingOffsets.length - 1] === 0) {
|
|
521
|
+
const offsets = required("positionOffsets").bytesAt(0, 4);
|
|
522
|
+
if (new DataView(offsets.buffer, offsets.byteOffset, 4).getUint32(0, true) !== 0)
|
|
523
|
+
loadFailed(`The position offsets of "${snapshot.key}" do not start at zero`);
|
|
524
|
+
}
|
|
525
|
+
requireIds(segment, snapshot);
|
|
526
|
+
requirePostings(createListReader(file, snapshot, sectionReader), snapshot);
|
|
527
|
+
segment.arrays.readSurfaceForms();
|
|
528
|
+
} finally {
|
|
529
|
+
closePagedFile(file);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
async function checkPagedSelections(selections) {
|
|
533
|
+
for (const { segment } of selections) {
|
|
534
|
+
const { paged } = segment;
|
|
535
|
+
if (paged === null || paged.payloadChecked) continue;
|
|
536
|
+
await requirePagedPathChecksum(paged.snapshot.path);
|
|
537
|
+
paged.payloadChecked = true;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
18
541
|
// src/serialization/keyword-segment/little-endian.ts
|
|
19
|
-
var
|
|
542
|
+
var LITTLE_ENDIAN_PLATFORM2 = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
20
543
|
function swappedCopy(values) {
|
|
21
544
|
const width = values.BYTES_PER_ELEMENT;
|
|
22
545
|
const source = new Uint8Array(values.buffer, values.byteOffset, values.byteLength);
|
|
@@ -27,7 +550,7 @@ function swappedCopy(values) {
|
|
|
27
550
|
return copy;
|
|
28
551
|
}
|
|
29
552
|
function littleEndianBytes(values) {
|
|
30
|
-
if (!
|
|
553
|
+
if (!LITTLE_ENDIAN_PLATFORM2) return swappedCopy(values);
|
|
31
554
|
return new Uint8Array(values.buffer, values.byteOffset, values.byteLength);
|
|
32
555
|
}
|
|
33
556
|
function allocate(byteLength, shared) {
|
|
@@ -36,7 +559,7 @@ function allocate(byteLength, shared) {
|
|
|
36
559
|
function copyInto(bytes, width, shared) {
|
|
37
560
|
const buffer = allocate(bytes.byteLength, shared);
|
|
38
561
|
const target = new Uint8Array(buffer);
|
|
39
|
-
if (
|
|
562
|
+
if (LITTLE_ENDIAN_PLATFORM2) {
|
|
40
563
|
target.set(bytes);
|
|
41
564
|
return buffer;
|
|
42
565
|
}
|
|
@@ -725,31 +1248,6 @@ function residentBytesOf(snapshot) {
|
|
|
725
1248
|
return bytes;
|
|
726
1249
|
}
|
|
727
1250
|
|
|
728
|
-
// src/persistence/paged/totals.ts
|
|
729
|
-
var VALUE_BYTES = Float64Array.BYTES_PER_ELEMENT;
|
|
730
|
-
function readPagedTotals(file, totals, first, output, fault) {
|
|
731
|
-
const section = totals.native;
|
|
732
|
-
if (section !== null && file.native !== null) {
|
|
733
|
-
const status = file.native.core.readPagedTotals(
|
|
734
|
-
section,
|
|
735
|
-
file.cache.pageCache(),
|
|
736
|
-
first,
|
|
737
|
-
output.length,
|
|
738
|
-
output,
|
|
739
|
-
fault
|
|
740
|
-
);
|
|
741
|
-
requireNativePage(status, file.path, "tokenTotals", fault[2]);
|
|
742
|
-
return;
|
|
743
|
-
}
|
|
744
|
-
const bytes = totals.bytesAt(first * VALUE_BYTES, output.byteLength);
|
|
745
|
-
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
746
|
-
for (let at = 0; at < output.length; at++) {
|
|
747
|
-
const value = view.getFloat64(at * VALUE_BYTES, true);
|
|
748
|
-
if (!Number.isFinite(value) || value < 0) loadFailed(`The term total at ${first + at} of "${file.path}" is invalid`);
|
|
749
|
-
}
|
|
750
|
-
for (let at = 0; at < output.length; at++) output[at] = view.getFloat64(at * VALUE_BYTES, true);
|
|
751
|
-
}
|
|
752
|
-
|
|
753
1251
|
// src/persistence/paged/expansion-source.ts
|
|
754
1252
|
var NUMBER_BYTES = 8;
|
|
755
1253
|
function createPagedExpansionSource(file, snapshot, table, tombstones, removedDocuments, removedOccurrences, totals) {
|
|
@@ -791,66 +1289,14 @@ function createPagedExpansionSource(file, snapshot, table, tombstones, removedDo
|
|
|
791
1289
|
readPagedTotals(file, totals, 0, values, new BigUint64Array(3));
|
|
792
1290
|
occurrences = values;
|
|
793
1291
|
}
|
|
794
|
-
return {
|
|
795
|
-
growing: null,
|
|
796
|
-
frozen,
|
|
797
|
-
state,
|
|
798
|
-
occurrences
|
|
799
|
-
};
|
|
800
|
-
},
|
|
801
|
-
bytes: () => partialFields.byteLength + (occurrences?.byteLength ?? 0)
|
|
802
|
-
};
|
|
803
|
-
}
|
|
804
|
-
|
|
805
|
-
// src/persistence/paged/removals.ts
|
|
806
|
-
var VALUE_BYTES2 = 4;
|
|
807
|
-
function createRemovalReader(file, removalOffsets, removals, tokenCount) {
|
|
808
|
-
const fault = new BigUint64Array(3);
|
|
809
|
-
return (ordinal) => {
|
|
810
|
-
const bounds = removalOffsets.bytesAt(ordinal * VALUE_BYTES2, VALUE_BYTES2 * 2);
|
|
811
|
-
const boundsView = new DataView(bounds.buffer, bounds.byteOffset, bounds.byteLength);
|
|
812
|
-
const from = boundsView.getUint32(0, true);
|
|
813
|
-
const to = boundsView.getUint32(VALUE_BYTES2, true);
|
|
814
|
-
if (to < from || to * VALUE_BYTES2 > removals.length || from % 2 !== 0 || to % 2 !== 0 || ordinal === 0 && from !== 0 || (ordinal + 2) * VALUE_BYTES2 === removalOffsets.length && to * VALUE_BYTES2 !== removals.length) {
|
|
815
|
-
loadFailed(`The removal entry of document ${ordinal} of "${file.path}" leaves its list`, { ordinal });
|
|
816
|
-
}
|
|
817
|
-
const values = new Uint32Array(to - from);
|
|
818
|
-
const offsetSection = removalOffsets.native;
|
|
819
|
-
const valueSection = removals.native;
|
|
820
|
-
if (offsetSection !== null && valueSection !== null && file.native !== null) {
|
|
821
|
-
const status = file.native.core.readPagedRemovals(
|
|
822
|
-
offsetSection,
|
|
823
|
-
valueSection,
|
|
824
|
-
file.cache.pageCache(),
|
|
825
|
-
ordinal,
|
|
826
|
-
tokenCount,
|
|
827
|
-
values,
|
|
828
|
-
fault
|
|
829
|
-
);
|
|
830
|
-
if (status < 0)
|
|
831
|
-
requireNativePage(
|
|
832
|
-
status,
|
|
833
|
-
file.path,
|
|
834
|
-
fault[1] === offsetSection.offset ? "removalOffsets" : "removals",
|
|
835
|
-
fault[2]
|
|
836
|
-
);
|
|
837
|
-
} else {
|
|
838
|
-
const entry = removals.bytesAt(from * VALUE_BYTES2, values.byteLength);
|
|
839
|
-
const view = new DataView(entry.buffer, entry.byteOffset, entry.byteLength);
|
|
840
|
-
let previous = -1;
|
|
841
|
-
for (let at = 0; at < values.length; at += 2) {
|
|
842
|
-
const token = view.getUint32(at * VALUE_BYTES2, true);
|
|
843
|
-
if (token >= tokenCount || token <= previous) {
|
|
844
|
-
loadFailed(`The removal entry of document ${ordinal} of "${file.path}" holds invalid token ${token}`, {
|
|
845
|
-
ordinal
|
|
846
|
-
});
|
|
847
|
-
}
|
|
848
|
-
previous = token;
|
|
849
|
-
values[at] = token;
|
|
850
|
-
values[at + 1] = view.getUint32((at + 1) * VALUE_BYTES2, true);
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
return values;
|
|
1292
|
+
return {
|
|
1293
|
+
growing: null,
|
|
1294
|
+
frozen,
|
|
1295
|
+
state,
|
|
1296
|
+
occurrences
|
|
1297
|
+
};
|
|
1298
|
+
},
|
|
1299
|
+
bytes: () => partialFields.byteLength + (occurrences?.byteLength ?? 0)
|
|
854
1300
|
};
|
|
855
1301
|
}
|
|
856
1302
|
|
|
@@ -979,375 +1425,117 @@ function createDocumentBlockReader(snapshot, documents, cache, fileNumber) {
|
|
|
979
1425
|
{ block }
|
|
980
1426
|
);
|
|
981
1427
|
}
|
|
982
|
-
if (raw.byteLength !== rawLength) {
|
|
983
|
-
loadFailed(`Document block ${block} of "${key}" inflates to ${raw.byteLength} bytes of ${rawLength}`, { block });
|
|
984
|
-
}
|
|
985
|
-
return raw;
|
|
986
|
-
}
|
|
987
|
-
function startsOf(block, raw) {
|
|
988
|
-
const first = firstOrdinalOf(block);
|
|
989
|
-
const end = block + 1 < blockCount ? firstOrdinalOf(block + 1) : documentCount;
|
|
990
|
-
const starts = new Uint32Array(end - first + 1);
|
|
991
|
-
const bytes = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);
|
|
992
|
-
let cursor = 0;
|
|
993
|
-
for (let at = 0; at < end - first; at++) {
|
|
994
|
-
starts[at] = cursor;
|
|
995
|
-
const next = cursor < raw.byteLength ? endOfValue(bytes, cursor) : null;
|
|
996
|
-
if (next === null)
|
|
997
|
-
loadFailed(`Document block ${block} of "${key}" holds a broken document at ${at}`, { block, at });
|
|
998
|
-
cursor = next;
|
|
999
|
-
}
|
|
1000
|
-
starts[end - first] = cursor;
|
|
1001
|
-
if (cursor !== raw.byteLength)
|
|
1002
|
-
loadFailed(`Document block ${block} of "${key}" holds bytes past its documents`, { block });
|
|
1003
|
-
return starts;
|
|
1004
|
-
}
|
|
1005
|
-
function loadBlock(block) {
|
|
1006
|
-
const held = cache.get(fileNumber, CACHED_DOCUMENT_BLOCK, block);
|
|
1007
|
-
if (held !== void 0) return held;
|
|
1008
|
-
const bytes = inflate(block);
|
|
1009
|
-
const loaded = { bytes, starts: startsOf(block, bytes) };
|
|
1010
|
-
cache.put(fileNumber, CACHED_DOCUMENT_BLOCK, block, loaded, bytes.byteLength + loaded.starts.byteLength);
|
|
1011
|
-
return loaded;
|
|
1012
|
-
}
|
|
1013
|
-
return {
|
|
1014
|
-
encodedDocumentAt(ordinal) {
|
|
1015
|
-
if (ordinal < 0 || ordinal >= documentCount) {
|
|
1016
|
-
loadFailed(`Document ${ordinal} lies outside "${key}", which holds ${documentCount}`, { ordinal });
|
|
1017
|
-
}
|
|
1018
|
-
const block = blockOf(ordinal);
|
|
1019
|
-
const { bytes, starts } = loadBlock(block);
|
|
1020
|
-
const row = ordinal - firstOrdinalOf(block);
|
|
1021
|
-
return bytes.subarray(starts[row], starts[row + 1]);
|
|
1022
|
-
}
|
|
1023
|
-
};
|
|
1024
|
-
}
|
|
1025
|
-
|
|
1026
|
-
// src/persistence/paged/ids.ts
|
|
1027
|
-
var VALUE_BYTES3 = 4;
|
|
1028
|
-
var decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
1029
|
-
function uint32At(reader, index) {
|
|
1030
|
-
const bytes = reader.bytesAt(index * VALUE_BYTES3, VALUE_BYTES3);
|
|
1031
|
-
return new DataView(bytes.buffer, bytes.byteOffset, VALUE_BYTES3).getUint32(0, true);
|
|
1032
|
-
}
|
|
1033
|
-
function createPagedIdTable(snapshot, ids, idOffsets, idOrder) {
|
|
1034
|
-
const count = snapshot.documentCount;
|
|
1035
|
-
const { idSampleIds, idSamplePositions, key } = snapshot;
|
|
1036
|
-
let sampleBytes = idSamplePositions.byteLength;
|
|
1037
|
-
for (const id of idSampleIds) sampleBytes += id.length;
|
|
1038
|
-
function idAt(ordinal) {
|
|
1039
|
-
const from = uint32At(idOffsets, ordinal);
|
|
1040
|
-
const to = uint32At(idOffsets, ordinal + 1);
|
|
1041
|
-
if (to < from || to > ids.length || ordinal === 0 && from !== 0 || ordinal + 1 === count && to !== ids.length)
|
|
1042
|
-
loadFailed(`The ID of document ${ordinal} of "${key}" leaves its list`, { ordinal });
|
|
1043
|
-
try {
|
|
1044
|
-
return decoder.decode(ids.bytesAt(from, to - from));
|
|
1045
|
-
} catch {
|
|
1046
|
-
return loadFailed(`The ID of document ${ordinal} of "${key}" is not valid UTF-8`, { ordinal });
|
|
1047
|
-
}
|
|
1048
|
-
}
|
|
1049
|
-
function ordinalAtPosition(position) {
|
|
1050
|
-
const ordinal = uint32At(idOrder, position);
|
|
1051
|
-
if (ordinal >= count) loadFailed(`idOrder of "${key}" names document ${ordinal} of ${count}`, { position });
|
|
1052
|
-
return ordinal;
|
|
1053
|
-
}
|
|
1054
|
-
function windowOf(docId) {
|
|
1055
|
-
let low = 0;
|
|
1056
|
-
let high = idSampleIds.length;
|
|
1057
|
-
while (low < high) {
|
|
1058
|
-
const middle = low + high >>> 1;
|
|
1059
|
-
if (compareCodePoints(idSampleIds[middle], docId) <= 0) low = middle + 1;
|
|
1060
|
-
else high = middle;
|
|
1061
|
-
}
|
|
1062
|
-
const from = low === 0 ? 0 : idSamplePositions[low - 1];
|
|
1063
|
-
const to = low < idSamplePositions.length ? idSamplePositions[low] + 1 : count;
|
|
1064
|
-
return { from, to: Math.min(to, count) };
|
|
1065
|
-
}
|
|
1066
|
-
return {
|
|
1067
|
-
count,
|
|
1068
|
-
bytes: sampleBytes,
|
|
1069
|
-
idAt,
|
|
1070
|
-
ordinalOf(docId) {
|
|
1071
|
-
if (count === 0) return -1;
|
|
1072
|
-
const window = windowOf(docId);
|
|
1073
|
-
let low = window.from;
|
|
1074
|
-
let high = window.to;
|
|
1075
|
-
while (low < high) {
|
|
1076
|
-
const middle = low + high >>> 1;
|
|
1077
|
-
const ordinal = ordinalAtPosition(middle);
|
|
1078
|
-
const order = compareCodePoints(idAt(ordinal), docId);
|
|
1079
|
-
if (order === 0) return ordinal;
|
|
1080
|
-
if (order < 0) low = middle + 1;
|
|
1081
|
-
else high = middle;
|
|
1082
|
-
}
|
|
1083
|
-
return -1;
|
|
1084
|
-
},
|
|
1085
|
-
sortedOrdinalAt: ordinalAtPosition,
|
|
1086
|
-
collectSortedIds(excluded) {
|
|
1087
|
-
const kept = [];
|
|
1088
|
-
for (let position = 0; position < count; position++) {
|
|
1089
|
-
const ordinal = ordinalAtPosition(position);
|
|
1090
|
-
if (!excluded(ordinal)) kept.push(idAt(ordinal));
|
|
1091
|
-
}
|
|
1092
|
-
return kept;
|
|
1093
|
-
}
|
|
1094
|
-
};
|
|
1095
|
-
}
|
|
1096
|
-
|
|
1097
|
-
// src/persistence/paged/lists.ts
|
|
1098
|
-
var LITTLE_ENDIAN_PLATFORM2 = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
1099
|
-
function swappedInPlace(bytes, width) {
|
|
1100
|
-
for (let at = 0; at + width <= bytes.length; at += width) {
|
|
1101
|
-
for (let low = 0, high = width - 1; low < high; low++, high--) {
|
|
1102
|
-
const held = bytes[at + low];
|
|
1103
|
-
bytes[at + low] = bytes[at + high];
|
|
1104
|
-
bytes[at + high] = held;
|
|
1105
|
-
}
|
|
1106
|
-
}
|
|
1107
|
-
}
|
|
1108
|
-
function fromLittleEndian(bytes, width) {
|
|
1109
|
-
if (!LITTLE_ENDIAN_PLATFORM2) swappedInPlace(bytes, width);
|
|
1110
|
-
}
|
|
1111
|
-
function requireListRules(list, snapshot, token) {
|
|
1112
|
-
const { docIds, fieldNameIndices } = list;
|
|
1113
|
-
const documents = snapshot.documentCount;
|
|
1114
|
-
const fields = snapshot.fieldNames.length;
|
|
1115
|
-
for (let row = 0; row < docIds.length; row++) {
|
|
1116
|
-
if (docIds[row] >= documents) {
|
|
1117
|
-
loadFailed(`Token ${token} of "${snapshot.key}" names document ${docIds[row]} of ${documents}`, { token, row });
|
|
1118
|
-
}
|
|
1119
|
-
if (fieldNameIndices[row] >= fields) {
|
|
1120
|
-
loadFailed(`Token ${token} of "${snapshot.key}" names field ${fieldNameIndices[row]} of ${fields}`, {
|
|
1121
|
-
token,
|
|
1122
|
-
row
|
|
1123
|
-
});
|
|
1124
|
-
}
|
|
1125
|
-
if (row === 0) continue;
|
|
1126
|
-
const before = docIds[row - 1];
|
|
1127
|
-
if (docIds[row] < before || docIds[row] === before && fieldNameIndices[row] <= fieldNameIndices[row - 1]) {
|
|
1128
|
-
loadFailed(`The list of token ${token} of "${snapshot.key}" descends at posting ${row}`, { token, row });
|
|
1129
|
-
}
|
|
1130
|
-
}
|
|
1131
|
-
}
|
|
1132
|
-
function createListReader(file, snapshot, sectionReader) {
|
|
1133
|
-
const { postingOffsets } = snapshot;
|
|
1134
|
-
const postings = sectionReader("postings");
|
|
1135
|
-
const positionOffsets = sectionReader("positionOffsets");
|
|
1136
|
-
const positions = sectionReader("positions");
|
|
1137
|
-
const totals = sectionReader("tokenTotals");
|
|
1138
|
-
const { cache, fileNumber } = file;
|
|
1139
|
-
const fault = new BigUint64Array(3);
|
|
1140
|
-
const total = new Float64Array(1);
|
|
1141
|
-
function listAt(token) {
|
|
1142
|
-
const held = cache.get(fileNumber, CACHED_LIST, token);
|
|
1143
|
-
if (held !== void 0) return held;
|
|
1144
|
-
if (postings === null) return loadFailed(`The segment "${snapshot.key}" holds no postings`);
|
|
1145
|
-
const start = postingOffsets[token];
|
|
1146
|
-
const count = postingOffsets[token + 1] - start;
|
|
1147
|
-
const buffer = new ArrayBuffer(count * POSTING_WIDTH_BYTES);
|
|
1148
|
-
const bytes = new Uint8Array(buffer);
|
|
1149
|
-
const descriptor = postings.native;
|
|
1150
|
-
if (descriptor !== null && file.native !== null) {
|
|
1151
|
-
const status = file.native.core.readPagedPostings(
|
|
1152
|
-
descriptor,
|
|
1153
|
-
cache.pageCache(),
|
|
1154
|
-
start,
|
|
1155
|
-
count,
|
|
1156
|
-
snapshot.documentCount,
|
|
1157
|
-
snapshot.fieldNames.length,
|
|
1158
|
-
bytes,
|
|
1159
|
-
fault
|
|
1160
|
-
);
|
|
1161
|
-
requireNativePage(status, file.path, "postings", fault[2]);
|
|
1162
|
-
} else {
|
|
1163
|
-
postings.read(start * POSTING_WIDTH_BYTES, bytes);
|
|
1164
|
-
}
|
|
1165
|
-
if (!LITTLE_ENDIAN_PLATFORM2) {
|
|
1166
|
-
swappedInPlace(bytes.subarray(0, count * 4), 4);
|
|
1167
|
-
swappedInPlace(bytes.subarray(count * 4), 2);
|
|
1168
|
-
}
|
|
1169
|
-
const list = {
|
|
1170
|
-
docIds: new Uint32Array(buffer, 0, count),
|
|
1171
|
-
termFrequencies: new Uint16Array(buffer, count * 4, count),
|
|
1172
|
-
fieldNameIndices: new Uint16Array(buffer, count * 6, count)
|
|
1173
|
-
};
|
|
1174
|
-
if (descriptor === null || file.native === null) requireListRules(list, snapshot, token);
|
|
1175
|
-
cache.put(fileNumber, CACHED_LIST, token, list, buffer.byteLength);
|
|
1176
|
-
return list;
|
|
1177
|
-
}
|
|
1178
|
-
function positionsAt(token) {
|
|
1179
|
-
if (positionOffsets === null || positions === null) return null;
|
|
1180
|
-
const held = cache.get(fileNumber, CACHED_POSITIONS, token);
|
|
1181
|
-
if (held !== void 0) return held;
|
|
1182
|
-
const start = postingOffsets[token];
|
|
1183
|
-
const count = postingOffsets[token + 1] - start;
|
|
1184
|
-
const offsetBytes = new Uint8Array((count + 1) * POSITION_WIDTH_BYTES);
|
|
1185
|
-
positionOffsets.read(start * POSITION_WIDTH_BYTES, offsetBytes);
|
|
1186
|
-
fromLittleEndian(offsetBytes, POSITION_WIDTH_BYTES);
|
|
1187
|
-
const absolute = new Uint32Array(offsetBytes.buffer);
|
|
1188
|
-
const first = absolute[0];
|
|
1189
|
-
const end = absolute[count];
|
|
1190
|
-
if (end < first || end * POSITION_WIDTH_BYTES > positions.length || start === 0 && first !== 0 || (start + count + 1) * POSITION_WIDTH_BYTES === positionOffsets.length && end * POSITION_WIDTH_BYTES !== positions.length)
|
|
1191
|
-
loadFailed(`The positions of token ${token} of "${snapshot.key}" leave their section`, { token });
|
|
1192
|
-
const offsets = new Uint32Array(count + 1);
|
|
1193
|
-
const valueBytes = new Uint8Array((end - first) * POSITION_WIDTH_BYTES);
|
|
1194
|
-
const loaded = { offsets, values: new Uint32Array(valueBytes.buffer) };
|
|
1195
|
-
const offsetSection = positionOffsets.native;
|
|
1196
|
-
const valueSection = positions.native;
|
|
1197
|
-
if (offsetSection !== null && valueSection !== null && file.native !== null) {
|
|
1198
|
-
const status = file.native.core.readPagedPositions(
|
|
1199
|
-
offsetSection,
|
|
1200
|
-
valueSection,
|
|
1201
|
-
cache.pageCache(),
|
|
1202
|
-
start,
|
|
1203
|
-
count,
|
|
1204
|
-
offsets,
|
|
1205
|
-
loaded.values,
|
|
1206
|
-
fault
|
|
1207
|
-
);
|
|
1208
|
-
requireNativePage(
|
|
1209
|
-
status,
|
|
1210
|
-
file.path,
|
|
1211
|
-
fault[1] === offsetSection.offset ? "positionOffsets" : "positions",
|
|
1212
|
-
fault[2]
|
|
1213
|
-
);
|
|
1214
|
-
} else {
|
|
1215
|
-
for (let row = 0; row <= count; row++) {
|
|
1216
|
-
if (absolute[row] < first || absolute[row] > end || row > 0 && absolute[row] < absolute[row - 1]) {
|
|
1217
|
-
loadFailed(`The position offsets of token ${token} of "${snapshot.key}" leave their list`, { token, row });
|
|
1218
|
-
}
|
|
1219
|
-
offsets[row] = absolute[row] - first;
|
|
1220
|
-
}
|
|
1221
|
-
positions.read(first * POSITION_WIDTH_BYTES, valueBytes);
|
|
1222
|
-
fromLittleEndian(valueBytes, POSITION_WIDTH_BYTES);
|
|
1223
|
-
for (let row = 0; row < count; row++) {
|
|
1224
|
-
for (let at = offsets[row] + 1; at < offsets[row + 1]; at++) {
|
|
1225
|
-
if (loaded.values[at] < loaded.values[at - 1])
|
|
1226
|
-
loadFailed(`The positions of token ${token} of "${snapshot.key}" descend`, { token, row });
|
|
1227
|
-
}
|
|
1228
|
-
}
|
|
1229
|
-
}
|
|
1230
|
-
cache.put(fileNumber, CACHED_POSITIONS, token, loaded, offsetBytes.byteLength + valueBytes.byteLength);
|
|
1428
|
+
if (raw.byteLength !== rawLength) {
|
|
1429
|
+
loadFailed(`Document block ${block} of "${key}" inflates to ${raw.byteLength} bytes of ${rawLength}`, { block });
|
|
1430
|
+
}
|
|
1431
|
+
return raw;
|
|
1432
|
+
}
|
|
1433
|
+
function startsOf(block, raw) {
|
|
1434
|
+
const first = firstOrdinalOf(block);
|
|
1435
|
+
const end = block + 1 < blockCount ? firstOrdinalOf(block + 1) : documentCount;
|
|
1436
|
+
const starts = new Uint32Array(end - first + 1);
|
|
1437
|
+
const bytes = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);
|
|
1438
|
+
let cursor = 0;
|
|
1439
|
+
for (let at = 0; at < end - first; at++) {
|
|
1440
|
+
starts[at] = cursor;
|
|
1441
|
+
const next = cursor < raw.byteLength ? endOfValue(bytes, cursor) : null;
|
|
1442
|
+
if (next === null)
|
|
1443
|
+
loadFailed(`Document block ${block} of "${key}" holds a broken document at ${at}`, { block, at });
|
|
1444
|
+
cursor = next;
|
|
1445
|
+
}
|
|
1446
|
+
starts[end - first] = cursor;
|
|
1447
|
+
if (cursor !== raw.byteLength)
|
|
1448
|
+
loadFailed(`Document block ${block} of "${key}" holds bytes past its documents`, { block });
|
|
1449
|
+
return starts;
|
|
1450
|
+
}
|
|
1451
|
+
function loadBlock(block) {
|
|
1452
|
+
const held = cache.get(fileNumber, CACHED_DOCUMENT_BLOCK, block);
|
|
1453
|
+
if (held !== void 0) return held;
|
|
1454
|
+
const bytes = inflate(block);
|
|
1455
|
+
const loaded = { bytes, starts: startsOf(block, bytes) };
|
|
1456
|
+
cache.put(fileNumber, CACHED_DOCUMENT_BLOCK, block, loaded, bytes.byteLength + loaded.starts.byteLength);
|
|
1231
1457
|
return loaded;
|
|
1232
1458
|
}
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1459
|
+
return {
|
|
1460
|
+
encodedDocumentAt(ordinal) {
|
|
1461
|
+
if (ordinal < 0 || ordinal >= documentCount) {
|
|
1462
|
+
loadFailed(`Document ${ordinal} lies outside "${key}", which holds ${documentCount}`, { ordinal });
|
|
1463
|
+
}
|
|
1464
|
+
const block = blockOf(ordinal);
|
|
1465
|
+
const { bytes, starts } = loadBlock(block);
|
|
1466
|
+
const row = ordinal - firstOrdinalOf(block);
|
|
1467
|
+
return bytes.subarray(starts[row], starts[row + 1]);
|
|
1468
|
+
}
|
|
1469
|
+
};
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
// src/persistence/paged/ids.ts
|
|
1473
|
+
var VALUE_BYTES3 = 4;
|
|
1474
|
+
var decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
1475
|
+
function uint32At(reader, index) {
|
|
1476
|
+
const bytes = reader.bytesAt(index * VALUE_BYTES3, VALUE_BYTES3);
|
|
1477
|
+
return new DataView(bytes.buffer, bytes.byteOffset, VALUE_BYTES3).getUint32(0, true);
|
|
1478
|
+
}
|
|
1479
|
+
function createPagedIdTable(snapshot, ids, idOffsets, idOrder) {
|
|
1480
|
+
const count = snapshot.documentCount;
|
|
1481
|
+
const { idSampleIds, idSamplePositions, key } = snapshot;
|
|
1482
|
+
let sampleBytes = idSamplePositions.byteLength;
|
|
1483
|
+
for (const id of idSampleIds) sampleBytes += id.length;
|
|
1484
|
+
function idAt(ordinal) {
|
|
1485
|
+
const from = uint32At(idOffsets, ordinal);
|
|
1486
|
+
const to = uint32At(idOffsets, ordinal + 1);
|
|
1487
|
+
if (to < from || to > ids.length || ordinal === 0 && from !== 0 || ordinal + 1 === count && to !== ids.length)
|
|
1488
|
+
loadFailed(`The ID of document ${ordinal} of "${key}" leaves its list`, { ordinal });
|
|
1489
|
+
try {
|
|
1490
|
+
return decoder.decode(ids.bytesAt(from, to - from));
|
|
1491
|
+
} catch {
|
|
1492
|
+
return loadFailed(`The ID of document ${ordinal} of "${key}" is not valid UTF-8`, { ordinal });
|
|
1493
|
+
}
|
|
1237
1494
|
}
|
|
1238
|
-
function
|
|
1239
|
-
const
|
|
1495
|
+
function ordinalAtPosition(position) {
|
|
1496
|
+
const ordinal = uint32At(idOrder, position);
|
|
1497
|
+
if (ordinal >= count) loadFailed(`idOrder of "${key}" names document ${ordinal} of ${count}`, { position });
|
|
1498
|
+
return ordinal;
|
|
1499
|
+
}
|
|
1500
|
+
function windowOf(docId) {
|
|
1240
1501
|
let low = 0;
|
|
1241
|
-
let high =
|
|
1502
|
+
let high = idSampleIds.length;
|
|
1242
1503
|
while (low < high) {
|
|
1243
1504
|
const middle = low + high >>> 1;
|
|
1244
|
-
if (
|
|
1505
|
+
if (compareCodePoints(idSampleIds[middle], docId) <= 0) low = middle + 1;
|
|
1245
1506
|
else high = middle;
|
|
1246
1507
|
}
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
const listed = listedIndexOf(token);
|
|
1251
|
-
if (listed < 0) return null;
|
|
1252
|
-
const first = snapshot.blockListOffsets[listed];
|
|
1253
|
-
const blockCount = snapshot.blockListOffsets[listed + 1] - first;
|
|
1254
|
-
const { blockSummaries } = snapshot;
|
|
1255
|
-
const view = new DataView(blockSummaries.buffer, blockSummaries.byteOffset, blockSummaries.byteLength);
|
|
1256
|
-
const bounds = {
|
|
1257
|
-
structureRevision: 0,
|
|
1258
|
-
blockCount,
|
|
1259
|
-
entryEnd: new Int32Array(blockCount),
|
|
1260
|
-
maxTermFrequency: new Int32Array(blockCount),
|
|
1261
|
-
minFieldLength: new Float64Array(blockCount),
|
|
1262
|
-
maxEntriesPerDocument: new Int32Array(blockCount),
|
|
1263
|
-
documentCount: new Int32Array(blockCount)
|
|
1264
|
-
};
|
|
1265
|
-
const length = postingOffsets[token + 1] - postingOffsets[token];
|
|
1266
|
-
for (let block = 0; block < blockCount; block++) {
|
|
1267
|
-
const place = (first + block) * BLOCK_SUMMARY_BYTES;
|
|
1268
|
-
bounds.entryEnd[block] = view.getUint32(place, true);
|
|
1269
|
-
bounds.maxTermFrequency[block] = view.getUint16(place + 4, true);
|
|
1270
|
-
bounds.minFieldLength[block] = view.getFloat64(place + 6, true);
|
|
1271
|
-
bounds.maxEntriesPerDocument[block] = view.getUint32(place + 14, true);
|
|
1272
|
-
bounds.documentCount[block] = view.getUint32(place + 18, true);
|
|
1273
|
-
const before = block === 0 ? 0 : bounds.entryEnd[block - 1];
|
|
1274
|
-
if (bounds.entryEnd[block] <= before || bounds.entryEnd[block] > length) {
|
|
1275
|
-
loadFailed(`Block ${block} of token ${token} of "${snapshot.key}" ends outside its list`, { token, block });
|
|
1276
|
-
}
|
|
1277
|
-
}
|
|
1278
|
-
if (blockCount === 0 || bounds.entryEnd[blockCount - 1] !== length) {
|
|
1279
|
-
loadFailed(`The blocks of token ${token} of "${snapshot.key}" fail to cover its list`, { token });
|
|
1280
|
-
}
|
|
1281
|
-
return bounds;
|
|
1282
|
-
}
|
|
1283
|
-
return { listAt, positionsAt, totalAt, storedBoundsAt };
|
|
1284
|
-
}
|
|
1285
|
-
function createPagedPostingViews(snapshot, lists, tombstones, counts) {
|
|
1286
|
-
const { postingOffsets, tracksPositions } = snapshot;
|
|
1287
|
-
const views = /* @__PURE__ */ new Map();
|
|
1288
|
-
function buildView(token) {
|
|
1289
|
-
const length = postingOffsets[token + 1] - postingOffsets[token];
|
|
1290
|
-
return {
|
|
1291
|
-
length,
|
|
1292
|
-
get docIds() {
|
|
1293
|
-
return lists.listAt(token).docIds;
|
|
1294
|
-
},
|
|
1295
|
-
get termFrequencies() {
|
|
1296
|
-
return lists.listAt(token).termFrequencies;
|
|
1297
|
-
},
|
|
1298
|
-
get fieldNameIndices() {
|
|
1299
|
-
return lists.listAt(token).fieldNameIndices;
|
|
1300
|
-
},
|
|
1301
|
-
get positions() {
|
|
1302
|
-
const loaded = lists.positionsAt(token);
|
|
1303
|
-
if (loaded === null) return null;
|
|
1304
|
-
const rows = new Array(length);
|
|
1305
|
-
for (let row = 0; row < length; row++) {
|
|
1306
|
-
rows[row] = [...loaded.values.subarray(loaded.offsets[row], loaded.offsets[row + 1])];
|
|
1307
|
-
}
|
|
1308
|
-
return rows;
|
|
1309
|
-
},
|
|
1310
|
-
holdsPositions: tracksPositions,
|
|
1311
|
-
positionCountAt(row) {
|
|
1312
|
-
const loaded = lists.positionsAt(token);
|
|
1313
|
-
return loaded === null ? 0 : loaded.offsets[row + 1] - loaded.offsets[row];
|
|
1314
|
-
},
|
|
1315
|
-
get positionStore() {
|
|
1316
|
-
return lists.positionsAt(token)?.values ?? null;
|
|
1317
|
-
},
|
|
1318
|
-
positionStartAt(row) {
|
|
1319
|
-
return lists.positionsAt(token)?.offsets[row] ?? 0;
|
|
1320
|
-
},
|
|
1321
|
-
docIdSet: {
|
|
1322
|
-
get size() {
|
|
1323
|
-
return counts.documentFrequency(token);
|
|
1324
|
-
}
|
|
1325
|
-
},
|
|
1326
|
-
deletedDocs: tombstones,
|
|
1327
|
-
get totalTermFrequency() {
|
|
1328
|
-
return lists.totalAt(token) - counts.removedOccurrences(token);
|
|
1329
|
-
},
|
|
1330
|
-
get structureRevision() {
|
|
1331
|
-
return tombstones.revision;
|
|
1332
|
-
},
|
|
1333
|
-
ordered: true,
|
|
1334
|
-
storedBlockBounds() {
|
|
1335
|
-
return tombstones.size === 0 ? lists.storedBoundsAt(token) : null;
|
|
1336
|
-
}
|
|
1337
|
-
};
|
|
1508
|
+
const from = low === 0 ? 0 : idSamplePositions[low - 1];
|
|
1509
|
+
const to = low < idSamplePositions.length ? idSamplePositions[low] + 1 : count;
|
|
1510
|
+
return { from, to: Math.min(to, count) };
|
|
1338
1511
|
}
|
|
1339
1512
|
return {
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1513
|
+
count,
|
|
1514
|
+
bytes: sampleBytes,
|
|
1515
|
+
idAt,
|
|
1516
|
+
ordinalOf(docId) {
|
|
1517
|
+
if (count === 0) return -1;
|
|
1518
|
+
const window = windowOf(docId);
|
|
1519
|
+
let low = window.from;
|
|
1520
|
+
let high = window.to;
|
|
1521
|
+
while (low < high) {
|
|
1522
|
+
const middle = low + high >>> 1;
|
|
1523
|
+
const ordinal = ordinalAtPosition(middle);
|
|
1524
|
+
const order = compareCodePoints(idAt(ordinal), docId);
|
|
1525
|
+
if (order === 0) return ordinal;
|
|
1526
|
+
if (order < 0) low = middle + 1;
|
|
1527
|
+
else high = middle;
|
|
1346
1528
|
}
|
|
1347
|
-
return
|
|
1529
|
+
return -1;
|
|
1348
1530
|
},
|
|
1349
|
-
|
|
1350
|
-
|
|
1531
|
+
sortedOrdinalAt: ordinalAtPosition,
|
|
1532
|
+
collectSortedIds(excluded) {
|
|
1533
|
+
const kept = [];
|
|
1534
|
+
for (let position = 0; position < count; position++) {
|
|
1535
|
+
const ordinal = ordinalAtPosition(position);
|
|
1536
|
+
if (!excluded(ordinal)) kept.push(idAt(ordinal));
|
|
1537
|
+
}
|
|
1538
|
+
return kept;
|
|
1351
1539
|
}
|
|
1352
1540
|
};
|
|
1353
1541
|
}
|
|
@@ -1577,76 +1765,59 @@ function assembleOpenedSegment(file, snapshot, cache) {
|
|
|
1577
1765
|
});
|
|
1578
1766
|
}
|
|
1579
1767
|
|
|
1580
|
-
// src/persistence/paged/
|
|
1581
|
-
async function
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
}
|
|
1600
|
-
async function isPagedSegmentFile(directory, key) {
|
|
1601
|
-
return await envelopeVersionOf(directory, key) === PAGED_SEGMENT_ENVELOPE_VERSION;
|
|
1602
|
-
}
|
|
1603
|
-
async function requirePagedPayloadChecksum(directory, listing) {
|
|
1604
|
-
const { readRange } = directory;
|
|
1605
|
-
const { key, bytes } = listing;
|
|
1606
|
-
if (readRange === void 0) return loadFailed(`The directory holding "${key}" reads no ranges`, { key });
|
|
1607
|
-
const header2 = readHeader(await readRange.call(directory, key, 0, HEADER_SIZE));
|
|
1608
|
-
if (HEADER_SIZE + header2.payloadLength !== bytes) {
|
|
1609
|
-
loadFailed(`The segment "${key}" holds ${bytes} bytes where its header gives ${HEADER_SIZE + header2.payloadLength}`);
|
|
1610
|
-
}
|
|
1611
|
-
await requirePayloadChecksum(key, header2, (offset, length) => readRange.call(directory, key, offset, length));
|
|
1612
|
-
}
|
|
1613
|
-
async function requirePagedPathChecksum(path) {
|
|
1614
|
-
const read = async (offset, length) => {
|
|
1615
|
-
const bytes = new Uint8Array(length);
|
|
1616
|
-
if (!await readFileRangeLater(path, offset, bytes))
|
|
1617
|
-
loadFailed(`The segment "${path}" cannot be read at ${offset}`);
|
|
1618
|
-
return bytes;
|
|
1768
|
+
// src/persistence/paged/segment-scan.ts
|
|
1769
|
+
async function openPagedSegmentScan(directory, ref) {
|
|
1770
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
1771
|
+
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
1772
|
+
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
1773
|
+
const { paged } = segment;
|
|
1774
|
+
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
1775
|
+
const ids = segment.arrays.idTable;
|
|
1776
|
+
const tombstones = new Set(snapshot.tombstones);
|
|
1777
|
+
return {
|
|
1778
|
+
count: snapshot.documentCount,
|
|
1779
|
+
schema: snapshot.schema,
|
|
1780
|
+
tombstones,
|
|
1781
|
+
idAt: (ordinal) => ids.idAt(ordinal),
|
|
1782
|
+
holds: (docId) => ids.ordinalOf(docId) >= 0,
|
|
1783
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
|
|
1784
|
+
close: () => {
|
|
1785
|
+
paged.close();
|
|
1786
|
+
cache.clear();
|
|
1787
|
+
}
|
|
1619
1788
|
};
|
|
1620
|
-
await requirePayloadChecksum(path, readHeader(await read(0, HEADER_SIZE)), read);
|
|
1621
1789
|
}
|
|
1622
|
-
async function
|
|
1623
|
-
const
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1790
|
+
async function checkPagedSegmentFully(directory, ref) {
|
|
1791
|
+
const listing = await pagedListingOf(directory, ref);
|
|
1792
|
+
await requirePagedPayloadChecksum(directory, listing);
|
|
1793
|
+
const snapshot = openPagedSnapshot(listing);
|
|
1794
|
+
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
1795
|
+
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
1796
|
+
const { paged } = segment;
|
|
1797
|
+
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
1798
|
+
try {
|
|
1799
|
+
await checkPagedContents(segment, cache);
|
|
1800
|
+
return snapshot.schema;
|
|
1801
|
+
} finally {
|
|
1802
|
+
paged.close();
|
|
1803
|
+
cache.clear();
|
|
1628
1804
|
}
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1805
|
+
}
|
|
1806
|
+
async function bytesThatSegmentsHoldInMemory(directory, manifest) {
|
|
1807
|
+
let bytes = 0;
|
|
1808
|
+
for (const partition of manifest.partitions) {
|
|
1809
|
+
for (const ref of partition.segments) {
|
|
1810
|
+
const paged = await isPagedSegmentFile(directory, ref.key);
|
|
1811
|
+
bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
|
|
1812
|
+
}
|
|
1636
1813
|
}
|
|
1814
|
+
return bytes;
|
|
1637
1815
|
}
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
if (length === null) loadFailed(`The segment "${ref.key}" that the manifest lists is missing`, { key: ref.key });
|
|
1642
|
-
if (length !== ref.bytes) {
|
|
1643
|
-
loadFailed(`The segment "${ref.key}" holds ${length} bytes where the manifest lists ${ref.bytes}`, {
|
|
1644
|
-
key: ref.key,
|
|
1645
|
-
bytes: length,
|
|
1646
|
-
listed: ref.bytes
|
|
1647
|
-
});
|
|
1816
|
+
function countsAsLatestHolder(scans, at, docId) {
|
|
1817
|
+
for (let later = at + 1; later < scans.length; later++) {
|
|
1818
|
+
if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
|
|
1648
1819
|
}
|
|
1649
|
-
return
|
|
1820
|
+
return true;
|
|
1650
1821
|
}
|
|
1651
1822
|
|
|
1652
1823
|
// src/persistence/paged/older-holders.ts
|
|
@@ -1702,122 +1873,6 @@ async function pagedTombstonesFor(directory, older, written, deleted) {
|
|
|
1702
1873
|
return [...tombstones];
|
|
1703
1874
|
}
|
|
1704
1875
|
|
|
1705
|
-
// src/persistence/paged/check-contents.ts
|
|
1706
|
-
function requireIds(segment, snapshot) {
|
|
1707
|
-
const ids = segment.arrays.idTable;
|
|
1708
|
-
let previous = null;
|
|
1709
|
-
for (let position = 0; position < snapshot.documentCount; position++) {
|
|
1710
|
-
const id = ids.idAt(ids.sortedOrdinalAt(position));
|
|
1711
|
-
if (previous !== null && compareCodePoints(previous, id) >= 0)
|
|
1712
|
-
loadFailed(`The ID order of "${snapshot.key}" does not ascend at position ${position}`, { position });
|
|
1713
|
-
if (position % ID_SAMPLE_INTERVAL === 0 && snapshot.idSampleIds[position / ID_SAMPLE_INTERVAL] !== id)
|
|
1714
|
-
loadFailed(`The ID sample of "${snapshot.key}" differs at position ${position}`, { position });
|
|
1715
|
-
previous = id;
|
|
1716
|
-
}
|
|
1717
|
-
}
|
|
1718
|
-
function requirePostings(lists, snapshot) {
|
|
1719
|
-
const fields = snapshot.fieldNames.map((field) => {
|
|
1720
|
-
const at = snapshot.fieldLengthNames.indexOf(field);
|
|
1721
|
-
return at < 0 ? null : snapshot.fieldLengthColumns[at];
|
|
1722
|
-
});
|
|
1723
|
-
for (let token = 0; token < snapshot.tokenTable.documentFrequencies.length; token++) {
|
|
1724
|
-
const list = lists.listAt(token);
|
|
1725
|
-
lists.positionsAt(token);
|
|
1726
|
-
const bounds = lists.storedBoundsAt(token);
|
|
1727
|
-
if (list.docIds.length > SMALLEST_BLOCK_POSTINGS !== (bounds !== null))
|
|
1728
|
-
loadFailed(`The block summaries of token ${token} of "${snapshot.key}" do not cover its postings`, { token });
|
|
1729
|
-
let documents = 0;
|
|
1730
|
-
let occurrences = 0;
|
|
1731
|
-
let block = 0;
|
|
1732
|
-
let start = 0;
|
|
1733
|
-
let maxFrequency = 0;
|
|
1734
|
-
let minLength = Number.POSITIVE_INFINITY;
|
|
1735
|
-
let maxEntries = 0;
|
|
1736
|
-
let blockDocuments = 0;
|
|
1737
|
-
let row = 0;
|
|
1738
|
-
while (row < list.docIds.length) {
|
|
1739
|
-
const document = list.docIds[row];
|
|
1740
|
-
let entries = 0;
|
|
1741
|
-
do {
|
|
1742
|
-
const frequency = list.termFrequencies[row];
|
|
1743
|
-
occurrences += frequency;
|
|
1744
|
-
maxFrequency = Math.max(maxFrequency, frequency);
|
|
1745
|
-
minLength = Math.min(minLength, fields[list.fieldNameIndices[row]]?.[document] ?? 0);
|
|
1746
|
-
row++;
|
|
1747
|
-
entries++;
|
|
1748
|
-
} while (row < list.docIds.length && list.docIds[row] === document);
|
|
1749
|
-
documents++;
|
|
1750
|
-
blockDocuments++;
|
|
1751
|
-
maxEntries = Math.max(maxEntries, entries);
|
|
1752
|
-
if (bounds === null || row - start < SMALLEST_BLOCK_POSTINGS && row < list.docIds.length) continue;
|
|
1753
|
-
if (block >= bounds.blockCount || bounds.entryEnd[block] !== row || bounds.maxTermFrequency[block] !== maxFrequency || bounds.minFieldLength[block] !== minLength || bounds.maxEntriesPerDocument[block] !== maxEntries || bounds.documentCount[block] !== blockDocuments)
|
|
1754
|
-
loadFailed(`Block ${block} of token ${token} of "${snapshot.key}" differs from its postings`, { token, block });
|
|
1755
|
-
block++;
|
|
1756
|
-
start = row;
|
|
1757
|
-
maxFrequency = 0;
|
|
1758
|
-
minLength = Number.POSITIVE_INFINITY;
|
|
1759
|
-
maxEntries = 0;
|
|
1760
|
-
blockDocuments = 0;
|
|
1761
|
-
}
|
|
1762
|
-
if (documents !== snapshot.tokenTable.documentFrequencies[token] || occurrences !== lists.totalAt(token))
|
|
1763
|
-
loadFailed(`The counts of token ${token} of "${snapshot.key}" differ from its postings`, { token });
|
|
1764
|
-
}
|
|
1765
|
-
}
|
|
1766
|
-
async function checkPagedContents(segment, cache) {
|
|
1767
|
-
const { paged } = segment;
|
|
1768
|
-
if (paged === null) return;
|
|
1769
|
-
const { snapshot } = paged;
|
|
1770
|
-
for (const section of snapshot.sections) await checkEverySectionPage(snapshot.path, section, () => true);
|
|
1771
|
-
const file = openPagedFile(snapshot.path, cache);
|
|
1772
|
-
try {
|
|
1773
|
-
const sectionReader = (name) => {
|
|
1774
|
-
const section = sectionNamed(snapshot.sections, name);
|
|
1775
|
-
return section === void 0 ? null : createSectionReader(file, section);
|
|
1776
|
-
};
|
|
1777
|
-
const required = (name) => {
|
|
1778
|
-
const section = sectionNamed(snapshot.sections, name);
|
|
1779
|
-
if (section === void 0) loadFailed(`The segment "${snapshot.key}" holds no section "${name}"`);
|
|
1780
|
-
return createSectionReader(file, section);
|
|
1781
|
-
};
|
|
1782
|
-
const removals = createRemovalReader(
|
|
1783
|
-
file,
|
|
1784
|
-
required("removalOffsets"),
|
|
1785
|
-
required("removals"),
|
|
1786
|
-
segment.arrays.tokenTable.size
|
|
1787
|
-
);
|
|
1788
|
-
for (let ordinal = 0; ordinal < snapshot.documentCount; ordinal++) {
|
|
1789
|
-
decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields);
|
|
1790
|
-
paged.writeRecordAt(ordinal);
|
|
1791
|
-
removals(ordinal);
|
|
1792
|
-
}
|
|
1793
|
-
if (snapshot.documentCount === 0) {
|
|
1794
|
-
for (const name of ["idOffsets", "removalOffsets"]) {
|
|
1795
|
-
const offsets = required(name).bytesAt(0, 4);
|
|
1796
|
-
if (new DataView(offsets.buffer, offsets.byteOffset, 4).getUint32(0, true) !== 0)
|
|
1797
|
-
loadFailed(`Section "${name}" of "${snapshot.key}" does not start at zero`);
|
|
1798
|
-
}
|
|
1799
|
-
}
|
|
1800
|
-
if (snapshot.tracksPositions && snapshot.postingOffsets[snapshot.postingOffsets.length - 1] === 0) {
|
|
1801
|
-
const offsets = required("positionOffsets").bytesAt(0, 4);
|
|
1802
|
-
if (new DataView(offsets.buffer, offsets.byteOffset, 4).getUint32(0, true) !== 0)
|
|
1803
|
-
loadFailed(`The position offsets of "${snapshot.key}" do not start at zero`);
|
|
1804
|
-
}
|
|
1805
|
-
requireIds(segment, snapshot);
|
|
1806
|
-
requirePostings(createListReader(file, snapshot, sectionReader), snapshot);
|
|
1807
|
-
segment.arrays.readSurfaceForms();
|
|
1808
|
-
} finally {
|
|
1809
|
-
closePagedFile(file);
|
|
1810
|
-
}
|
|
1811
|
-
}
|
|
1812
|
-
async function checkPagedSelections(selections) {
|
|
1813
|
-
for (const { segment } of selections) {
|
|
1814
|
-
const { paged } = segment;
|
|
1815
|
-
if (paged === null || paged.payloadChecked) continue;
|
|
1816
|
-
await requirePagedPathChecksum(paged.snapshot.path);
|
|
1817
|
-
paged.payloadChecked = true;
|
|
1818
|
-
}
|
|
1819
|
-
}
|
|
1820
|
-
|
|
1821
1876
|
// src/core/segment-merges/metadata.ts
|
|
1822
1877
|
function emptySource(input) {
|
|
1823
1878
|
return {
|
|
@@ -3608,61 +3663,6 @@ async function mergePagedWindow(input) {
|
|
|
3608
3663
|
}
|
|
3609
3664
|
}
|
|
3610
3665
|
|
|
3611
|
-
// src/persistence/paged/segment-scan.ts
|
|
3612
|
-
async function openPagedSegmentScan(directory, ref) {
|
|
3613
|
-
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
3614
|
-
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
3615
|
-
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
3616
|
-
const { paged } = segment;
|
|
3617
|
-
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
3618
|
-
const ids = segment.arrays.idTable;
|
|
3619
|
-
const tombstones = new Set(snapshot.tombstones);
|
|
3620
|
-
return {
|
|
3621
|
-
count: snapshot.documentCount,
|
|
3622
|
-
schema: snapshot.schema,
|
|
3623
|
-
tombstones,
|
|
3624
|
-
idAt: (ordinal) => ids.idAt(ordinal),
|
|
3625
|
-
holds: (docId) => ids.ordinalOf(docId) >= 0,
|
|
3626
|
-
storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
|
|
3627
|
-
close: () => {
|
|
3628
|
-
paged.close();
|
|
3629
|
-
cache.clear();
|
|
3630
|
-
}
|
|
3631
|
-
};
|
|
3632
|
-
}
|
|
3633
|
-
async function checkPagedSegmentFully(directory, ref) {
|
|
3634
|
-
const listing = await pagedListingOf(directory, ref);
|
|
3635
|
-
await requirePagedPayloadChecksum(directory, listing);
|
|
3636
|
-
const snapshot = openPagedSnapshot(listing);
|
|
3637
|
-
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
3638
|
-
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
3639
|
-
const { paged } = segment;
|
|
3640
|
-
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
3641
|
-
try {
|
|
3642
|
-
await checkPagedContents(segment, cache);
|
|
3643
|
-
return snapshot.schema;
|
|
3644
|
-
} finally {
|
|
3645
|
-
paged.close();
|
|
3646
|
-
cache.clear();
|
|
3647
|
-
}
|
|
3648
|
-
}
|
|
3649
|
-
async function bytesThatSegmentsHoldInMemory(directory, manifest) {
|
|
3650
|
-
let bytes = 0;
|
|
3651
|
-
for (const partition of manifest.partitions) {
|
|
3652
|
-
for (const ref of partition.segments) {
|
|
3653
|
-
const paged = await isPagedSegmentFile(directory, ref.key);
|
|
3654
|
-
bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
|
|
3655
|
-
}
|
|
3656
|
-
}
|
|
3657
|
-
return bytes;
|
|
3658
|
-
}
|
|
3659
|
-
function countsAsLatestHolder(scans, at, docId) {
|
|
3660
|
-
for (let later = at + 1; later < scans.length; later++) {
|
|
3661
|
-
if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
|
|
3662
|
-
}
|
|
3663
|
-
return true;
|
|
3664
|
-
}
|
|
3665
|
-
|
|
3666
3666
|
// src/persistence/durability/partition-checkpoint.ts
|
|
3667
3667
|
function checkpointLastSeqNo(checkpoint, partitionId) {
|
|
3668
3668
|
for (const c of checkpoint) {
|
|
@@ -3905,6 +3905,9 @@ function validateEntryPayload(decoded) {
|
|
|
3905
3905
|
if (entry.document !== null && !(entry.document instanceof Uint8Array)) {
|
|
3906
3906
|
throw new Error('Invalid EntryPayload: "entry.document" must be Uint8Array or null');
|
|
3907
3907
|
}
|
|
3908
|
+
if ("previousDocument" in entry && entry.previousDocument !== null && !(entry.previousDocument instanceof Uint8Array)) {
|
|
3909
|
+
throw new Error('Invalid EntryPayload: "entry.previousDocument" must be Uint8Array or null where present');
|
|
3910
|
+
}
|
|
3908
3911
|
if (typeof entry.checksum !== "number") {
|
|
3909
3912
|
throw new Error('Invalid EntryPayload: "entry.checksum" must be a number');
|
|
3910
3913
|
}
|
|
@@ -4045,9 +4048,15 @@ function validateAckPayload(decoded) {
|
|
|
4045
4048
|
return decoded;
|
|
4046
4049
|
}
|
|
4047
4050
|
|
|
4051
|
+
// src/distribution/replication/constants.ts
|
|
4052
|
+
var REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES = 40;
|
|
4053
|
+
var DEFAULT_LOG_RETENTION_BYTES = 268435456;
|
|
4054
|
+
var INSYNC_CAS_ATTEMPTS = 5;
|
|
4055
|
+
var SNAPSHOT_CHUNK_SIZE = 65536;
|
|
4056
|
+
|
|
4048
4057
|
// src/distribution/replication/entry-checksum.ts
|
|
4049
|
-
function
|
|
4050
|
-
const
|
|
4058
|
+
function checksumFields(entry) {
|
|
4059
|
+
const fields = [
|
|
4051
4060
|
entry.seqNo,
|
|
4052
4061
|
entry.primaryTerm,
|
|
4053
4062
|
entry.operation,
|
|
@@ -4055,24 +4064,54 @@ function computeEntryChecksum(entry) {
|
|
|
4055
4064
|
entry.indexName,
|
|
4056
4065
|
entry.documentId,
|
|
4057
4066
|
entry.document
|
|
4058
|
-
]
|
|
4059
|
-
|
|
4067
|
+
];
|
|
4068
|
+
if (entry.previousDocument !== void 0) fields.push(entry.previousDocument);
|
|
4069
|
+
return fields;
|
|
4070
|
+
}
|
|
4071
|
+
function computeEntryChecksum(entry) {
|
|
4072
|
+
return crc32(encode(checksumFields(entry)));
|
|
4060
4073
|
}
|
|
4061
4074
|
function buildEntry(fields) {
|
|
4062
|
-
|
|
4075
|
+
if (fields.previousDocument === void 0) {
|
|
4076
|
+
return {
|
|
4077
|
+
seqNo: fields.seqNo,
|
|
4078
|
+
primaryTerm: fields.primaryTerm,
|
|
4079
|
+
operation: fields.operation,
|
|
4080
|
+
partitionId: fields.partitionId,
|
|
4081
|
+
indexName: fields.indexName,
|
|
4082
|
+
documentId: fields.documentId,
|
|
4083
|
+
document: fields.document,
|
|
4084
|
+
checksum: computeEntryChecksum(fields)
|
|
4085
|
+
};
|
|
4086
|
+
}
|
|
4087
|
+
return {
|
|
4088
|
+
seqNo: fields.seqNo,
|
|
4089
|
+
primaryTerm: fields.primaryTerm,
|
|
4090
|
+
operation: fields.operation,
|
|
4091
|
+
partitionId: fields.partitionId,
|
|
4092
|
+
indexName: fields.indexName,
|
|
4093
|
+
documentId: fields.documentId,
|
|
4094
|
+
document: fields.document,
|
|
4095
|
+
previousDocument: fields.previousDocument,
|
|
4096
|
+
checksum: computeEntryChecksum(fields)
|
|
4097
|
+
};
|
|
4098
|
+
}
|
|
4099
|
+
function entryByteEstimate(entry) {
|
|
4100
|
+
return REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0) + (entry.previousDocument?.byteLength ?? 0);
|
|
4063
4101
|
}
|
|
4064
4102
|
|
|
4065
4103
|
// src/persistence/durability/wal-framing.ts
|
|
4066
4104
|
var WAL_MAGIC = new Uint8Array([78, 82, 83, 87]);
|
|
4067
4105
|
var WAL_FORMAT_VERSION = 1;
|
|
4106
|
+
var WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT = 2;
|
|
4068
4107
|
var SEGMENT_HEADER_SIZE = 8;
|
|
4069
4108
|
var RECORD_LENGTH_SIZE = 4;
|
|
4070
4109
|
var FRAME_CRC_SIZE = 4;
|
|
4071
4110
|
var MAX_WAL_RECORD_BYTES = 268435456;
|
|
4072
|
-
function writeSegmentHeader() {
|
|
4111
|
+
function writeSegmentHeader(formatVersion = WAL_FORMAT_VERSION) {
|
|
4073
4112
|
const header2 = new Uint8Array(SEGMENT_HEADER_SIZE);
|
|
4074
4113
|
header2.set(WAL_MAGIC, 0);
|
|
4075
|
-
header2[4] =
|
|
4114
|
+
header2[4] = formatVersion & 255;
|
|
4076
4115
|
return header2;
|
|
4077
4116
|
}
|
|
4078
4117
|
function frameRecord(entry) {
|
|
@@ -4094,10 +4133,10 @@ function checkSegmentHeader(data) {
|
|
|
4094
4133
|
}
|
|
4095
4134
|
}
|
|
4096
4135
|
const formatVersion = data[4];
|
|
4097
|
-
if (formatVersion >
|
|
4136
|
+
if (formatVersion > WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT) {
|
|
4098
4137
|
return {
|
|
4099
4138
|
ok: false,
|
|
4100
|
-
reason: `WAL format version ${formatVersion} is newer than supported version ${
|
|
4139
|
+
reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT}, so a newer release wrote this segment`,
|
|
4101
4140
|
code: ErrorCodes.ENVELOPE_VERSION_MISMATCH
|
|
4102
4141
|
};
|
|
4103
4142
|
}
|
|
@@ -4215,12 +4254,38 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
|
|
|
4215
4254
|
}
|
|
4216
4255
|
|
|
4217
4256
|
// src/persistence/durability/wal-segments.ts
|
|
4257
|
+
var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
|
|
4218
4258
|
function walPrefix(indexName, partitionId) {
|
|
4219
4259
|
return `${indexName}/wal/${partitionId}/`;
|
|
4220
4260
|
}
|
|
4261
|
+
function segmentKey2(indexName, partitionId, startSeqNo) {
|
|
4262
|
+
return `${walPrefix(indexName, partitionId)}${startSeqNo.toString().padStart(16, "0")}`;
|
|
4263
|
+
}
|
|
4264
|
+
function parseSegmentStartSeqNo(key) {
|
|
4265
|
+
const tail = key.slice(key.lastIndexOf("/") + 1);
|
|
4266
|
+
if (!SEGMENT_TAIL_PATTERN.test(tail)) {
|
|
4267
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Malformed WAL segment key: "${key}"`, { key });
|
|
4268
|
+
}
|
|
4269
|
+
const value = Number.parseInt(tail, 10);
|
|
4270
|
+
if (!Number.isSafeInteger(value)) {
|
|
4271
|
+
throw new NarsilError(
|
|
4272
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
4273
|
+
`WAL segment start sequence number is out of range: "${key}"`,
|
|
4274
|
+
{ key }
|
|
4275
|
+
);
|
|
4276
|
+
}
|
|
4277
|
+
return value;
|
|
4278
|
+
}
|
|
4221
4279
|
function nextSegmentKey(indexName, partitionId) {
|
|
4222
4280
|
return `${walPrefix(indexName, partitionId)}next`;
|
|
4223
4281
|
}
|
|
4282
|
+
function rewrittenHeaderKey(indexName, partitionId) {
|
|
4283
|
+
return `${walPrefix(indexName, partitionId)}header`;
|
|
4284
|
+
}
|
|
4285
|
+
async function removeUnusedSegmentFiles(directory, indexName, partitionId) {
|
|
4286
|
+
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
4287
|
+
await directory.remove(rewrittenHeaderKey(indexName, partitionId));
|
|
4288
|
+
}
|
|
4224
4289
|
function segmentStartSeqNo(key, prefix) {
|
|
4225
4290
|
if (!key.startsWith(prefix)) {
|
|
4226
4291
|
return null;
|
|
@@ -4924,4 +4989,4 @@ async function removeCheckpointGarbage(store, garbage) {
|
|
|
4924
4989
|
}
|
|
4925
4990
|
}
|
|
4926
4991
|
|
|
4927
|
-
export { FRAME_CRC_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, NOTHING_SAVED, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, buildEntry, bytesThatSegmentsHoldInMemory, captureWholePartition, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkSegmentHeader, checkpointGarbage, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, countsAsLatestHolder, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPagedFrozenSegment, createPrimaryResignMessage, decodeKeywordSegment, decodeWalEntry, durableRecordMissing, encodeMarkerSlot, frameRecord, isPagedSegmentFile, markerSlotOffset, mergePagedWindow, nextMarkerSlot, nextSegmentKey, openPagedSegmentScan, openPagedSnapshot, optionalIndexUuid, pagedListingOf, pagedTombstonesFor, partitionContentsOf, readCommitMarker, readSegmentFile, readWalSegments, removeCheckpointGarbage, residentSectionsOf, savedBytesOfManifest, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writePagedSegment, writeSegmentHeader };
|
|
4992
|
+
export { DEFAULT_LOG_RETENTION_BYTES, FRAME_CRC_SIZE, INSYNC_CAS_ATTEMPTS, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, NOTHING_SAVED, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, SEGMENT_TAIL_PATTERN, SNAPSHOT_CHUNK_SIZE, WAL_FORMAT_VERSION, WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT, buildEntry, bytesThatSegmentsHoldInMemory, captureWholePartition, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkSegmentHeader, checkpointGarbage, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, countsAsLatestHolder, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPagedFrozenSegment, createPrimaryResignMessage, decodeKeywordSegment, decodeWalEntry, durableRecordMissing, encodeMarkerSlot, entryByteEstimate, frameRecord, isPagedSegmentFile, markerSlotOffset, mergePagedWindow, nextMarkerSlot, nextSegmentKey, openPagedSegmentScan, openPagedSnapshot, optionalIndexUuid, pagedListingOf, pagedTombstonesFor, parseSegmentStartSeqNo, partitionContentsOf, readCommitMarker, readSegmentFile, readWalSegments, removeCheckpointGarbage, removeUnusedSegmentFiles, residentSectionsOf, rewrittenHeaderKey, savedBytesOfManifest, segmentKey2 as segmentKey, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writePagedSegment, writeSegmentHeader };
|