@delali/narsil 0.4.1-next.215 → 0.4.2-next.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-DPRCAVC2.mjs} +14 -8
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-WRRXR57N.mjs} +1 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-XBWABEL3.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-Y5CSZDOH.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-FEPCLXXY.mjs} +7 -1
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-HVPPDHSY.mjs} +259 -104
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-DBC4EJWB.mjs → chunk-33ZMFOAR.mjs} +1 -1
- package/dist/{chunk-NCIGNQJE.mjs → chunk-4GCHJ3XD.mjs} +930 -387
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-4WQNWAQE.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-5E3TJ4P3.mjs} +5 -5
- package/dist/{chunk-GXCDWLQI.mjs → chunk-5GYGHPCH.mjs} +7 -1
- package/dist/{chunk-SMQPZHPS.mjs → chunk-ATQ6JR3I.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-BBIAUHAB.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-DA32WXBT.mjs} +2 -2
- package/dist/{chunk-MSJD74GN.mjs → chunk-EALKCTWH.mjs} +5 -5
- package/dist/{chunk-755NM4PL.mjs → chunk-FOGTBAYN.mjs} +183 -21
- package/dist/{chunk-6GHX35ZM.mjs → chunk-GTBJSNN2.mjs} +1 -1
- package/dist/{chunk-DDUXEJXW.mjs → chunk-H6LWGMPQ.mjs} +45 -22
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-IL5DRCNU.mjs → chunk-KQWTS7AM.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-KX3OP6VP.mjs} +2 -2
- package/dist/{chunk-D456ZFPQ.mjs → chunk-MIIJ5O3O.mjs} +2 -2
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-MS746XNY.mjs} +285 -118
- package/dist/{chunk-WM5KJSQV.mjs → chunk-NFA3WHJU.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-OETLM4EH.mjs} +1 -1
- package/dist/{chunk-MYN4K5IS.mjs → chunk-OLXFIUIZ.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-X3EQCQLP.mjs → chunk-Q4NLL66U.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-QJFKL4NK.mjs} +4 -4
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-3IE3A74H.mjs → chunk-SSITKQU6.mjs} +3 -3
- package/dist/{chunk-B3MO37YF.mjs → chunk-TANKI7TG.mjs} +1 -1
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-TI4G3SIG.mjs} +63 -144
- package/dist/{chunk-OGYSPNL2.mjs → chunk-TT3UDDL3.mjs} +7 -7
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-VHIOGDVD.mjs} +1 -1
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-W3JH2IK6.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-WIT6B5OB.mjs} +2 -2
- package/dist/{chunk-MI6UZGUR.mjs → chunk-XGVSG6ZU.mjs} +3 -3
- package/dist/{chunk-6I6OO6B2.mjs → chunk-XWPNRHTG.mjs} +4 -4
- package/dist/{chunk-WTMCGZ74.mjs → chunk-YHZSSPG3.mjs} +2 -2
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-PCBB6FCO.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-4ALYIIC4.mjs} +14 -14
- package/dist/msgpack-leaf-6ZBZWFRD.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-2OK4TOLY.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-Y5OAKNRB.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-UQVQZFIJ.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-4OOUY2K7.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-5MYHYBET.mjs} +286 -118
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AVCMTWET.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-GS6LGXY3.mjs} +1 -1
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-LE4DTMDL.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-VIO5IPF7.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-VRI5XH2M.mjs} +7 -1
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-XGS2UYVI.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-XW6GZQWP.mjs} +4 -4
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-Z44S6WW5.mjs} +2 -2
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-4MIGXWZT.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-LZDHB7FX.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-J7SB263E.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PEKX7TBL.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-I2EC7H6R.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-RQXM7HBO.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-EZGCLSLM.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-JNGH2JYC.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-2OWLLBCV.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-TQ4SU76L.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-W7U2X6V3.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-WGP4FBXF.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-FYWJQSPL.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-GRG44RQA.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-YDJVH73G.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-5UFJV7SC.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-6YLVL2NC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-IP4L7KEW.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -10
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import { sha256 } from './chunk-H3MDVPBO.mjs';
|
|
2
2
|
import { oneSyncChangeAtATime } from './chunk-Q32DDFKX.mjs';
|
|
3
|
-
import { openChecksummedEnvelope, HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, hasNrslMagic, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, packEnvelopeWithChecksum, checksumOfChunks, QUERY_HISTORY_FOLDER, storedStateToWire, storedStateFromWire } from './chunk-
|
|
4
|
-
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-
|
|
5
|
-
import { deserializePartition } from './chunk-
|
|
6
|
-
import { mergeFieldIndexes, selectionKeeps, fieldIndexRemapOf, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, shareSegmentArrays, freezeSegmentShared, holdsADiscardedField, toError, segmentArraysOf, mergeSegmentContents, buildSegmentPayloads, createRequestId } from './chunk-
|
|
7
|
-
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, loadSchemaCodeFor, decodeStoredDocument, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, yieldToEventLoop, buildPartialWordTableFrom, configurationPayloadOf, withOpenIndex, emitSyncChange, newSyncEpoch, isCompositePartition, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, isSyncEpoch, prepareDocumentVectors, removedSurfaceSpellings, prepareConfiguredSynonyms, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, mergeKeywordSources, createSharedFrozenSegment, ARRAY_POSITION_GAP, createFrozenSegment, createPublicationGate, prepareRemovalText, prepareDocumentText, writePartitionDocument, updateDocumentVectors, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, readHeapStatistics, readProcessMemoryUse, holdSyncEpochChange, emptyPendingRegions, mergePostings2, scanPostingBlocks, SegmentRefusal, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, validQueryFilter, createPatternWorkMeter, compareScores, endOfValue, decodeEveryStoredDocument, createPartitionManager, createPartitionRouter, loadGzip, arrayPositionsAreStale, insertDocumentVectors, decodeMapWithoutFields, survivorDocumentsOf, partitionStateOf, noteDocumentChurn, removeDocumentVectors } from './chunk-
|
|
3
|
+
import { openChecksummedEnvelope, HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, hasNrslMagic, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, packEnvelopeWithChecksum, checksumOfChunks, QUERY_HISTORY_FOLDER, storedStateToWire, storedStateFromWire } from './chunk-Z44S6WW5.mjs';
|
|
4
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-4OOUY2K7.mjs';
|
|
5
|
+
import { deserializePartition } from './chunk-LE4DTMDL.mjs';
|
|
6
|
+
import { mergeFieldIndexes, selectionKeeps, fieldIndexRemapOf, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, shareSegmentArrays, freezeSegmentShared, holdsADiscardedField, toError, segmentArraysOf, mergeSegmentContents, buildSegmentPayloads, createRequestId } from './chunk-XW6GZQWP.mjs';
|
|
7
|
+
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, loadSchemaCodeFor, decodeStoredDocument, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, yieldToEventLoop, buildPartialWordTableFrom, configurationPayloadOf, withOpenIndex, emitSyncChange, newSyncEpoch, isCompositePartition, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, isSyncEpoch, prepareDocumentVectors, removedSurfaceSpellings, prepareConfiguredSynonyms, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, mergeKeywordSources, createSharedFrozenSegment, ARRAY_POSITION_GAP, createFrozenSegment, createPublicationGate, prepareRemovalText, prepareDocumentText, writePartitionDocument, updateDocumentVectors, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, readHeapStatistics, readProcessMemoryUse, holdSyncEpochChange, emptyPendingRegions, mergePostings2, scanPostingBlocks, SegmentRefusal, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, validQueryFilter, createPatternWorkMeter, compareScores, endOfValue, decodeEveryStoredDocument, createPartitionManager, createPartitionRouter, loadGzip, arrayPositionsAreStale, insertDocumentVectors, decodeMapWithoutFields, survivorDocumentsOf, partitionStateOf, noteDocumentChurn, removeDocumentVectors } from './chunk-5MYHYBET.mjs';
|
|
8
8
|
import './chunk-HQEVQEDJ.mjs';
|
|
9
9
|
import { turnsAreBudgeted, turnIsSpent, yieldTurn, isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, countWrite, collectAndReturnPages, requireBoundedUpperLists, osqRecordBytes, createFixedBuffer, magnitude, rescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, createVectorFileReader, decodeVectorIndexParts, signalOfSearch, requireSearchActive, osqQuantize, createOsqScratch, stageQueryLevels, scalarQueryProducts, readTrailer, osqEstimate, osqDistance, toDistance, decodeVectorFilePayload, DEFAULT_VECTOR_METRIC, defaultQuantizationFor, VECTOR_FILE_MAX_VECTORS, deadBytesFor, vectorsPerFile, vectorsToBytes, decodeVectorGraphPartPayload, assembleGraphParts, waitOutsideCountedWrite, fnv1a, osqBitsOf, countDead, nodeThreadId, detectRuntime, graphPartsOfEarlierPayload, decodeVectorGraphPayload } from './chunk-AKRYHAVG.mjs';
|
|
10
10
|
export { createHNSWIndex, osqBitsOf } from './chunk-AKRYHAVG.mjs';
|
|
11
11
|
import { DEFAULT_MAX_PARTITIONS, EXPORT_CHUNK_BYTES, MAX_PARTITIONS, STATISTICS_BROADCAST_INTERVAL_MS, ANALYSIS_CHANGE_MEMBERS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-LX62Y4B5.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-GS6LGXY3.mjs';
|
|
13
13
|
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-UYKIV2KT.mjs';
|
|
14
14
|
import { reactNativeHost } from './chunk-CW2ZFXSE.mjs';
|
|
15
|
-
import { setNestedValue, restoreVectorFields } from './chunk-
|
|
16
|
-
import { decode, decodeMulti, encodeDocument } from './chunk-
|
|
17
|
-
import { flattenSchema, validateDocument, validateDocumentStrict, schemaFieldsOf, VECTOR_PATTERN, parseVectorDimension, requireValidFieldTypes, extractVectorFieldsFromSchema, raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, messagePackUpperBoundBytes, partialFieldsOf, getNestedValue, MAX_DOCUMENT_NESTING_DEPTH, isQuantizationMode, isStorageMode } from './chunk-
|
|
15
|
+
import { setNestedValue, restoreVectorFields } from './chunk-XGS2UYVI.mjs';
|
|
16
|
+
import { decode, decodeMulti, encodeDocument } from './chunk-VRI5XH2M.mjs';
|
|
17
|
+
import { flattenSchema, validateDocument, validateDocumentStrict, schemaFieldsOf, VECTOR_PATTERN, parseVectorDimension, requireValidFieldTypes, extractVectorFieldsFromSchema, raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, messagePackUpperBoundBytes, partialFieldsOf, getNestedValue, MAX_DOCUMENT_NESTING_DEPTH, isQuantizationMode, isStorageMode } from './chunk-AVCMTWET.mjs';
|
|
18
18
|
import { MIN_M, MAX_M, MAX_LAYER_CAP, compareCodePoints } from './chunk-HRDMIQ63.mjs';
|
|
19
19
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-VUJCSQG5.mjs';
|
|
20
20
|
import { NarsilError, ErrorCodes } from './chunk-43R4GCM7.mjs';
|
|
@@ -1251,6 +1251,159 @@ async function checkpointInTurn(turns, turn) {
|
|
|
1251
1251
|
return run;
|
|
1252
1252
|
}
|
|
1253
1253
|
|
|
1254
|
+
// src/persistence/durability/mutation-batch.ts
|
|
1255
|
+
function shutDownRefusal() {
|
|
1256
|
+
return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
1257
|
+
}
|
|
1258
|
+
function toError2(err) {
|
|
1259
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
1260
|
+
}
|
|
1261
|
+
function requireAscendingSeqNo(record, seqNo, previousSeqNo) {
|
|
1262
|
+
if (seqNo > previousSeqNo) return;
|
|
1263
|
+
throw new NarsilError(
|
|
1264
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1265
|
+
`Log record ${seqNo} for partition ${record.partitionId} of index "${record.indexName}" does not follow record ${previousSeqNo}`,
|
|
1266
|
+
{ indexName: record.indexName, partitionId: record.partitionId, seqNo, previousSeqNo }
|
|
1267
|
+
);
|
|
1268
|
+
}
|
|
1269
|
+
function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
1270
|
+
for (let i = 0; i < count; i++) {
|
|
1271
|
+
const outcome = outcomes[i];
|
|
1272
|
+
if (outcome === void 0 || outcome.ok) {
|
|
1273
|
+
outcomes[i] = { ok: false, error };
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
function failPartition(partition, address, deps, err) {
|
|
1278
|
+
const error = toError2(err);
|
|
1279
|
+
if (partition.failed !== null) return partition.failed;
|
|
1280
|
+
partition.failed = error;
|
|
1281
|
+
if (!deps.failsPartitionAlone(address.indexName, address.partitionId, error)) deps.markFatal(error);
|
|
1282
|
+
return error;
|
|
1283
|
+
}
|
|
1284
|
+
async function commitEveryPartition(indexes, deps) {
|
|
1285
|
+
for (const [indexName, indexState] of indexes) {
|
|
1286
|
+
for (const [partitionId, partition] of indexState.partitions) {
|
|
1287
|
+
if (partition.failed !== null || deps.fatalError() !== null) continue;
|
|
1288
|
+
try {
|
|
1289
|
+
await partition.walWriter.commit();
|
|
1290
|
+
} catch (err) {
|
|
1291
|
+
failPartition(partition, { indexName, partitionId }, deps, err);
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
function partitionFailureFor(indexName, deps) {
|
|
1297
|
+
return (partitionId, partition, error) => failPartition(partition, { indexName, partitionId }, deps, error);
|
|
1298
|
+
}
|
|
1299
|
+
async function recordBatchesByPartition(records, partitionOf, deps) {
|
|
1300
|
+
const positionsByPartition = /* @__PURE__ */ new Map();
|
|
1301
|
+
for (let i = 0; i < records.length; i++) {
|
|
1302
|
+
const partition = partitionOf(records[i]);
|
|
1303
|
+
const positions = positionsByPartition.get(partition);
|
|
1304
|
+
if (positions === void 0) {
|
|
1305
|
+
positionsByPartition.set(partition, [i]);
|
|
1306
|
+
} else {
|
|
1307
|
+
positions.push(i);
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
const outcomes = new Array(records.length);
|
|
1311
|
+
await Promise.all(
|
|
1312
|
+
[...positionsByPartition].map(async ([partition, positions]) => {
|
|
1313
|
+
const partitionOutcomes = await recordPartitionBatch(
|
|
1314
|
+
partition,
|
|
1315
|
+
positions.map((position) => records[position]),
|
|
1316
|
+
deps
|
|
1317
|
+
);
|
|
1318
|
+
for (let i = 0; i < positions.length; i++) {
|
|
1319
|
+
outcomes[positions[i]] = partitionOutcomes[i];
|
|
1320
|
+
}
|
|
1321
|
+
})
|
|
1322
|
+
);
|
|
1323
|
+
return outcomes;
|
|
1324
|
+
}
|
|
1325
|
+
async function recordPartitionBatch(partition, records, deps) {
|
|
1326
|
+
const outcomes = new Array(records.length);
|
|
1327
|
+
let appendedCount = 0;
|
|
1328
|
+
let commitsHere = false;
|
|
1329
|
+
const appended = partition.appendChain.then(async () => {
|
|
1330
|
+
const fatal = deps.fatalError();
|
|
1331
|
+
if (fatal !== null) {
|
|
1332
|
+
throw fatal;
|
|
1333
|
+
}
|
|
1334
|
+
if (partition.failed !== null) {
|
|
1335
|
+
throw partition.failed;
|
|
1336
|
+
}
|
|
1337
|
+
const applied = [];
|
|
1338
|
+
for (let i = 0; i < records.length; i++) {
|
|
1339
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
1340
|
+
try {
|
|
1341
|
+
await records[i].apply();
|
|
1342
|
+
applied.push(i);
|
|
1343
|
+
} catch (err) {
|
|
1344
|
+
outcomes[i] = { ok: false, error: err };
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
if (applied.length === 0) {
|
|
1348
|
+
return;
|
|
1349
|
+
}
|
|
1350
|
+
let lastSeqNo = partition.appliedSeqNo;
|
|
1351
|
+
const entries = [];
|
|
1352
|
+
try {
|
|
1353
|
+
for (const i of applied) {
|
|
1354
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
1355
|
+
let numbered;
|
|
1356
|
+
try {
|
|
1357
|
+
numbered = deps.numberEntry(records[i]);
|
|
1358
|
+
} catch (err) {
|
|
1359
|
+
outcomes[i] = { ok: false, error: err };
|
|
1360
|
+
continue;
|
|
1361
|
+
}
|
|
1362
|
+
if (numbered === null) {
|
|
1363
|
+
lastSeqNo = partition.seqOwner.next();
|
|
1364
|
+
entries.push(deps.buildEntry(records[i], lastSeqNo));
|
|
1365
|
+
} else {
|
|
1366
|
+
requireAscendingSeqNo(records[i], numbered.seqNo, lastSeqNo);
|
|
1367
|
+
partition.seqOwner.advanceTo(numbered.seqNo, numbered.primaryTerm);
|
|
1368
|
+
lastSeqNo = numbered.seqNo;
|
|
1369
|
+
entries.push(numbered);
|
|
1370
|
+
}
|
|
1371
|
+
outcomes[i] = { ok: true, seqNo: lastSeqNo };
|
|
1372
|
+
if (!deps.defersCommit(records[i])) commitsHere = true;
|
|
1373
|
+
}
|
|
1374
|
+
if (entries.length > 0) await partition.walWriter.appendAll(entries);
|
|
1375
|
+
} catch (err) {
|
|
1376
|
+
throw failPartition(partition, records[0], deps, err);
|
|
1377
|
+
}
|
|
1378
|
+
partition.appliedSeqNo = lastSeqNo;
|
|
1379
|
+
appendedCount = entries.length;
|
|
1380
|
+
});
|
|
1381
|
+
partition.appendChain = appended.catch(() => void 0);
|
|
1382
|
+
try {
|
|
1383
|
+
await appended;
|
|
1384
|
+
if (deps.syncEachBatch && appendedCount > 0 && commitsHere) {
|
|
1385
|
+
try {
|
|
1386
|
+
await partition.walWriter.commit();
|
|
1387
|
+
} catch (err) {
|
|
1388
|
+
throw failPartition(partition, records[0], deps, err);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
} catch (err) {
|
|
1392
|
+
failUnlessAlreadyFailed(outcomes, records.length, toError2(err));
|
|
1393
|
+
}
|
|
1394
|
+
return outcomes;
|
|
1395
|
+
}
|
|
1396
|
+
async function commitAppended(partition, address, deps) {
|
|
1397
|
+
await partition.appendChain;
|
|
1398
|
+
const failed = partition.failed ?? deps.fatalError();
|
|
1399
|
+
if (failed !== null) throw failed;
|
|
1400
|
+
try {
|
|
1401
|
+
await partition.walWriter.commit();
|
|
1402
|
+
} catch (err) {
|
|
1403
|
+
throw failPartition(partition, address, deps, err);
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1254
1407
|
// src/persistence/durability/segment/store.ts
|
|
1255
1408
|
function directoryCheckpointStore(directory) {
|
|
1256
1409
|
const store = {
|
|
@@ -1258,11 +1411,11 @@ function directoryCheckpointStore(directory) {
|
|
|
1258
1411
|
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
1259
1412
|
remove: (key) => directory.remove(key),
|
|
1260
1413
|
list: (prefix) => directory.list(prefix),
|
|
1261
|
-
pathOf: (key) => directory.pathOf(key),
|
|
1262
1414
|
directory
|
|
1263
1415
|
};
|
|
1264
|
-
const byteLength = directory
|
|
1416
|
+
const { byteLength, pathOf } = directory;
|
|
1265
1417
|
if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
|
|
1418
|
+
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(directory, key);
|
|
1266
1419
|
return store;
|
|
1267
1420
|
}
|
|
1268
1421
|
function joined(chunks) {
|
|
@@ -1489,7 +1642,8 @@ async function writeTransition(store, indexName, transition) {
|
|
|
1489
1642
|
|
|
1490
1643
|
// src/persistence/durability/layout-transition/index.ts
|
|
1491
1644
|
async function transitionWalLayout(registry, indexName, manager, replay) {
|
|
1492
|
-
const state = registry.
|
|
1645
|
+
const state = registry.indexes.get(indexName);
|
|
1646
|
+
if (state === void 0) throw shutDownRefusal();
|
|
1493
1647
|
await switchDurableRouting(
|
|
1494
1648
|
manager,
|
|
1495
1649
|
(oldPartitionCount) => checkpointInTurn(state, {
|
|
@@ -1829,7 +1983,7 @@ function termBytesOf(core, indexName) {
|
|
|
1829
1983
|
}
|
|
1830
1984
|
return Math.max(0, bytes);
|
|
1831
1985
|
}
|
|
1832
|
-
function
|
|
1986
|
+
function toError3(value) {
|
|
1833
1987
|
return value instanceof Error ? value : new Error(String(value));
|
|
1834
1988
|
}
|
|
1835
1989
|
async function requireNoUndurableSwitch(core, indexName, entry) {
|
|
@@ -1928,7 +2082,7 @@ async function buildAndSwitch(scope) {
|
|
|
1928
2082
|
}
|
|
1929
2083
|
emit("completed", partitionsRebuilt);
|
|
1930
2084
|
} catch (error) {
|
|
1931
|
-
emit("failed", partitionsRebuilt,
|
|
2085
|
+
emit("failed", partitionsRebuilt, toError3(error));
|
|
1932
2086
|
throw error;
|
|
1933
2087
|
} finally {
|
|
1934
2088
|
if (core.executor.getManager(indexName) === manager) {
|
|
@@ -2049,7 +2203,7 @@ function resumeAnalysisChange(core, indexName) {
|
|
|
2049
2203
|
status: "failed",
|
|
2050
2204
|
partitionsRebuilt: 0,
|
|
2051
2205
|
partitionCount: entry.partitionCount,
|
|
2052
|
-
error:
|
|
2206
|
+
error: toError3(error)
|
|
2053
2207
|
});
|
|
2054
2208
|
return Promise.reject(error);
|
|
2055
2209
|
}
|
|
@@ -5523,7 +5677,14 @@ async function pagedListingOf(directory, ref) {
|
|
|
5523
5677
|
listed: ref.bytes
|
|
5524
5678
|
});
|
|
5525
5679
|
}
|
|
5526
|
-
return { key: ref.key, path: await directory
|
|
5680
|
+
return { key: ref.key, path: await filePathOf(directory, ref.key), bytes: ref.bytes, checksum: ref.checksum };
|
|
5681
|
+
}
|
|
5682
|
+
function filePathOf(directory, key) {
|
|
5683
|
+
const { pathOf } = directory;
|
|
5684
|
+
if (pathOf === void 0) {
|
|
5685
|
+
return loadFailed(`The storage that holds "${key}" keeps no files, so a paged reader cannot open it`, { key });
|
|
5686
|
+
}
|
|
5687
|
+
return pathOf.call(directory, key);
|
|
5527
5688
|
}
|
|
5528
5689
|
|
|
5529
5690
|
// src/persistence/paged/check-contents.ts
|
|
@@ -6558,7 +6719,10 @@ async function coldFieldOf(checkpoint, field) {
|
|
|
6558
6719
|
const fieldType = checkpoint.manifest.schema[field.fieldPath];
|
|
6559
6720
|
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
6560
6721
|
if (dimension === null) return null;
|
|
6561
|
-
const withPath = async (ref) => ({
|
|
6722
|
+
const withPath = async (ref) => ({
|
|
6723
|
+
ref,
|
|
6724
|
+
path: await filePathOf(directory, ref.key)
|
|
6725
|
+
});
|
|
6562
6726
|
return openColdVectorField({
|
|
6563
6727
|
fieldPath: field.fieldPath,
|
|
6564
6728
|
dimension,
|
|
@@ -6811,6 +6975,8 @@ var DIRECTORY_LOCK_ATTEMPTS = 3;
|
|
|
6811
6975
|
var DIRECTORY_LOCK_SETTLE_MS = 50;
|
|
6812
6976
|
var DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS = 1e3;
|
|
6813
6977
|
var PHONE_FILE_CHUNK_BYTES = 1048576;
|
|
6978
|
+
var INDEXEDDB_BLOCK_BYTES = 32768;
|
|
6979
|
+
var INDEXEDDB_CACHED_BLOCKS = 64;
|
|
6814
6980
|
var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
|
|
6815
6981
|
var DEFAULT_COMPACTION_THRESHOLD = 12;
|
|
6816
6982
|
var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
|
|
@@ -10003,15 +10169,27 @@ function windowFits(segments, start, windowLength, paged) {
|
|
|
10003
10169
|
if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
|
|
10004
10170
|
return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
|
|
10005
10171
|
}
|
|
10006
|
-
function
|
|
10172
|
+
function balanceOf(segments, start, windowLength) {
|
|
10173
|
+
let largest = 0;
|
|
10174
|
+
let bytes = 0;
|
|
10175
|
+
for (let at = start; at < start + windowLength; at++) {
|
|
10176
|
+
largest = Math.max(largest, segments[at].bytes);
|
|
10177
|
+
bytes += segments[at].bytes;
|
|
10178
|
+
}
|
|
10179
|
+
return bytes === 0 ? { skew: 1, bytes: 0 } : { skew: largest / bytes, bytes };
|
|
10180
|
+
}
|
|
10181
|
+
function balancedBefore(balance, other) {
|
|
10182
|
+
if (balance.skew !== other.skew) return balance.skew < other.skew;
|
|
10183
|
+
return balance.bytes < other.bytes;
|
|
10184
|
+
}
|
|
10185
|
+
function mostBalancedFittingWindow(segments, windowLength, paged) {
|
|
10007
10186
|
let best = null;
|
|
10008
|
-
let
|
|
10187
|
+
let bestBalance = null;
|
|
10009
10188
|
for (let start = 0; start + windowLength <= segments.length; start++) {
|
|
10010
10189
|
if (!windowFits(segments, start, windowLength, paged)) continue;
|
|
10011
|
-
|
|
10012
|
-
|
|
10013
|
-
|
|
10014
|
-
bestBytes = bytes;
|
|
10190
|
+
const balance = balanceOf(segments, start, windowLength);
|
|
10191
|
+
if (bestBalance === null || balancedBefore(balance, bestBalance)) {
|
|
10192
|
+
bestBalance = balance;
|
|
10015
10193
|
best = { start, windowLength };
|
|
10016
10194
|
}
|
|
10017
10195
|
}
|
|
@@ -10019,7 +10197,7 @@ function smallestFittingWindow(segments, windowLength, paged) {
|
|
|
10019
10197
|
}
|
|
10020
10198
|
function chooseWindow(segments, wantedLength, paged) {
|
|
10021
10199
|
for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
|
|
10022
|
-
const window =
|
|
10200
|
+
const window = mostBalancedFittingWindow(segments, windowLength, paged);
|
|
10023
10201
|
if (window !== null) return window;
|
|
10024
10202
|
}
|
|
10025
10203
|
return null;
|
|
@@ -10205,8 +10383,14 @@ async function documentsReplacedLater(store, segments) {
|
|
|
10205
10383
|
function exceedsRemovedShare(removed, held) {
|
|
10206
10384
|
return held > 0 && removed > held * REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE;
|
|
10207
10385
|
}
|
|
10208
|
-
function
|
|
10209
|
-
return ref.docCount === 0 ? 0 :
|
|
10386
|
+
function removedShareOf(ref, replacedCount) {
|
|
10387
|
+
return ref.docCount === 0 ? 0 : replacedCount / ref.docCount;
|
|
10388
|
+
}
|
|
10389
|
+
function rewriteOrder(segments, replaced, left, right) {
|
|
10390
|
+
const leftShare = removedShareOf(segments[left], replaced[left].length);
|
|
10391
|
+
const rightShare = removedShareOf(segments[right], replaced[right].length);
|
|
10392
|
+
if (leftShare !== rightShare) return leftShare > rightShare ? -1 : 1;
|
|
10393
|
+
return segments[left].bytes - segments[right].bytes;
|
|
10210
10394
|
}
|
|
10211
10395
|
function chooseRewrites(segments, replaced, budget) {
|
|
10212
10396
|
let held = 0;
|
|
@@ -10217,9 +10401,7 @@ function chooseRewrites(segments, replaced, budget) {
|
|
|
10217
10401
|
}
|
|
10218
10402
|
const candidates = [];
|
|
10219
10403
|
for (let at = 0; at < segments.length; at++) if (replaced[at].length > 0) candidates.push(at);
|
|
10220
|
-
candidates.sort(
|
|
10221
|
-
(left, right) => freedBytesOf(segments[right], replaced[right].length) - freedBytesOf(segments[left], replaced[left].length)
|
|
10222
|
-
);
|
|
10404
|
+
candidates.sort((left, right) => rewriteOrder(segments, replaced, left, right));
|
|
10223
10405
|
const chosen = [];
|
|
10224
10406
|
for (const at of candidates) {
|
|
10225
10407
|
if (!exceedsRemovedShare(removed, held)) break;
|
|
@@ -11346,7 +11528,7 @@ async function headOf(path, ref) {
|
|
|
11346
11528
|
return decode(head);
|
|
11347
11529
|
}
|
|
11348
11530
|
async function levelsOfPart(directory, ref) {
|
|
11349
|
-
const path = await directory
|
|
11531
|
+
const path = await filePathOf(directory, ref.key);
|
|
11350
11532
|
const head = await headOf(path, ref);
|
|
11351
11533
|
const graphs = head?.graphs;
|
|
11352
11534
|
if (!Array.isArray(graphs) || graphs.length !== 1) return [];
|
|
@@ -13566,6 +13748,10 @@ async function getFs() {
|
|
|
13566
13748
|
}
|
|
13567
13749
|
return cachedFs;
|
|
13568
13750
|
}
|
|
13751
|
+
async function renameReplacing(from, to) {
|
|
13752
|
+
await (await getFs()).rename(from, to);
|
|
13753
|
+
}
|
|
13754
|
+
var renameUnlessMissing = renameReplacing;
|
|
13569
13755
|
|
|
13570
13756
|
// src/persistence/durability/lock-holder.react-native.ts
|
|
13571
13757
|
function errnoOf2(err) {
|
|
@@ -13990,7 +14176,7 @@ function createDurableDirectory(root) {
|
|
|
13990
14176
|
}
|
|
13991
14177
|
await handle.close();
|
|
13992
14178
|
handle = null;
|
|
13993
|
-
await
|
|
14179
|
+
await renameReplacing(tempPath, filePath);
|
|
13994
14180
|
await fsyncDirectory(dir, fs);
|
|
13995
14181
|
} catch (err) {
|
|
13996
14182
|
if (handle !== null) {
|
|
@@ -14077,7 +14263,7 @@ function createDurableDirectory(root) {
|
|
|
14077
14263
|
const target = await resolve(to);
|
|
14078
14264
|
const fs = await getFs();
|
|
14079
14265
|
await ensureDir(target.path, target.pathMod, fs);
|
|
14080
|
-
await
|
|
14266
|
+
await renameReplacing(source.path, target.path);
|
|
14081
14267
|
const directories = new Set([target.path, source.path].map((path) => target.pathMod.dirname(path)));
|
|
14082
14268
|
for (const movedIn of directories) await fsyncDirectory(movedIn, fs);
|
|
14083
14269
|
},
|
|
@@ -14494,6 +14680,11 @@ async function createMarkerWriter(directory, indexName, partitionId) {
|
|
|
14494
14680
|
// src/persistence/durability/recovery.ts
|
|
14495
14681
|
async function removeLeftoverRewriteFiles(directory, names) {
|
|
14496
14682
|
for (const indexName of names) {
|
|
14683
|
+
const prefix = `${indexName}/`;
|
|
14684
|
+
for (const key of await directory.list(prefix)) {
|
|
14685
|
+
const relative = key.slice(prefix.length);
|
|
14686
|
+
if (!relative.includes("/") && /^(meta|manifest)\.tmp-/.test(relative)) await directory.remove(key);
|
|
14687
|
+
}
|
|
14497
14688
|
for (const place of ["rebuilt", "received"]) {
|
|
14498
14689
|
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
14499
14690
|
}
|
|
@@ -15613,10 +15804,10 @@ function createLogBytesLedger(base, sizes) {
|
|
|
15613
15804
|
const moved = untrack(from);
|
|
15614
15805
|
if (loggedFileOf(to) !== null) track(to, moved?.bytes ?? await currentLength(to));
|
|
15615
15806
|
},
|
|
15616
|
-
list: (prefix) => base.list(prefix)
|
|
15617
|
-
pathOf: (key) => base.pathOf(key)
|
|
15807
|
+
list: (prefix) => base.list(prefix)
|
|
15618
15808
|
};
|
|
15619
|
-
const { readRange, byteLength, identity, lock } = base;
|
|
15809
|
+
const { readRange, byteLength, identity, lock, pathOf } = base;
|
|
15810
|
+
if (pathOf !== void 0) directory.pathOf = (key) => pathOf.call(base, key);
|
|
15620
15811
|
if (readRange !== void 0) directory.readRange = (key, offset, length) => readRange.call(base, key, offset, length);
|
|
15621
15812
|
if (byteLength !== void 0) directory.byteLength = (key) => byteLength.call(base, key);
|
|
15622
15813
|
if (identity !== void 0) directory.identity = () => identity.call(base);
|
|
@@ -15804,21 +15995,27 @@ function storageFailed(code, message, key, error) {
|
|
|
15804
15995
|
return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
|
|
15805
15996
|
}
|
|
15806
15997
|
function directoryRestoreStorage(directory) {
|
|
15807
|
-
|
|
15998
|
+
const storage = {
|
|
15808
15999
|
directory,
|
|
15809
16000
|
read: (key) => directory.read(key),
|
|
15810
16001
|
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
15811
16002
|
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
15812
16003
|
remove: (key) => directory.remove(key),
|
|
15813
16004
|
list: (prefix) => directory.list(prefix),
|
|
15814
|
-
moveKey: (from, to) => directory.move(from, to)
|
|
15815
|
-
|
|
16005
|
+
moveKey: (from, to) => directory.move(from, to)
|
|
16006
|
+
};
|
|
16007
|
+
const { pathOf } = directory;
|
|
16008
|
+
if (pathOf === void 0) return storage;
|
|
16009
|
+
return { ...storage, ...fileTreeOperations(directory, (key) => pathOf.call(directory, key)) };
|
|
16010
|
+
}
|
|
16011
|
+
function fileTreeOperations(directory, pathOf) {
|
|
16012
|
+
return {
|
|
16013
|
+
pathOf,
|
|
15816
16014
|
async moveTree(from, to) {
|
|
15817
|
-
const source = await
|
|
15818
|
-
const target = await
|
|
15819
|
-
const fs = await getFs();
|
|
16015
|
+
const source = await pathOf(from);
|
|
16016
|
+
const target = await pathOf(to);
|
|
15820
16017
|
try {
|
|
15821
|
-
await
|
|
16018
|
+
await renameUnlessMissing(source, target);
|
|
15822
16019
|
} catch (error) {
|
|
15823
16020
|
if (missing(error)) return;
|
|
15824
16021
|
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
@@ -15829,7 +16026,7 @@ function directoryRestoreStorage(directory) {
|
|
|
15829
16026
|
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
15830
16027
|
const fs = await getFs();
|
|
15831
16028
|
try {
|
|
15832
|
-
await fs.rm(await
|
|
16029
|
+
await fs.rm(await pathOf(key), { recursive: true, force: true });
|
|
15833
16030
|
} catch (error) {
|
|
15834
16031
|
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
15835
16032
|
}
|
|
@@ -15842,7 +16039,7 @@ function recoverDirectoryRestores(directory) {
|
|
|
15842
16039
|
}
|
|
15843
16040
|
|
|
15844
16041
|
// src/persistence/durability/restore-staging.ts
|
|
15845
|
-
function
|
|
16042
|
+
function toError4(error) {
|
|
15846
16043
|
return error instanceof Error ? error : new Error(String(error));
|
|
15847
16044
|
}
|
|
15848
16045
|
function pathUnder(path, root) {
|
|
@@ -15851,7 +16048,7 @@ function pathUnder(path, root) {
|
|
|
15851
16048
|
return next === "" || next === "/" || next === "\\";
|
|
15852
16049
|
}
|
|
15853
16050
|
function stagedDirectory(directory, staged, unstaged) {
|
|
15854
|
-
const { readRange, byteLength } = directory;
|
|
16051
|
+
const { readRange, byteLength, pathOf } = directory;
|
|
15855
16052
|
return {
|
|
15856
16053
|
root: directory.root,
|
|
15857
16054
|
appendHandle: (key) => directory.appendHandle(staged(key)),
|
|
@@ -15862,7 +16059,7 @@ function stagedDirectory(directory, staged, unstaged) {
|
|
|
15862
16059
|
remove: (key) => directory.remove(staged(key)),
|
|
15863
16060
|
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
15864
16061
|
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
15865
|
-
pathOf: (key) =>
|
|
16062
|
+
...pathOf === void 0 ? {} : { pathOf: (key) => pathOf.call(directory, staged(key)) },
|
|
15866
16063
|
...readRange === void 0 ? {} : {
|
|
15867
16064
|
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
15868
16065
|
},
|
|
@@ -15911,7 +16108,7 @@ function createRestoreStaging(storage, indexName, options = {}) {
|
|
|
15911
16108
|
await storage.remove(journalKey(indexName));
|
|
15912
16109
|
} catch (removal) {
|
|
15913
16110
|
committed = true;
|
|
15914
|
-
options.onStorageLost?.(
|
|
16111
|
+
options.onStorageLost?.(toError4(removal));
|
|
15915
16112
|
}
|
|
15916
16113
|
throw error;
|
|
15917
16114
|
}
|
|
@@ -15921,7 +16118,7 @@ function createRestoreStaging(storage, indexName, options = {}) {
|
|
|
15921
16118
|
try {
|
|
15922
16119
|
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
15923
16120
|
} catch (error) {
|
|
15924
|
-
options.onStorageLost?.(
|
|
16121
|
+
options.onStorageLost?.(toError4(error));
|
|
15925
16122
|
}
|
|
15926
16123
|
},
|
|
15927
16124
|
async removeRetired() {
|
|
@@ -15953,7 +16150,7 @@ function adapterRestoreStorage(adapter) {
|
|
|
15953
16150
|
}
|
|
15954
16151
|
|
|
15955
16152
|
// src/persistence/durability/archive-checkpoint.ts
|
|
15956
|
-
function
|
|
16153
|
+
function toError5(error) {
|
|
15957
16154
|
return error instanceof Error ? error : new Error(String(error));
|
|
15958
16155
|
}
|
|
15959
16156
|
function stageDirectoryRestore(deps, indexName) {
|
|
@@ -15962,7 +16159,7 @@ function stageDirectoryRestore(deps, indexName) {
|
|
|
15962
16159
|
try {
|
|
15963
16160
|
await run();
|
|
15964
16161
|
} catch (error) {
|
|
15965
|
-
deps.markFatal(
|
|
16162
|
+
deps.markFatal(toError5(error));
|
|
15966
16163
|
throw error;
|
|
15967
16164
|
}
|
|
15968
16165
|
},
|
|
@@ -16619,6 +16816,18 @@ function createIndexStorageAccess(deps) {
|
|
|
16619
16816
|
}
|
|
16620
16817
|
|
|
16621
16818
|
// src/persistence/durability/metadata-write-queue.ts
|
|
16819
|
+
async function writeIndexMetadataFile(directory, indexName, bytes) {
|
|
16820
|
+
try {
|
|
16821
|
+
await directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
16822
|
+
} catch (error) {
|
|
16823
|
+
if (error instanceof NarsilError) throw error;
|
|
16824
|
+
throw new NarsilError(
|
|
16825
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
16826
|
+
`The engine cannot write the metadata of index "${indexName}" to the durability directory "${directory.root}"`,
|
|
16827
|
+
{ indexName, directory: directory.root, cause: error instanceof Error ? error.message : String(error) }
|
|
16828
|
+
);
|
|
16829
|
+
}
|
|
16830
|
+
}
|
|
16622
16831
|
function createMetadataWriteQueue() {
|
|
16623
16832
|
const writes = /* @__PURE__ */ new Map();
|
|
16624
16833
|
return {
|
|
@@ -16634,201 +16843,59 @@ function createMetadataWriteQueue() {
|
|
|
16634
16843
|
pending: (indexName) => writes.get(indexName),
|
|
16635
16844
|
forget(indexName) {
|
|
16636
16845
|
writes.delete(indexName);
|
|
16846
|
+
},
|
|
16847
|
+
async settleAll() {
|
|
16848
|
+
let waiting = [...writes.values()];
|
|
16849
|
+
while (waiting.length > 0) {
|
|
16850
|
+
await Promise.all(waiting);
|
|
16851
|
+
const settled = waiting;
|
|
16852
|
+
waiting = [...writes.values()].filter((write) => !settled.includes(write));
|
|
16853
|
+
}
|
|
16637
16854
|
}
|
|
16638
16855
|
};
|
|
16639
16856
|
}
|
|
16640
16857
|
|
|
16641
|
-
// src/persistence/durability/mutation-
|
|
16642
|
-
function
|
|
16643
|
-
return
|
|
16644
|
-
|
|
16645
|
-
|
|
16646
|
-
|
|
16647
|
-
|
|
16648
|
-
|
|
16649
|
-
|
|
16650
|
-
|
|
16651
|
-
|
|
16858
|
+
// src/persistence/durability/mutation-entry.ts
|
|
16859
|
+
function buildMutationEntry(record, seqNo) {
|
|
16860
|
+
return buildEntry({
|
|
16861
|
+
seqNo,
|
|
16862
|
+
primaryTerm: SINGLE_NODE_PRIMARY_TERM,
|
|
16863
|
+
operation: record.operation,
|
|
16864
|
+
partitionId: record.partitionId,
|
|
16865
|
+
indexName: record.indexName,
|
|
16866
|
+
documentId: record.documentId,
|
|
16867
|
+
document: record.document,
|
|
16868
|
+
previousDocument: record.previousDocument
|
|
16869
|
+
});
|
|
16652
16870
|
}
|
|
16653
|
-
|
|
16654
|
-
|
|
16655
|
-
|
|
16656
|
-
|
|
16657
|
-
|
|
16871
|
+
|
|
16872
|
+
// src/persistence/durability/partition-failure.ts
|
|
16873
|
+
function createPartitionFailureRouting() {
|
|
16874
|
+
let onFailed = null;
|
|
16875
|
+
return {
|
|
16876
|
+
isolate(handler) {
|
|
16877
|
+
onFailed = handler;
|
|
16878
|
+
},
|
|
16879
|
+
failsAlone(indexName, partitionId, error) {
|
|
16880
|
+
if (onFailed === null) return false;
|
|
16881
|
+
onFailed(indexName, partitionId, error);
|
|
16882
|
+
return true;
|
|
16658
16883
|
}
|
|
16659
|
-
}
|
|
16660
|
-
}
|
|
16661
|
-
function failPartition(partition, address, deps, err) {
|
|
16662
|
-
const error = toError5(err);
|
|
16663
|
-
if (partition.failed !== null) return partition.failed;
|
|
16664
|
-
partition.failed = error;
|
|
16665
|
-
if (!deps.failsPartitionAlone(address.indexName, address.partitionId, error)) deps.markFatal(error);
|
|
16666
|
-
return error;
|
|
16884
|
+
};
|
|
16667
16885
|
}
|
|
16668
|
-
|
|
16669
|
-
|
|
16670
|
-
|
|
16671
|
-
|
|
16672
|
-
|
|
16673
|
-
|
|
16674
|
-
|
|
16675
|
-
|
|
16676
|
-
|
|
16677
|
-
|
|
16678
|
-
|
|
16679
|
-
}
|
|
16680
|
-
|
|
16681
|
-
return (partitionId, partition, error) => failPartition(partition, { indexName, partitionId }, deps, error);
|
|
16682
|
-
}
|
|
16683
|
-
async function recordBatchesByPartition(records, partitionOf, deps) {
|
|
16684
|
-
const positionsByPartition = /* @__PURE__ */ new Map();
|
|
16685
|
-
for (let i = 0; i < records.length; i++) {
|
|
16686
|
-
const partition = partitionOf(records[i]);
|
|
16687
|
-
const positions = positionsByPartition.get(partition);
|
|
16688
|
-
if (positions === void 0) {
|
|
16689
|
-
positionsByPartition.set(partition, [i]);
|
|
16690
|
-
} else {
|
|
16691
|
-
positions.push(i);
|
|
16692
|
-
}
|
|
16693
|
-
}
|
|
16694
|
-
const outcomes = new Array(records.length);
|
|
16695
|
-
await Promise.all(
|
|
16696
|
-
[...positionsByPartition].map(async ([partition, positions]) => {
|
|
16697
|
-
const partitionOutcomes = await recordPartitionBatch(
|
|
16698
|
-
partition,
|
|
16699
|
-
positions.map((position) => records[position]),
|
|
16700
|
-
deps
|
|
16701
|
-
);
|
|
16702
|
-
for (let i = 0; i < positions.length; i++) {
|
|
16703
|
-
outcomes[positions[i]] = partitionOutcomes[i];
|
|
16704
|
-
}
|
|
16705
|
-
})
|
|
16706
|
-
);
|
|
16707
|
-
return outcomes;
|
|
16708
|
-
}
|
|
16709
|
-
async function recordPartitionBatch(partition, records, deps) {
|
|
16710
|
-
const outcomes = new Array(records.length);
|
|
16711
|
-
let appendedCount = 0;
|
|
16712
|
-
let commitsHere = false;
|
|
16713
|
-
const appended = partition.appendChain.then(async () => {
|
|
16714
|
-
const fatal = deps.fatalError();
|
|
16715
|
-
if (fatal !== null) {
|
|
16716
|
-
throw fatal;
|
|
16717
|
-
}
|
|
16718
|
-
if (partition.failed !== null) {
|
|
16719
|
-
throw partition.failed;
|
|
16720
|
-
}
|
|
16721
|
-
const applied = [];
|
|
16722
|
-
for (let i = 0; i < records.length; i++) {
|
|
16723
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
16724
|
-
try {
|
|
16725
|
-
await records[i].apply();
|
|
16726
|
-
applied.push(i);
|
|
16727
|
-
} catch (err) {
|
|
16728
|
-
outcomes[i] = { ok: false, error: err };
|
|
16729
|
-
}
|
|
16730
|
-
}
|
|
16731
|
-
if (applied.length === 0) {
|
|
16732
|
-
return;
|
|
16733
|
-
}
|
|
16734
|
-
let lastSeqNo = partition.appliedSeqNo;
|
|
16735
|
-
const entries = [];
|
|
16736
|
-
try {
|
|
16737
|
-
for (const i of applied) {
|
|
16738
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
16739
|
-
let numbered;
|
|
16740
|
-
try {
|
|
16741
|
-
numbered = deps.numberEntry(records[i]);
|
|
16742
|
-
} catch (err) {
|
|
16743
|
-
outcomes[i] = { ok: false, error: err };
|
|
16744
|
-
continue;
|
|
16745
|
-
}
|
|
16746
|
-
if (numbered === null) {
|
|
16747
|
-
lastSeqNo = partition.seqOwner.next();
|
|
16748
|
-
entries.push(deps.buildEntry(records[i], lastSeqNo));
|
|
16749
|
-
} else {
|
|
16750
|
-
requireAscendingSeqNo(records[i], numbered.seqNo, lastSeqNo);
|
|
16751
|
-
partition.seqOwner.advanceTo(numbered.seqNo, numbered.primaryTerm);
|
|
16752
|
-
lastSeqNo = numbered.seqNo;
|
|
16753
|
-
entries.push(numbered);
|
|
16754
|
-
}
|
|
16755
|
-
outcomes[i] = { ok: true, seqNo: lastSeqNo };
|
|
16756
|
-
if (!deps.defersCommit(records[i])) commitsHere = true;
|
|
16757
|
-
}
|
|
16758
|
-
if (entries.length > 0) await partition.walWriter.appendAll(entries);
|
|
16759
|
-
} catch (err) {
|
|
16760
|
-
throw failPartition(partition, records[0], deps, err);
|
|
16761
|
-
}
|
|
16762
|
-
partition.appliedSeqNo = lastSeqNo;
|
|
16763
|
-
appendedCount = entries.length;
|
|
16764
|
-
});
|
|
16765
|
-
partition.appendChain = appended.catch(() => void 0);
|
|
16766
|
-
try {
|
|
16767
|
-
await appended;
|
|
16768
|
-
if (deps.syncEachBatch && appendedCount > 0 && commitsHere) {
|
|
16769
|
-
try {
|
|
16770
|
-
await partition.walWriter.commit();
|
|
16771
|
-
} catch (err) {
|
|
16772
|
-
throw failPartition(partition, records[0], deps, err);
|
|
16773
|
-
}
|
|
16774
|
-
}
|
|
16775
|
-
} catch (err) {
|
|
16776
|
-
failUnlessAlreadyFailed(outcomes, records.length, toError5(err));
|
|
16777
|
-
}
|
|
16778
|
-
return outcomes;
|
|
16779
|
-
}
|
|
16780
|
-
async function commitAppended(partition, address, deps) {
|
|
16781
|
-
await partition.appendChain;
|
|
16782
|
-
const failed = partition.failed ?? deps.fatalError();
|
|
16783
|
-
if (failed !== null) throw failed;
|
|
16784
|
-
try {
|
|
16785
|
-
await partition.walWriter.commit();
|
|
16786
|
-
} catch (err) {
|
|
16787
|
-
throw failPartition(partition, address, deps, err);
|
|
16788
|
-
}
|
|
16789
|
-
}
|
|
16790
|
-
|
|
16791
|
-
// src/persistence/durability/mutation-entry.ts
|
|
16792
|
-
function buildMutationEntry(record, seqNo) {
|
|
16793
|
-
return buildEntry({
|
|
16794
|
-
seqNo,
|
|
16795
|
-
primaryTerm: SINGLE_NODE_PRIMARY_TERM,
|
|
16796
|
-
operation: record.operation,
|
|
16797
|
-
partitionId: record.partitionId,
|
|
16798
|
-
indexName: record.indexName,
|
|
16799
|
-
documentId: record.documentId,
|
|
16800
|
-
document: record.document,
|
|
16801
|
-
previousDocument: record.previousDocument
|
|
16802
|
-
});
|
|
16803
|
-
}
|
|
16804
|
-
|
|
16805
|
-
// src/persistence/durability/partition-failure.ts
|
|
16806
|
-
function createPartitionFailureRouting() {
|
|
16807
|
-
let onFailed = null;
|
|
16808
|
-
return {
|
|
16809
|
-
isolate(handler) {
|
|
16810
|
-
onFailed = handler;
|
|
16811
|
-
},
|
|
16812
|
-
failsAlone(indexName, partitionId, error) {
|
|
16813
|
-
if (onFailed === null) return false;
|
|
16814
|
-
onFailed(indexName, partitionId, error);
|
|
16815
|
-
return true;
|
|
16816
|
-
}
|
|
16817
|
-
};
|
|
16818
|
-
}
|
|
16819
|
-
|
|
16820
|
-
// src/persistence/durability/group-commit.ts
|
|
16821
|
-
function createGroupCommitCoordinator(sync) {
|
|
16822
|
-
let inFlight = null;
|
|
16823
|
-
let next = null;
|
|
16824
|
-
function makeBatch() {
|
|
16825
|
-
let resolve = () => void 0;
|
|
16826
|
-
let reject = () => void 0;
|
|
16827
|
-
const promise = new Promise((res, rej) => {
|
|
16828
|
-
resolve = res;
|
|
16829
|
-
reject = rej;
|
|
16830
|
-
});
|
|
16831
|
-
return { promise, resolve, reject, waiters: 0 };
|
|
16886
|
+
|
|
16887
|
+
// src/persistence/durability/group-commit.ts
|
|
16888
|
+
function createGroupCommitCoordinator(sync) {
|
|
16889
|
+
let inFlight = null;
|
|
16890
|
+
let next = null;
|
|
16891
|
+
function makeBatch() {
|
|
16892
|
+
let resolve = () => void 0;
|
|
16893
|
+
let reject = () => void 0;
|
|
16894
|
+
const promise = new Promise((res, rej) => {
|
|
16895
|
+
resolve = res;
|
|
16896
|
+
reject = rej;
|
|
16897
|
+
});
|
|
16898
|
+
return { promise, resolve, reject, waiters: 0 };
|
|
16832
16899
|
}
|
|
16833
16900
|
function drain() {
|
|
16834
16901
|
if (inFlight !== null || next === null) {
|
|
@@ -18153,7 +18220,7 @@ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkp
|
|
|
18153
18220
|
// src/persistence/durability/manager.ts
|
|
18154
18221
|
function createDurabilityManager(config, hooks, directoryOverride) {
|
|
18155
18222
|
const sizes = createStoredSizesRecord(true);
|
|
18156
|
-
const logBytes = createLogBytesLedger(createDurableDirectory(config.directory ?? ""), sizes);
|
|
18223
|
+
const logBytes = createLogBytesLedger(directoryOverride ?? createDurableDirectory(config.directory ?? ""), sizes);
|
|
18157
18224
|
const { directory } = logBytes;
|
|
18158
18225
|
const canOffloadCheckpoint = directoryOverride === void 0;
|
|
18159
18226
|
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
@@ -18277,10 +18344,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
18277
18344
|
defersCommit: (record) => commitDeferral?.(record) === true
|
|
18278
18345
|
};
|
|
18279
18346
|
async function recordMutations(records) {
|
|
18280
|
-
|
|
18281
|
-
|
|
18282
|
-
return records.map(() => ({ ok: false, error }));
|
|
18283
|
-
}
|
|
18347
|
+
const refusal = fatalError ?? (shuttingDown ? shutDownRefusal() : null);
|
|
18348
|
+
if (refusal !== null) return records.map(() => ({ ok: false, error: refusal }));
|
|
18284
18349
|
const outcomes = await recordBatchesByPartition(
|
|
18285
18350
|
records,
|
|
18286
18351
|
(record) => getOrCreatePartition(record.indexName, record.partitionId, 0),
|
|
@@ -18395,7 +18460,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
18395
18460
|
return;
|
|
18396
18461
|
}
|
|
18397
18462
|
const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
|
|
18398
|
-
await directory
|
|
18463
|
+
await writeIndexMetadataFile(directory, indexName, bytes);
|
|
18399
18464
|
getOrCreateIndexState(indexName);
|
|
18400
18465
|
});
|
|
18401
18466
|
},
|
|
@@ -18432,14 +18497,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
18432
18497
|
async shutdown() {
|
|
18433
18498
|
shuttingDown = true;
|
|
18434
18499
|
stopTimers();
|
|
18500
|
+
for (const indexState of indexes.values()) indexState.unloading = true;
|
|
18435
18501
|
for (const indexState of indexes.values()) {
|
|
18436
18502
|
if (indexState.checkpointInFlight !== null) {
|
|
18437
18503
|
await indexState.checkpointInFlight.catch(() => void 0);
|
|
18438
18504
|
}
|
|
18439
18505
|
for (const partition of indexState.partitions.values()) {
|
|
18506
|
+
await partition.appendChain;
|
|
18440
18507
|
await closeWriterReportingFailure(closePartitionWriter(partition));
|
|
18441
18508
|
}
|
|
18442
18509
|
}
|
|
18510
|
+
await metadataWrites.settleAll();
|
|
18443
18511
|
indexes.clear();
|
|
18444
18512
|
terminateCheckpointWorker();
|
|
18445
18513
|
await directoryWatch.release();
|
|
@@ -18447,72 +18515,1130 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
18447
18515
|
};
|
|
18448
18516
|
}
|
|
18449
18517
|
|
|
18450
|
-
// src/persistence/
|
|
18451
|
-
|
|
18452
|
-
|
|
18453
|
-
|
|
18454
|
-
|
|
18455
|
-
|
|
18456
|
-
|
|
18457
|
-
|
|
18458
|
-
};
|
|
18459
|
-
}
|
|
18460
|
-
function metadataKey(indexName) {
|
|
18461
|
-
return `${indexName}/meta`;
|
|
18518
|
+
// src/persistence/constants.ts
|
|
18519
|
+
var INDEXEDDB_BLOCKED_TIMEOUT_MS = 5e3;
|
|
18520
|
+
|
|
18521
|
+
// src/persistence/durability/indexeddb-directory/connection.ts
|
|
18522
|
+
function requestResult(request) {
|
|
18523
|
+
return new Promise((resolve, reject) => {
|
|
18524
|
+
request.onsuccess = () => resolve(request.result);
|
|
18525
|
+
request.onerror = () => reject(request.error);
|
|
18526
|
+
});
|
|
18462
18527
|
}
|
|
18463
|
-
function
|
|
18464
|
-
|
|
18465
|
-
|
|
18466
|
-
|
|
18467
|
-
|
|
18468
|
-
|
|
18469
|
-
|
|
18470
|
-
|
|
18528
|
+
function causeOf(error) {
|
|
18529
|
+
if (error instanceof Error || typeof error === "object" && error !== null && "message" in error) {
|
|
18530
|
+
return String(error.message);
|
|
18531
|
+
}
|
|
18532
|
+
return String(error);
|
|
18533
|
+
}
|
|
18534
|
+
function rootOf(location) {
|
|
18535
|
+
return `indexeddb:${location.databaseName}/${location.storeName}`;
|
|
18536
|
+
}
|
|
18537
|
+
function openLogConnection(location) {
|
|
18538
|
+
const root = rootOf(location);
|
|
18539
|
+
let opening = null;
|
|
18540
|
+
let lostReason = null;
|
|
18541
|
+
let closedByEngine = false;
|
|
18542
|
+
const token = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
18543
|
+
function storeFailed(failure, cause) {
|
|
18544
|
+
return new NarsilError(failure.code, `IndexedDB could not ${failure.action} in ${root}: ${cause}`, {
|
|
18545
|
+
directory: root,
|
|
18546
|
+
cause
|
|
18547
|
+
});
|
|
18471
18548
|
}
|
|
18472
|
-
|
|
18473
|
-
|
|
18474
|
-
|
|
18475
|
-
|
|
18476
|
-
|
|
18477
|
-
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
18478
|
-
const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
|
|
18479
|
-
const targets = [];
|
|
18480
|
-
const contents = [];
|
|
18481
|
-
const positions = /* @__PURE__ */ new Map();
|
|
18482
|
-
for (let partitionId = 0; partitionId < input.manager.partitionCount; partitionId += 1) {
|
|
18483
|
-
const lastSeqNo = Math.min(
|
|
18484
|
-
input.appliedSeqNoByPartition.get(partitionId) ?? 0,
|
|
18485
|
-
cutoffs.get(partitionId) ?? Infinity
|
|
18549
|
+
function replaced() {
|
|
18550
|
+
return new NarsilError(
|
|
18551
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
18552
|
+
`The engine treats ${root} as replaced, because ${lostReason ?? "the engine closed it"}, so it acknowledges no later write`,
|
|
18553
|
+
{ directory: root }
|
|
18486
18554
|
);
|
|
18487
|
-
positions.set(partitionId, lastSeqNo);
|
|
18488
|
-
const unchanged = !input.rewritesEveryPartition && prior !== null && !partitionCountChanged && listedIds.has(partitionId) && checkpointLastSeqNo(prior.checkpoint, partitionId) === lastSeqNo;
|
|
18489
|
-
if (unchanged) continue;
|
|
18490
|
-
targets.push({ partitionId, lastSeqNo, primaryTerm: SINGLE_NODE_PRIMARY_TERM });
|
|
18491
|
-
contents.push(partitionContentsOf(input.manager, partitionId));
|
|
18492
18555
|
}
|
|
18493
|
-
|
|
18494
|
-
|
|
18495
|
-
|
|
18496
|
-
|
|
18497
|
-
|
|
18498
|
-
|
|
18499
|
-
|
|
18556
|
+
function watch(database2) {
|
|
18557
|
+
database2.onclose = () => {
|
|
18558
|
+
lostReason ??= "the browser closed the connection";
|
|
18559
|
+
};
|
|
18560
|
+
database2.onversionchange = () => {
|
|
18561
|
+
lostReason ??= "another connection asked for a version change of the database";
|
|
18562
|
+
database2.close();
|
|
18563
|
+
};
|
|
18564
|
+
}
|
|
18565
|
+
function open() {
|
|
18566
|
+
const factory = globalThis.indexedDB;
|
|
18567
|
+
if (factory === void 0) {
|
|
18568
|
+
return Promise.reject(
|
|
18569
|
+
new NarsilError(ErrorCodes.CONFIG_INVALID, "IndexedDB is not available, so the write-ahead log cannot open")
|
|
18500
18570
|
);
|
|
18501
18571
|
}
|
|
18502
|
-
return {
|
|
18503
|
-
|
|
18504
|
-
|
|
18505
|
-
|
|
18506
|
-
|
|
18507
|
-
|
|
18508
|
-
|
|
18509
|
-
|
|
18510
|
-
|
|
18511
|
-
|
|
18512
|
-
|
|
18572
|
+
return new Promise((resolve, reject) => {
|
|
18573
|
+
const request = factory.open(location.databaseName);
|
|
18574
|
+
let blockedTimer = null;
|
|
18575
|
+
const settle = () => {
|
|
18576
|
+
if (blockedTimer !== null) clearTimeout(blockedTimer);
|
|
18577
|
+
};
|
|
18578
|
+
request.onblocked = () => {
|
|
18579
|
+
blockedTimer = setTimeout(
|
|
18580
|
+
() => reject(storeFailed({ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open" }, "the open is blocked")),
|
|
18581
|
+
INDEXEDDB_BLOCKED_TIMEOUT_MS
|
|
18582
|
+
);
|
|
18583
|
+
};
|
|
18584
|
+
request.onupgradeneeded = () => {
|
|
18585
|
+
const database2 = request.result;
|
|
18586
|
+
if (!database2.objectStoreNames.contains(location.storeName)) database2.createObjectStore(location.storeName);
|
|
18587
|
+
};
|
|
18588
|
+
request.onsuccess = () => {
|
|
18589
|
+
settle();
|
|
18590
|
+
const database2 = request.result;
|
|
18591
|
+
if (!database2.objectStoreNames.contains(location.storeName)) {
|
|
18592
|
+
database2.close();
|
|
18593
|
+
reject(
|
|
18594
|
+
storeFailed(
|
|
18595
|
+
{ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open" },
|
|
18596
|
+
`the database holds no object store named "${location.storeName}"`
|
|
18597
|
+
)
|
|
18598
|
+
);
|
|
18599
|
+
return;
|
|
18600
|
+
}
|
|
18601
|
+
watch(database2);
|
|
18602
|
+
resolve(database2);
|
|
18603
|
+
};
|
|
18604
|
+
request.onerror = () => {
|
|
18605
|
+
settle();
|
|
18606
|
+
reject(storeFailed({ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open" }, causeOf(request.error)));
|
|
18607
|
+
};
|
|
18608
|
+
});
|
|
18513
18609
|
}
|
|
18610
|
+
async function database() {
|
|
18611
|
+
if (closedByEngine || lostReason !== null) throw replaced();
|
|
18612
|
+
if (opening === null) {
|
|
18613
|
+
opening = open();
|
|
18614
|
+
opening.catch(() => {
|
|
18615
|
+
opening = null;
|
|
18616
|
+
});
|
|
18617
|
+
}
|
|
18618
|
+
const opened = await opening;
|
|
18619
|
+
if (lostReason !== null || closedByEngine) throw replaced();
|
|
18620
|
+
return opened;
|
|
18621
|
+
}
|
|
18622
|
+
async function transact(mode, durability, failure, body) {
|
|
18623
|
+
const opened = await database();
|
|
18624
|
+
let transaction;
|
|
18625
|
+
try {
|
|
18626
|
+
transaction = durability === null ? opened.transaction(location.storeName, mode) : opened.transaction(location.storeName, mode, { durability });
|
|
18627
|
+
} catch (error) {
|
|
18628
|
+
if (lostReason === null && error.name === "InvalidStateError") {
|
|
18629
|
+
lostReason = "the browser closed the connection";
|
|
18630
|
+
}
|
|
18631
|
+
throw lostReason === null ? storeFailed(failure, causeOf(error)) : replaced();
|
|
18632
|
+
}
|
|
18633
|
+
const committed = new Promise((resolve, reject) => {
|
|
18634
|
+
transaction.oncomplete = () => resolve();
|
|
18635
|
+
transaction.onabort = () => reject(transaction.error ?? new Error("the transaction aborted"));
|
|
18636
|
+
});
|
|
18637
|
+
let result;
|
|
18638
|
+
try {
|
|
18639
|
+
result = await body(transaction.objectStore(location.storeName));
|
|
18640
|
+
} catch (error) {
|
|
18641
|
+
try {
|
|
18642
|
+
transaction.abort();
|
|
18643
|
+
} catch {
|
|
18644
|
+
}
|
|
18645
|
+
await committed.catch(() => void 0);
|
|
18646
|
+
throw transactionFailed(transaction, failure, error);
|
|
18647
|
+
}
|
|
18648
|
+
try {
|
|
18649
|
+
await committed;
|
|
18650
|
+
} catch (error) {
|
|
18651
|
+
throw transactionFailed(transaction, failure, error);
|
|
18652
|
+
}
|
|
18653
|
+
if (lostReason !== null || closedByEngine) throw replaced();
|
|
18654
|
+
return result;
|
|
18655
|
+
}
|
|
18656
|
+
function transactionFailed(transaction, failure, error) {
|
|
18657
|
+
const aborted = transaction.error?.name === "AbortError" || error instanceof Error && error.name === "AbortError";
|
|
18658
|
+
if (lostReason === null && aborted) {
|
|
18659
|
+
lostReason = "the browser closed the connection";
|
|
18660
|
+
}
|
|
18661
|
+
if (lostReason !== null || closedByEngine) return replaced();
|
|
18662
|
+
return error instanceof NarsilError ? error : storeFailed(failure, causeOf(error));
|
|
18663
|
+
}
|
|
18664
|
+
return {
|
|
18665
|
+
root,
|
|
18666
|
+
read: (failure, body) => transact("readonly", null, failure, body),
|
|
18667
|
+
write: (durability, failure, body) => transact("readwrite", durability, failure, body),
|
|
18668
|
+
async identity() {
|
|
18669
|
+
await database().catch((error) => {
|
|
18670
|
+
if (lostReason === null && !closedByEngine) throw error;
|
|
18671
|
+
});
|
|
18672
|
+
return lostReason === null && !closedByEngine ? token : null;
|
|
18673
|
+
},
|
|
18674
|
+
close() {
|
|
18675
|
+
closedByEngine = true;
|
|
18676
|
+
const pending = opening;
|
|
18677
|
+
opening = null;
|
|
18678
|
+
void pending?.then(
|
|
18679
|
+
(opened) => opened.close(),
|
|
18680
|
+
() => void 0
|
|
18681
|
+
);
|
|
18682
|
+
}
|
|
18683
|
+
};
|
|
18514
18684
|
}
|
|
18515
|
-
|
|
18685
|
+
|
|
18686
|
+
// src/persistence/durability/indexeddb-directory/records.ts
|
|
18687
|
+
function appendKey(key, n) {
|
|
18688
|
+
return [key, n];
|
|
18689
|
+
}
|
|
18690
|
+
function appendRecordsOf(key) {
|
|
18691
|
+
return IDBKeyRange.bound([key], [key, []], true, true);
|
|
18692
|
+
}
|
|
18693
|
+
function compactBytes(bytes) {
|
|
18694
|
+
return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength ? bytes : bytes.slice();
|
|
18695
|
+
}
|
|
18696
|
+
function joinBytes(chunks) {
|
|
18697
|
+
if (chunks.length === 1) return compactBytes(chunks[0]);
|
|
18698
|
+
let total = 0;
|
|
18699
|
+
for (const chunk of chunks) total += chunk.byteLength;
|
|
18700
|
+
const joined4 = new Uint8Array(total);
|
|
18701
|
+
let offset = 0;
|
|
18702
|
+
for (const chunk of chunks) {
|
|
18703
|
+
joined4.set(chunk, offset);
|
|
18704
|
+
offset += chunk.byteLength;
|
|
18705
|
+
}
|
|
18706
|
+
return joined4;
|
|
18707
|
+
}
|
|
18708
|
+
function requireBytes2(key, value) {
|
|
18709
|
+
if (value instanceof Uint8Array) return value;
|
|
18710
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The IndexedDB record of "${key}" holds no bytes`, { key });
|
|
18711
|
+
}
|
|
18712
|
+
function requireAppendNumber(key, recordKey) {
|
|
18713
|
+
const n = Array.isArray(recordKey) && recordKey.length === 2 && recordKey[0] === key ? recordKey[1] : void 0;
|
|
18714
|
+
if (typeof n === "number" && Number.isSafeInteger(n) && n >= 1) return n;
|
|
18715
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `An append record of "${key}" carries no valid position`, {
|
|
18716
|
+
key
|
|
18717
|
+
});
|
|
18718
|
+
}
|
|
18719
|
+
async function readRecords(store, key) {
|
|
18720
|
+
const first = await requestResult(store.get(key));
|
|
18721
|
+
if (first === void 0 && await requestResult(store.getKey(key)) === void 0) return null;
|
|
18722
|
+
const bytes = requireBytes2(key, first);
|
|
18723
|
+
const range = appendRecordsOf(key);
|
|
18724
|
+
const keys = await requestResult(store.getAllKeys(range));
|
|
18725
|
+
const values = await requestResult(store.getAll(range));
|
|
18726
|
+
if (keys.length !== values.length) {
|
|
18727
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The append records of "${key}" changed during a read`, {
|
|
18728
|
+
key
|
|
18729
|
+
});
|
|
18730
|
+
}
|
|
18731
|
+
const appended = [];
|
|
18732
|
+
for (let at = 0; at < keys.length; at++) {
|
|
18733
|
+
appended.push({ n: requireAppendNumber(key, keys[at]), bytes: requireBytes2(key, values[at]) });
|
|
18734
|
+
}
|
|
18735
|
+
return { first: bytes, appended };
|
|
18736
|
+
}
|
|
18737
|
+
function lengthsOf(records) {
|
|
18738
|
+
return [
|
|
18739
|
+
{ n: 0, length: records.first.byteLength },
|
|
18740
|
+
...records.appended.map((record) => ({ n: record.n, length: record.bytes.byteLength }))
|
|
18741
|
+
];
|
|
18742
|
+
}
|
|
18743
|
+
function walkCursor(store, key, range, visit) {
|
|
18744
|
+
return new Promise((resolve, reject) => {
|
|
18745
|
+
const request = store.openCursor(range);
|
|
18746
|
+
request.onsuccess = () => {
|
|
18747
|
+
const cursor = request.result;
|
|
18748
|
+
if (cursor === null) {
|
|
18749
|
+
resolve(true);
|
|
18750
|
+
return;
|
|
18751
|
+
}
|
|
18752
|
+
let enough;
|
|
18753
|
+
try {
|
|
18754
|
+
enough = visit(requireAppendNumber(key, cursor.key), requireBytes2(key, cursor.value));
|
|
18755
|
+
} catch (error) {
|
|
18756
|
+
reject(error);
|
|
18757
|
+
return;
|
|
18758
|
+
}
|
|
18759
|
+
if (enough) resolve(false);
|
|
18760
|
+
else cursor.continue();
|
|
18761
|
+
};
|
|
18762
|
+
request.onerror = () => reject(request.error);
|
|
18763
|
+
});
|
|
18764
|
+
}
|
|
18765
|
+
async function walkBlocks(store, key, afterN, visit) {
|
|
18766
|
+
if (afterN === null) {
|
|
18767
|
+
const first = await requestResult(store.get(key));
|
|
18768
|
+
if (first === void 0 && await requestResult(store.getKey(key)) === void 0) return null;
|
|
18769
|
+
if (visit(0, requireBytes2(key, first))) return false;
|
|
18770
|
+
return walkCursor(store, key, appendRecordsOf(key), visit);
|
|
18771
|
+
}
|
|
18772
|
+
return walkCursor(store, key, IDBKeyRange.bound(appendKey(key, afterN), [key, []], true, true), visit);
|
|
18773
|
+
}
|
|
18774
|
+
function blocksChanged(key) {
|
|
18775
|
+
return new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The blocks of "${key}" changed during a read`, { key });
|
|
18776
|
+
}
|
|
18777
|
+
async function readBlocks(store, key, wanted) {
|
|
18778
|
+
const found = /* @__PURE__ */ new Map();
|
|
18779
|
+
const appended = wanted.filter((block) => block.n > 0);
|
|
18780
|
+
const reads = [];
|
|
18781
|
+
if (appended.length < wanted.length) {
|
|
18782
|
+
reads.push(
|
|
18783
|
+
requestResult(store.get(key)).then((value) => {
|
|
18784
|
+
found.set(0, requireBytes2(key, value));
|
|
18785
|
+
})
|
|
18786
|
+
);
|
|
18787
|
+
}
|
|
18788
|
+
const lowest = appended[0];
|
|
18789
|
+
const highest = appended[appended.length - 1];
|
|
18790
|
+
if (lowest !== void 0 && highest !== void 0) {
|
|
18791
|
+
const range = IDBKeyRange.bound(appendKey(key, lowest.n), appendKey(key, highest.n));
|
|
18792
|
+
reads.push(
|
|
18793
|
+
Promise.all([requestResult(store.getAllKeys(range)), requestResult(store.getAll(range))]).then(
|
|
18794
|
+
([keys, values]) => {
|
|
18795
|
+
if (keys.length !== values.length) throw blocksChanged(key);
|
|
18796
|
+
for (let at = 0; at < keys.length; at++) {
|
|
18797
|
+
found.set(requireAppendNumber(key, keys[at]), requireBytes2(key, values[at]));
|
|
18798
|
+
}
|
|
18799
|
+
}
|
|
18800
|
+
)
|
|
18801
|
+
);
|
|
18802
|
+
}
|
|
18803
|
+
await Promise.all(reads);
|
|
18804
|
+
for (const block of wanted) {
|
|
18805
|
+
if (found.get(block.n)?.byteLength !== block.length) throw blocksChanged(key);
|
|
18806
|
+
}
|
|
18807
|
+
return found;
|
|
18808
|
+
}
|
|
18809
|
+
function splitIntoBlocks(chunks, blockBytes) {
|
|
18810
|
+
let total = 0;
|
|
18811
|
+
for (const chunk of chunks) total += chunk.byteLength;
|
|
18812
|
+
const blocks = [];
|
|
18813
|
+
let block = new Uint8Array(Math.min(blockBytes, total));
|
|
18814
|
+
let filled = 0;
|
|
18815
|
+
for (const chunk of chunks) {
|
|
18816
|
+
let read = 0;
|
|
18817
|
+
while (read < chunk.byteLength) {
|
|
18818
|
+
const taken = Math.min(block.byteLength - filled, chunk.byteLength - read);
|
|
18819
|
+
block.set(chunk.subarray(read, read + taken), filled);
|
|
18820
|
+
filled += taken;
|
|
18821
|
+
read += taken;
|
|
18822
|
+
total -= taken;
|
|
18823
|
+
if (filled === block.byteLength && total > 0) {
|
|
18824
|
+
blocks.push(block);
|
|
18825
|
+
block = new Uint8Array(Math.min(blockBytes, total));
|
|
18826
|
+
filled = 0;
|
|
18827
|
+
}
|
|
18828
|
+
}
|
|
18829
|
+
}
|
|
18830
|
+
blocks.push(block);
|
|
18831
|
+
return blocks;
|
|
18832
|
+
}
|
|
18833
|
+
function contentOf(records) {
|
|
18834
|
+
if (records.appended.length === 0) return records.first;
|
|
18835
|
+
return joinBytes([records.first, ...records.appended.map((record) => record.bytes)]);
|
|
18836
|
+
}
|
|
18837
|
+
async function removeRecords(store, key) {
|
|
18838
|
+
await requestResult(store.delete(appendRecordsOf(key)));
|
|
18839
|
+
await requestResult(store.delete(key));
|
|
18840
|
+
}
|
|
18841
|
+
async function replaceRecords(store, key, records) {
|
|
18842
|
+
await removeRecords(store, key);
|
|
18843
|
+
await requestResult(store.put(records.first, key));
|
|
18844
|
+
for (const record of records.appended) await requestResult(store.put(record.bytes, appendKey(key, record.n)));
|
|
18845
|
+
}
|
|
18846
|
+
function listKeys(store, prefix) {
|
|
18847
|
+
return new Promise((resolve, reject) => {
|
|
18848
|
+
const keys = [];
|
|
18849
|
+
const request = store.openKeyCursor(IDBKeyRange.lowerBound(prefix));
|
|
18850
|
+
request.onsuccess = () => {
|
|
18851
|
+
const cursor = request.result;
|
|
18852
|
+
if (cursor === null || typeof cursor.key !== "string" || !cursor.key.startsWith(prefix)) {
|
|
18853
|
+
resolve(keys);
|
|
18854
|
+
return;
|
|
18855
|
+
}
|
|
18856
|
+
keys.push(cursor.key);
|
|
18857
|
+
cursor.continue();
|
|
18858
|
+
};
|
|
18859
|
+
request.onerror = () => reject(request.error);
|
|
18860
|
+
});
|
|
18861
|
+
}
|
|
18862
|
+
|
|
18863
|
+
// src/persistence/durability/indexeddb-directory/append-handle.ts
|
|
18864
|
+
async function openFileRecords(connection, key) {
|
|
18865
|
+
const parts = await connection.write(
|
|
18866
|
+
"strict",
|
|
18867
|
+
{ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open a log file" },
|
|
18868
|
+
async (store) => {
|
|
18869
|
+
const records = await readRecords(store, key);
|
|
18870
|
+
if (records !== null) return lengthsOf(records);
|
|
18871
|
+
await requestResult(store.put(new Uint8Array(0), key));
|
|
18872
|
+
return [{ n: 0, length: 0 }];
|
|
18873
|
+
}
|
|
18874
|
+
);
|
|
18875
|
+
const nextN = parts[parts.length - 1].n + 1;
|
|
18876
|
+
return { key, parts, unsynced: [], nextN, handles: 0, detached: false, failure: null, moving: null };
|
|
18877
|
+
}
|
|
18878
|
+
async function waitForMove(file) {
|
|
18879
|
+
while (file.moving !== null) await file.moving;
|
|
18880
|
+
}
|
|
18881
|
+
function lengthsOfRecords(records) {
|
|
18882
|
+
return records.map((record) => ({ n: record.n, length: record.bytes.byteLength }));
|
|
18883
|
+
}
|
|
18884
|
+
function sizeOf(file) {
|
|
18885
|
+
let size = 0;
|
|
18886
|
+
for (const part of file.parts) size += part.length;
|
|
18887
|
+
return size;
|
|
18888
|
+
}
|
|
18889
|
+
function recordKeyOf(key, n) {
|
|
18890
|
+
return n === 0 ? key : appendKey(key, n);
|
|
18891
|
+
}
|
|
18892
|
+
function putRecords(store, key, records) {
|
|
18893
|
+
return Promise.all(records.map((record) => requestResult(store.put(record.bytes, recordKeyOf(key, record.n)))));
|
|
18894
|
+
}
|
|
18895
|
+
function holdsNoBytes(file) {
|
|
18896
|
+
return file.parts.length === 1 && file.parts[0].length === 0;
|
|
18897
|
+
}
|
|
18898
|
+
function createAppendHandle(file, deps) {
|
|
18899
|
+
const { connection, blocks } = deps;
|
|
18900
|
+
const appendFailure = { code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "append to a log file" };
|
|
18901
|
+
const strictCommitFailure = {
|
|
18902
|
+
code: ErrorCodes.PERSISTENCE_FSYNC_FAILED,
|
|
18903
|
+
action: "commit a log file with strict durability"
|
|
18904
|
+
};
|
|
18905
|
+
let closed = false;
|
|
18906
|
+
file.handles += 1;
|
|
18907
|
+
function requireUsable() {
|
|
18908
|
+
if (closed) throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The log file "${file.key}" is closed`);
|
|
18909
|
+
if (file.failure !== null) throw file.failure;
|
|
18910
|
+
if (file.detached) {
|
|
18911
|
+
throw new NarsilError(
|
|
18912
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
18913
|
+
`The log file "${file.key}" was removed or replaced while a handle held it open`,
|
|
18914
|
+
{ key: file.key }
|
|
18915
|
+
);
|
|
18916
|
+
}
|
|
18917
|
+
}
|
|
18918
|
+
function latch(error) {
|
|
18919
|
+
const failed = error instanceof Error ? error : new Error(String(error));
|
|
18920
|
+
file.failure ??= failed;
|
|
18921
|
+
return file.failure;
|
|
18922
|
+
}
|
|
18923
|
+
async function putCommitted(durability, failure, records) {
|
|
18924
|
+
const key = file.key;
|
|
18925
|
+
const epoch = blocks.appendEpochOf(key);
|
|
18926
|
+
try {
|
|
18927
|
+
await connection.write(durability, failure, (store) => putRecords(store, key, records));
|
|
18928
|
+
} catch (error) {
|
|
18929
|
+
throw latch(error);
|
|
18930
|
+
}
|
|
18931
|
+
blocks.appended(key, epoch, lengthsOfRecords(records));
|
|
18932
|
+
}
|
|
18933
|
+
async function writeUnsynced(durability) {
|
|
18934
|
+
if (file.moving !== null) await waitForMove(file);
|
|
18935
|
+
const batch = file.unsynced.slice();
|
|
18936
|
+
if (batch.length === 0) return;
|
|
18937
|
+
await putCommitted(durability, durability === "strict" ? strictCommitFailure : appendFailure, batch);
|
|
18938
|
+
file.unsynced.splice(0, batch.length);
|
|
18939
|
+
}
|
|
18940
|
+
async function truncateTo(byteLength) {
|
|
18941
|
+
if (file.moving !== null) await waitForMove(file);
|
|
18942
|
+
const key = file.key;
|
|
18943
|
+
const batch = file.unsynced.slice();
|
|
18944
|
+
const kept = [];
|
|
18945
|
+
let start = 0;
|
|
18946
|
+
for (const part of file.parts) {
|
|
18947
|
+
if (start >= byteLength && part.n !== 0) break;
|
|
18948
|
+
kept.push({ ...part, length: Math.min(part.length, byteLength - start) });
|
|
18949
|
+
start += part.length;
|
|
18950
|
+
}
|
|
18951
|
+
const dropped = file.parts.slice(kept.length);
|
|
18952
|
+
const cut = kept[kept.length - 1];
|
|
18953
|
+
const cutLength = cut.length;
|
|
18954
|
+
const cutIsPartial = cutLength < (file.parts[kept.length - 1]?.length ?? 0);
|
|
18955
|
+
const epoch = blocks.begin(key);
|
|
18956
|
+
try {
|
|
18957
|
+
await connection.write("strict", strictCommitFailure, async (store) => {
|
|
18958
|
+
await putRecords(store, key, batch);
|
|
18959
|
+
for (const part of dropped) await requestResult(store.delete(recordKeyOf(key, part.n)));
|
|
18960
|
+
if (!cutIsPartial) return;
|
|
18961
|
+
const recordKey = recordKeyOf(key, cut.n);
|
|
18962
|
+
const stored = await requestResult(store.get(recordKey));
|
|
18963
|
+
if (!(stored instanceof Uint8Array) || stored.byteLength < cutLength) {
|
|
18964
|
+
throw new NarsilError(
|
|
18965
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
18966
|
+
`A record of "${key}" holds fewer bytes than its log`,
|
|
18967
|
+
{
|
|
18968
|
+
key
|
|
18969
|
+
}
|
|
18970
|
+
);
|
|
18971
|
+
}
|
|
18972
|
+
await requestResult(store.put(stored.slice(0, cutLength), recordKey));
|
|
18973
|
+
});
|
|
18974
|
+
} catch (error) {
|
|
18975
|
+
blocks.settle(key, epoch, void 0);
|
|
18976
|
+
throw error;
|
|
18977
|
+
}
|
|
18978
|
+
blocks.settle(key, epoch, kept);
|
|
18979
|
+
file.parts = kept;
|
|
18980
|
+
file.unsynced = [];
|
|
18981
|
+
file.nextN = kept[kept.length - 1].n + 1;
|
|
18982
|
+
}
|
|
18983
|
+
return {
|
|
18984
|
+
async append(bytes) {
|
|
18985
|
+
requireUsable();
|
|
18986
|
+
if (bytes.byteLength === 0) return;
|
|
18987
|
+
const records = [];
|
|
18988
|
+
for (const piece of splitIntoBlocks([bytes], INDEXEDDB_BLOCK_BYTES)) {
|
|
18989
|
+
if (holdsNoBytes(file)) {
|
|
18990
|
+
file.parts[0] = { n: 0, length: piece.byteLength };
|
|
18991
|
+
records.push({ n: 0, bytes: piece });
|
|
18992
|
+
continue;
|
|
18993
|
+
}
|
|
18994
|
+
records.push({ n: file.nextN, bytes: piece });
|
|
18995
|
+
file.parts.push({ n: file.nextN, length: piece.byteLength });
|
|
18996
|
+
file.nextN += 1;
|
|
18997
|
+
}
|
|
18998
|
+
file.unsynced.push(...records);
|
|
18999
|
+
if (!deps.relaxedAppends) return;
|
|
19000
|
+
if (file.moving !== null) await waitForMove(file);
|
|
19001
|
+
await putCommitted("relaxed", appendFailure, records);
|
|
19002
|
+
},
|
|
19003
|
+
async sync() {
|
|
19004
|
+
requireUsable();
|
|
19005
|
+
await writeUnsynced("strict");
|
|
19006
|
+
},
|
|
19007
|
+
async truncate(byteLength) {
|
|
19008
|
+
requireUsable();
|
|
19009
|
+
if (!Number.isSafeInteger(byteLength) || byteLength < 0 || byteLength > sizeOf(file)) {
|
|
19010
|
+
throw new NarsilError(
|
|
19011
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
19012
|
+
`The log file "${file.key}" cannot be cut to ${byteLength} bytes, because it holds ${sizeOf(file)}`,
|
|
19013
|
+
{ key: file.key, byteLength }
|
|
19014
|
+
);
|
|
19015
|
+
}
|
|
19016
|
+
try {
|
|
19017
|
+
await truncateTo(byteLength);
|
|
19018
|
+
} catch (error) {
|
|
19019
|
+
throw latch(error);
|
|
19020
|
+
}
|
|
19021
|
+
},
|
|
19022
|
+
async size() {
|
|
19023
|
+
if (closed) throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The log file "${file.key}" is closed`);
|
|
19024
|
+
return sizeOf(file);
|
|
19025
|
+
},
|
|
19026
|
+
async close() {
|
|
19027
|
+
if (closed) return;
|
|
19028
|
+
closed = true;
|
|
19029
|
+
file.handles -= 1;
|
|
19030
|
+
try {
|
|
19031
|
+
if (!deps.relaxedAppends && file.failure === null && !file.detached) await writeUnsynced("relaxed");
|
|
19032
|
+
} finally {
|
|
19033
|
+
if (file.handles === 0) deps.release(file);
|
|
19034
|
+
}
|
|
19035
|
+
}
|
|
19036
|
+
};
|
|
19037
|
+
}
|
|
19038
|
+
|
|
19039
|
+
// src/persistence/durability/indexeddb-directory/block-map.ts
|
|
19040
|
+
function laidOut(lengths) {
|
|
19041
|
+
const blocks = [];
|
|
19042
|
+
let start = 0;
|
|
19043
|
+
for (const { n, length } of lengths) {
|
|
19044
|
+
blocks.push({ n, length, start });
|
|
19045
|
+
start += length;
|
|
19046
|
+
}
|
|
19047
|
+
return blocks;
|
|
19048
|
+
}
|
|
19049
|
+
function sizeOfBlocks(blocks) {
|
|
19050
|
+
const last = blocks[blocks.length - 1];
|
|
19051
|
+
return last === void 0 ? 0 : last.start + last.length;
|
|
19052
|
+
}
|
|
19053
|
+
function firstBlockAt(blocks, offset) {
|
|
19054
|
+
let low = 0;
|
|
19055
|
+
let high = blocks.length - 1;
|
|
19056
|
+
while (low < high) {
|
|
19057
|
+
const middle = low + high + 1 >>> 1;
|
|
19058
|
+
if (blocks[middle].start <= offset) low = middle;
|
|
19059
|
+
else high = middle - 1;
|
|
19060
|
+
}
|
|
19061
|
+
return low;
|
|
19062
|
+
}
|
|
19063
|
+
function blockNumbered(blocks, n) {
|
|
19064
|
+
let low = 0;
|
|
19065
|
+
let high = blocks.length - 1;
|
|
19066
|
+
while (low <= high) {
|
|
19067
|
+
const middle = low + high >>> 1;
|
|
19068
|
+
const block = blocks[middle];
|
|
19069
|
+
if (block.n === n) return block;
|
|
19070
|
+
if (block.n < n) low = middle + 1;
|
|
19071
|
+
else high = middle - 1;
|
|
19072
|
+
}
|
|
19073
|
+
return void 0;
|
|
19074
|
+
}
|
|
19075
|
+
function mergedBlocks(known, complete, lengths) {
|
|
19076
|
+
let merged = known;
|
|
19077
|
+
for (const { n, length } of lengths) {
|
|
19078
|
+
const last = merged[merged.length - 1];
|
|
19079
|
+
if (last === void 0) return void 0;
|
|
19080
|
+
if (n > last.n) {
|
|
19081
|
+
if (complete) merged.push({ n, length, start: last.start + last.length });
|
|
19082
|
+
continue;
|
|
19083
|
+
}
|
|
19084
|
+
if (n === 0 && merged.length === 1 && last.length === 0) {
|
|
19085
|
+
if (!complete) return void 0;
|
|
19086
|
+
merged = [{ n: 0, length, start: 0 }];
|
|
19087
|
+
continue;
|
|
19088
|
+
}
|
|
19089
|
+
const existing = blockNumbered(merged, n);
|
|
19090
|
+
if (existing === void 0 || existing.length !== length) return void 0;
|
|
19091
|
+
}
|
|
19092
|
+
return merged;
|
|
19093
|
+
}
|
|
19094
|
+
function createBlockMap() {
|
|
19095
|
+
const states = /* @__PURE__ */ new Map();
|
|
19096
|
+
const cache = /* @__PURE__ */ new Map();
|
|
19097
|
+
let counter = 0;
|
|
19098
|
+
function next() {
|
|
19099
|
+
counter += 1;
|
|
19100
|
+
return counter;
|
|
19101
|
+
}
|
|
19102
|
+
function stateOf2(key) {
|
|
19103
|
+
const known = states.get(key);
|
|
19104
|
+
if (known !== void 0) return known;
|
|
19105
|
+
const created = { generation: next(), epoch: next(), changes: 0, blocks: void 0, complete: false };
|
|
19106
|
+
states.set(key, created);
|
|
19107
|
+
return created;
|
|
19108
|
+
}
|
|
19109
|
+
function cacheKeyOf(key, n) {
|
|
19110
|
+
return `${n}\0${key}`;
|
|
19111
|
+
}
|
|
19112
|
+
function invalidate(state) {
|
|
19113
|
+
state.generation = next();
|
|
19114
|
+
state.changes += 1;
|
|
19115
|
+
state.blocks = void 0;
|
|
19116
|
+
state.complete = false;
|
|
19117
|
+
}
|
|
19118
|
+
return {
|
|
19119
|
+
viewOf(key) {
|
|
19120
|
+
const state = stateOf2(key);
|
|
19121
|
+
const layout = state.blocks === void 0 ? void 0 : { blocks: state.blocks, complete: state.complete };
|
|
19122
|
+
return { generation: state.generation, changes: state.changes, layout };
|
|
19123
|
+
},
|
|
19124
|
+
generationOf: (key) => stateOf2(key).generation,
|
|
19125
|
+
install(key, view, blocks, reachedEnd) {
|
|
19126
|
+
const state = stateOf2(key);
|
|
19127
|
+
if (state.generation !== view.generation) return;
|
|
19128
|
+
const complete = reachedEnd && state.changes === view.changes;
|
|
19129
|
+
const current = state.blocks;
|
|
19130
|
+
if (current !== void 0 && (state.complete || !complete && blocks.length <= current.length)) return;
|
|
19131
|
+
state.blocks = blocks;
|
|
19132
|
+
state.complete = complete;
|
|
19133
|
+
},
|
|
19134
|
+
forget: (key) => invalidate(stateOf2(key)),
|
|
19135
|
+
begin(key) {
|
|
19136
|
+
const state = stateOf2(key);
|
|
19137
|
+
invalidate(state);
|
|
19138
|
+
state.epoch = next();
|
|
19139
|
+
return state.epoch;
|
|
19140
|
+
},
|
|
19141
|
+
settle(key, epoch, lengths) {
|
|
19142
|
+
const state = stateOf2(key);
|
|
19143
|
+
if (state.epoch !== epoch) return;
|
|
19144
|
+
invalidate(state);
|
|
19145
|
+
if (lengths === void 0) return;
|
|
19146
|
+
state.blocks = laidOut(lengths);
|
|
19147
|
+
state.complete = true;
|
|
19148
|
+
},
|
|
19149
|
+
appendEpochOf: (key) => stateOf2(key).epoch,
|
|
19150
|
+
appended(key, epoch, lengths) {
|
|
19151
|
+
const state = stateOf2(key);
|
|
19152
|
+
if (state.epoch !== epoch) return;
|
|
19153
|
+
state.changes += 1;
|
|
19154
|
+
if (state.blocks === void 0) return;
|
|
19155
|
+
const merged = mergedBlocks(state.blocks, state.complete, lengths);
|
|
19156
|
+
if (merged === void 0) invalidate(state);
|
|
19157
|
+
else state.blocks = merged;
|
|
19158
|
+
},
|
|
19159
|
+
covering(key, generation, blocks, offset, length) {
|
|
19160
|
+
const covered = [];
|
|
19161
|
+
const end = offset + length;
|
|
19162
|
+
for (let at = firstBlockAt(blocks, offset); at < blocks.length; at++) {
|
|
19163
|
+
const block = blocks[at];
|
|
19164
|
+
if (block.start >= end && covered.length > 0) break;
|
|
19165
|
+
const cacheKey = cacheKeyOf(key, block.n);
|
|
19166
|
+
const entry = cache.get(cacheKey);
|
|
19167
|
+
const cached = entry !== void 0 && entry.generation === generation && entry.bytes.byteLength === block.length ? entry.bytes : void 0;
|
|
19168
|
+
if (cached !== void 0) {
|
|
19169
|
+
cache.delete(cacheKey);
|
|
19170
|
+
cache.set(cacheKey, { generation, bytes: cached });
|
|
19171
|
+
}
|
|
19172
|
+
covered.push({ ...block, cached });
|
|
19173
|
+
if (block.start + block.length >= end) break;
|
|
19174
|
+
}
|
|
19175
|
+
return covered;
|
|
19176
|
+
},
|
|
19177
|
+
keep(key, generation, n, bytes) {
|
|
19178
|
+
if (bytes.byteLength > INDEXEDDB_BLOCK_BYTES || stateOf2(key).generation !== generation) return;
|
|
19179
|
+
const cacheKey = cacheKeyOf(key, n);
|
|
19180
|
+
cache.delete(cacheKey);
|
|
19181
|
+
cache.set(cacheKey, { generation, bytes });
|
|
19182
|
+
while (cache.size > INDEXEDDB_CACHED_BLOCKS) {
|
|
19183
|
+
const oldest = cache.keys().next();
|
|
19184
|
+
if (oldest.done === true) break;
|
|
19185
|
+
cache.delete(oldest.value);
|
|
19186
|
+
}
|
|
19187
|
+
}
|
|
19188
|
+
};
|
|
19189
|
+
}
|
|
19190
|
+
|
|
19191
|
+
// src/persistence/durability/indexeddb-directory/lock.ts
|
|
19192
|
+
var WEB_LOCK_NAME_PREFIX = "narsil-wal";
|
|
19193
|
+
var heldWithoutWebLocks = /* @__PURE__ */ new Set();
|
|
19194
|
+
function webLockNameOf(location) {
|
|
19195
|
+
return `${WEB_LOCK_NAME_PREFIX}${JSON.stringify([location.databaseName, location.storeName])}`;
|
|
19196
|
+
}
|
|
19197
|
+
function heldElsewhere2(location) {
|
|
19198
|
+
return new NarsilError(
|
|
19199
|
+
ErrorCodes.CONFIG_INVALID,
|
|
19200
|
+
`Another tab or engine holds the write-ahead log of IndexedDB database "${location.databaseName}" store "${location.storeName}". Several tabs that share one store need an invalidation adapter or durability.tier "snapshot"`,
|
|
19201
|
+
{ databaseName: location.databaseName, storeName: location.storeName }
|
|
19202
|
+
);
|
|
19203
|
+
}
|
|
19204
|
+
function lockWithinProcess(name, location) {
|
|
19205
|
+
if (heldWithoutWebLocks.has(name)) throw heldElsewhere2(location);
|
|
19206
|
+
heldWithoutWebLocks.add(name);
|
|
19207
|
+
let held = true;
|
|
19208
|
+
return {
|
|
19209
|
+
holds: async () => held,
|
|
19210
|
+
async release() {
|
|
19211
|
+
if (!held) return;
|
|
19212
|
+
held = false;
|
|
19213
|
+
heldWithoutWebLocks.delete(name);
|
|
19214
|
+
}
|
|
19215
|
+
};
|
|
19216
|
+
}
|
|
19217
|
+
function lockAcrossTabs(locks, name, location) {
|
|
19218
|
+
return new Promise((resolve, reject) => {
|
|
19219
|
+
let held = false;
|
|
19220
|
+
let releaseLock = () => void 0;
|
|
19221
|
+
const releasing = new Promise((settle) => {
|
|
19222
|
+
releaseLock = settle;
|
|
19223
|
+
});
|
|
19224
|
+
locks.request(name, { mode: "exclusive", ifAvailable: true }, (granted) => {
|
|
19225
|
+
if (granted === null) {
|
|
19226
|
+
reject(heldElsewhere2(location));
|
|
19227
|
+
return void 0;
|
|
19228
|
+
}
|
|
19229
|
+
held = true;
|
|
19230
|
+
resolve({
|
|
19231
|
+
holds: async () => held,
|
|
19232
|
+
async release() {
|
|
19233
|
+
held = false;
|
|
19234
|
+
releaseLock();
|
|
19235
|
+
}
|
|
19236
|
+
});
|
|
19237
|
+
return releasing;
|
|
19238
|
+
}).then(
|
|
19239
|
+
() => {
|
|
19240
|
+
held = false;
|
|
19241
|
+
},
|
|
19242
|
+
(error) => {
|
|
19243
|
+
held = false;
|
|
19244
|
+
reject(error);
|
|
19245
|
+
}
|
|
19246
|
+
);
|
|
19247
|
+
});
|
|
19248
|
+
}
|
|
19249
|
+
function lockIndexedDBStore(location) {
|
|
19250
|
+
const name = webLockNameOf(location);
|
|
19251
|
+
const locks = globalThis.navigator?.locks;
|
|
19252
|
+
if (locks === void 0) return Promise.resolve().then(() => lockWithinProcess(name, location));
|
|
19253
|
+
return lockAcrossTabs(locks, name, location);
|
|
19254
|
+
}
|
|
19255
|
+
|
|
19256
|
+
// src/persistence/durability/indexeddb-directory/range-read.ts
|
|
19257
|
+
function endsBefore(key, offset, length) {
|
|
19258
|
+
return new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `"${key}" ends before the bytes a read asked for`, {
|
|
19259
|
+
key,
|
|
19260
|
+
offset,
|
|
19261
|
+
length
|
|
19262
|
+
});
|
|
19263
|
+
}
|
|
19264
|
+
function isLoadFailure(error) {
|
|
19265
|
+
return error instanceof NarsilError && error.code === ErrorCodes.PERSISTENCE_LOAD_FAILED;
|
|
19266
|
+
}
|
|
19267
|
+
function assembled(key, covered, fetched, offset, length) {
|
|
19268
|
+
const bytes = new Uint8Array(length);
|
|
19269
|
+
for (const block of covered) {
|
|
19270
|
+
if (block.length === 0) continue;
|
|
19271
|
+
const source = block.cached ?? fetched.get(block.n);
|
|
19272
|
+
if (source === void 0) throw endsBefore(key, offset, length);
|
|
19273
|
+
const from = Math.max(offset, block.start);
|
|
19274
|
+
const to = Math.min(offset + length, block.start + block.length);
|
|
19275
|
+
bytes.set(source.subarray(from - block.start, to - block.start), from - offset);
|
|
19276
|
+
}
|
|
19277
|
+
return bytes;
|
|
19278
|
+
}
|
|
19279
|
+
function createRangeReads(connection, blocks, readContent) {
|
|
19280
|
+
async function extendTo(key, view, offset, end) {
|
|
19281
|
+
const known = view.layout === void 0 ? [] : view.layout.blocks.slice();
|
|
19282
|
+
const fetched = /* @__PURE__ */ new Map();
|
|
19283
|
+
let size = sizeOfBlocks(known);
|
|
19284
|
+
const last = known[known.length - 1];
|
|
19285
|
+
const reachedEnd = await connection.read(
|
|
19286
|
+
{ code: ErrorCodes.PERSISTENCE_LOAD_FAILED, action: `read the blocks of "${key}"` },
|
|
19287
|
+
(store) => walkBlocks(store, key, last === void 0 ? null : last.n, (n, bytes) => {
|
|
19288
|
+
const start = size;
|
|
19289
|
+
known.push({ n, length: bytes.byteLength, start });
|
|
19290
|
+
size += bytes.byteLength;
|
|
19291
|
+
if (bytes.byteLength > 0 && start < end && size > offset) fetched.set(n, bytes);
|
|
19292
|
+
return size >= end;
|
|
19293
|
+
})
|
|
19294
|
+
);
|
|
19295
|
+
if (reachedEnd === null) return null;
|
|
19296
|
+
blocks.install(key, view, known, reachedEnd);
|
|
19297
|
+
return { blocks: known, fetched };
|
|
19298
|
+
}
|
|
19299
|
+
async function wholeFileRange(key, offset, length) {
|
|
19300
|
+
const content = await readContent(key);
|
|
19301
|
+
if (content === null || offset + length > content.byteLength) throw endsBefore(key, offset, length);
|
|
19302
|
+
return content.slice(offset, offset + length);
|
|
19303
|
+
}
|
|
19304
|
+
async function blockRange(key, offset, length) {
|
|
19305
|
+
const end = offset + length;
|
|
19306
|
+
const view = blocks.viewOf(key);
|
|
19307
|
+
let layout = view.layout?.blocks;
|
|
19308
|
+
let fetched = /* @__PURE__ */ new Map();
|
|
19309
|
+
if (view.layout === void 0 || !view.layout.complete && sizeOfBlocks(view.layout.blocks) < end) {
|
|
19310
|
+
const extended = await extendTo(key, view, offset, end);
|
|
19311
|
+
layout = extended?.blocks;
|
|
19312
|
+
if (extended !== null) fetched = extended.fetched;
|
|
19313
|
+
}
|
|
19314
|
+
if (layout === void 0 || sizeOfBlocks(layout) < end) {
|
|
19315
|
+
if (blocks.generationOf(key) !== view.generation) return wholeFileRange(key, offset, length);
|
|
19316
|
+
throw endsBefore(key, offset, length);
|
|
19317
|
+
}
|
|
19318
|
+
if (length === 0) return new Uint8Array(0);
|
|
19319
|
+
const covered = blocks.covering(key, view.generation, layout, offset, length);
|
|
19320
|
+
const missing2 = covered.filter((block) => block.length > 0 && block.cached === void 0 && !fetched.has(block.n));
|
|
19321
|
+
if (missing2.length > 0) {
|
|
19322
|
+
try {
|
|
19323
|
+
const read = await connection.read(
|
|
19324
|
+
{ code: ErrorCodes.PERSISTENCE_LOAD_FAILED, action: `read "${key}"` },
|
|
19325
|
+
(store) => readBlocks(store, key, missing2)
|
|
19326
|
+
);
|
|
19327
|
+
for (const [n, bytes] of read) fetched.set(n, bytes);
|
|
19328
|
+
} catch (error) {
|
|
19329
|
+
if (!isLoadFailure(error)) throw error;
|
|
19330
|
+
blocks.forget(key);
|
|
19331
|
+
return wholeFileRange(key, offset, length);
|
|
19332
|
+
}
|
|
19333
|
+
}
|
|
19334
|
+
if (blocks.generationOf(key) !== view.generation) return wholeFileRange(key, offset, length);
|
|
19335
|
+
for (const [n, bytes] of fetched) blocks.keep(key, view.generation, n, bytes);
|
|
19336
|
+
return assembled(key, covered, fetched, offset, length);
|
|
19337
|
+
}
|
|
19338
|
+
return {
|
|
19339
|
+
async readRange(key, offset, length) {
|
|
19340
|
+
if (!Number.isSafeInteger(offset) || !Number.isSafeInteger(length) || offset < 0 || length < 0) {
|
|
19341
|
+
throw endsBefore(key, offset, length);
|
|
19342
|
+
}
|
|
19343
|
+
return blockRange(key, offset, length);
|
|
19344
|
+
},
|
|
19345
|
+
async byteLength(key) {
|
|
19346
|
+
const view = blocks.viewOf(key);
|
|
19347
|
+
if (view.layout?.complete === true) return sizeOfBlocks(view.layout.blocks);
|
|
19348
|
+
const extended = await extendTo(key, view, Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY);
|
|
19349
|
+
if (blocks.generationOf(key) !== view.generation) return (await readContent(key))?.byteLength ?? null;
|
|
19350
|
+
return extended === null ? null : sizeOfBlocks(extended.blocks);
|
|
19351
|
+
}
|
|
19352
|
+
};
|
|
19353
|
+
}
|
|
19354
|
+
|
|
19355
|
+
// src/persistence/durability/indexeddb-directory/index.ts
|
|
19356
|
+
function noValue() {
|
|
19357
|
+
return void 0;
|
|
19358
|
+
}
|
|
19359
|
+
function requireKey(key) {
|
|
19360
|
+
if (typeof key !== "string" || key.length === 0) {
|
|
19361
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: empty");
|
|
19362
|
+
}
|
|
19363
|
+
if (key.includes("\0")) {
|
|
19364
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: null byte detected", { key });
|
|
19365
|
+
}
|
|
19366
|
+
}
|
|
19367
|
+
function createIndexedDBDirectory(location, options) {
|
|
19368
|
+
const saveFailed = ErrorCodes.PERSISTENCE_SAVE_FAILED;
|
|
19369
|
+
const loadFailed2 = ErrorCodes.PERSISTENCE_LOAD_FAILED;
|
|
19370
|
+
const connection = openLogConnection(location);
|
|
19371
|
+
const blocks = createBlockMap();
|
|
19372
|
+
const openFiles2 = /* @__PURE__ */ new Map();
|
|
19373
|
+
const opening = /* @__PURE__ */ new Map();
|
|
19374
|
+
async function settledWrite(key, write, lengthsAfter) {
|
|
19375
|
+
const epoch = blocks.begin(key);
|
|
19376
|
+
let result;
|
|
19377
|
+
try {
|
|
19378
|
+
result = await write();
|
|
19379
|
+
} catch (error) {
|
|
19380
|
+
blocks.settle(key, epoch, void 0);
|
|
19381
|
+
throw error;
|
|
19382
|
+
}
|
|
19383
|
+
blocks.settle(key, epoch, lengthsAfter(result));
|
|
19384
|
+
return result;
|
|
19385
|
+
}
|
|
19386
|
+
function detach(key) {
|
|
19387
|
+
const file = openFiles2.get(key);
|
|
19388
|
+
if (file === void 0) return;
|
|
19389
|
+
file.detached = true;
|
|
19390
|
+
openFiles2.delete(key);
|
|
19391
|
+
}
|
|
19392
|
+
function release(file) {
|
|
19393
|
+
if (openFiles2.get(file.key) === file) openFiles2.delete(file.key);
|
|
19394
|
+
}
|
|
19395
|
+
async function openFile(key) {
|
|
19396
|
+
const existing = openFiles2.get(key);
|
|
19397
|
+
if (existing !== void 0) return existing;
|
|
19398
|
+
const pending = opening.get(key);
|
|
19399
|
+
if (pending !== void 0) return pending;
|
|
19400
|
+
const opened = settledWrite(
|
|
19401
|
+
key,
|
|
19402
|
+
() => openFileRecords(connection, key),
|
|
19403
|
+
(file) => file.parts
|
|
19404
|
+
).then(
|
|
19405
|
+
(file) => {
|
|
19406
|
+
opening.delete(key);
|
|
19407
|
+
openFiles2.set(key, file);
|
|
19408
|
+
return file;
|
|
19409
|
+
},
|
|
19410
|
+
(error) => {
|
|
19411
|
+
opening.delete(key);
|
|
19412
|
+
throw error;
|
|
19413
|
+
}
|
|
19414
|
+
);
|
|
19415
|
+
opening.set(key, opened);
|
|
19416
|
+
return opened;
|
|
19417
|
+
}
|
|
19418
|
+
async function readContent(key) {
|
|
19419
|
+
requireKey(key);
|
|
19420
|
+
const view = blocks.viewOf(key);
|
|
19421
|
+
const records = await connection.read(
|
|
19422
|
+
{ code: loadFailed2, action: `read "${key}"` },
|
|
19423
|
+
(store) => readRecords(store, key)
|
|
19424
|
+
);
|
|
19425
|
+
if (records === null) return null;
|
|
19426
|
+
blocks.install(key, view, laidOut(lengthsOf(records)), true);
|
|
19427
|
+
return contentOf(records);
|
|
19428
|
+
}
|
|
19429
|
+
const ranges = createRangeReads(connection, blocks, readContent);
|
|
19430
|
+
function renameAfterMove(moving, from, to) {
|
|
19431
|
+
moving.key = to;
|
|
19432
|
+
if (openFiles2.get(from) !== moving) return;
|
|
19433
|
+
openFiles2.delete(from);
|
|
19434
|
+
detach(to);
|
|
19435
|
+
openFiles2.set(to, moving);
|
|
19436
|
+
}
|
|
19437
|
+
async function moveRecords(from, to) {
|
|
19438
|
+
const fromEpoch = blocks.begin(from);
|
|
19439
|
+
const moved = settledWrite(
|
|
19440
|
+
to,
|
|
19441
|
+
() => connection.write("strict", { code: saveFailed, action: `move "${from}" to "${to}"` }, async (store) => {
|
|
19442
|
+
const records = await readRecords(store, from);
|
|
19443
|
+
if (records === null) {
|
|
19444
|
+
throw new NarsilError(saveFailed, `Failed to move "${from}" to "${to}": "${from}" holds nothing`, {
|
|
19445
|
+
from,
|
|
19446
|
+
to
|
|
19447
|
+
});
|
|
19448
|
+
}
|
|
19449
|
+
await replaceRecords(store, to, records);
|
|
19450
|
+
await removeRecords(store, from);
|
|
19451
|
+
return lengthsOf(records);
|
|
19452
|
+
}),
|
|
19453
|
+
(lengths) => lengths
|
|
19454
|
+
);
|
|
19455
|
+
const moving = openFiles2.get(from);
|
|
19456
|
+
if (moving !== void 0) moving.moving = moved.then(noValue, noValue);
|
|
19457
|
+
try {
|
|
19458
|
+
await moved;
|
|
19459
|
+
if (moving !== void 0) renameAfterMove(moving, from, to);
|
|
19460
|
+
} finally {
|
|
19461
|
+
blocks.settle(from, fromEpoch, void 0);
|
|
19462
|
+
if (moving !== void 0) moving.moving = null;
|
|
19463
|
+
}
|
|
19464
|
+
}
|
|
19465
|
+
return {
|
|
19466
|
+
root: connection.root,
|
|
19467
|
+
async appendHandle(key) {
|
|
19468
|
+
requireKey(key);
|
|
19469
|
+
const file = await openFile(key);
|
|
19470
|
+
return createAppendHandle(file, { connection, blocks, relaxedAppends: options.relaxedAppends, release });
|
|
19471
|
+
},
|
|
19472
|
+
async markerHandle(key) {
|
|
19473
|
+
requireKey(key);
|
|
19474
|
+
let current = await settledWrite(
|
|
19475
|
+
key,
|
|
19476
|
+
() => connection.write("strict", { code: saveFailed, action: `open "${key}"` }, async (store) => {
|
|
19477
|
+
const records = await readRecords(store, key);
|
|
19478
|
+
if (records !== null) return contentOf(records);
|
|
19479
|
+
const empty = new Uint8Array(0);
|
|
19480
|
+
await requestResult(store.put(empty, key));
|
|
19481
|
+
return empty;
|
|
19482
|
+
}),
|
|
19483
|
+
(bytes) => [{ n: 0, length: bytes.byteLength }]
|
|
19484
|
+
);
|
|
19485
|
+
return {
|
|
19486
|
+
read: async () => current.slice(),
|
|
19487
|
+
async writeSlot(offset, slot) {
|
|
19488
|
+
const next = new Uint8Array(Math.max(current.byteLength, offset + slot.byteLength));
|
|
19489
|
+
next.set(current);
|
|
19490
|
+
next.set(slot, offset);
|
|
19491
|
+
await settledWrite(
|
|
19492
|
+
key,
|
|
19493
|
+
() => connection.write(
|
|
19494
|
+
"strict",
|
|
19495
|
+
{ code: ErrorCodes.PERSISTENCE_FSYNC_FAILED, action: `commit "${key}"` },
|
|
19496
|
+
(store) => replaceRecords(store, key, { first: next, appended: [] })
|
|
19497
|
+
),
|
|
19498
|
+
() => [{ n: 0, length: next.byteLength }]
|
|
19499
|
+
);
|
|
19500
|
+
current = next;
|
|
19501
|
+
},
|
|
19502
|
+
close: async () => void 0
|
|
19503
|
+
};
|
|
19504
|
+
},
|
|
19505
|
+
async atomicWrite(key, data) {
|
|
19506
|
+
requireKey(key);
|
|
19507
|
+
const pieces = splitIntoBlocks(Array.isArray(data) ? data : [data], INDEXEDDB_BLOCK_BYTES);
|
|
19508
|
+
const records = {
|
|
19509
|
+
first: pieces[0],
|
|
19510
|
+
appended: pieces.slice(1).map((bytes, at) => ({ n: at + 1, bytes }))
|
|
19511
|
+
};
|
|
19512
|
+
detach(key);
|
|
19513
|
+
await settledWrite(
|
|
19514
|
+
key,
|
|
19515
|
+
() => connection.write(
|
|
19516
|
+
"strict",
|
|
19517
|
+
{ code: saveFailed, action: `write "${key}"` },
|
|
19518
|
+
(store) => replaceRecords(store, key, records)
|
|
19519
|
+
),
|
|
19520
|
+
() => lengthsOf(records)
|
|
19521
|
+
);
|
|
19522
|
+
},
|
|
19523
|
+
async syncDirectoryOf(key) {
|
|
19524
|
+
requireKey(key);
|
|
19525
|
+
},
|
|
19526
|
+
read: readContent,
|
|
19527
|
+
async readRange(key, offset, length) {
|
|
19528
|
+
requireKey(key);
|
|
19529
|
+
return ranges.readRange(key, offset, length);
|
|
19530
|
+
},
|
|
19531
|
+
async byteLength(key) {
|
|
19532
|
+
requireKey(key);
|
|
19533
|
+
return ranges.byteLength(key);
|
|
19534
|
+
},
|
|
19535
|
+
async remove(key) {
|
|
19536
|
+
requireKey(key);
|
|
19537
|
+
detach(key);
|
|
19538
|
+
await settledWrite(
|
|
19539
|
+
key,
|
|
19540
|
+
() => connection.write(
|
|
19541
|
+
"strict",
|
|
19542
|
+
{ code: ErrorCodes.PERSISTENCE_DELETE_FAILED, action: `delete "${key}"` },
|
|
19543
|
+
(store) => removeRecords(store, key)
|
|
19544
|
+
),
|
|
19545
|
+
() => void 0
|
|
19546
|
+
);
|
|
19547
|
+
},
|
|
19548
|
+
async move(from, to) {
|
|
19549
|
+
requireKey(from);
|
|
19550
|
+
requireKey(to);
|
|
19551
|
+
if (from === to) return;
|
|
19552
|
+
detach(to);
|
|
19553
|
+
await moveRecords(from, to);
|
|
19554
|
+
},
|
|
19555
|
+
async list(prefix) {
|
|
19556
|
+
const keys = await connection.read(
|
|
19557
|
+
{ code: loadFailed2, action: `list "${prefix}"` },
|
|
19558
|
+
(store) => listKeys(store, prefix)
|
|
19559
|
+
);
|
|
19560
|
+
return keys.sort(compareCodePoints);
|
|
19561
|
+
},
|
|
19562
|
+
identity: () => connection.identity(),
|
|
19563
|
+
async lock() {
|
|
19564
|
+
const held = await lockIndexedDBStore(location);
|
|
19565
|
+
return {
|
|
19566
|
+
holds: () => held.holds(),
|
|
19567
|
+
async release() {
|
|
19568
|
+
connection.close();
|
|
19569
|
+
await held.release();
|
|
19570
|
+
}
|
|
19571
|
+
};
|
|
19572
|
+
}
|
|
19573
|
+
};
|
|
19574
|
+
}
|
|
19575
|
+
|
|
19576
|
+
// src/persistence/durability/snapshot-only/index-state.ts
|
|
19577
|
+
function createSnapshotIndexState() {
|
|
19578
|
+
return {
|
|
19579
|
+
mutationsSinceCheckpoint: 0,
|
|
19580
|
+
checkpointInFlight: null,
|
|
19581
|
+
applyChain: Promise.resolve(),
|
|
19582
|
+
unloading: false,
|
|
19583
|
+
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
19584
|
+
};
|
|
19585
|
+
}
|
|
19586
|
+
function metadataKey(indexName) {
|
|
19587
|
+
return `${indexName}/meta`;
|
|
19588
|
+
}
|
|
19589
|
+
function noteSnapshotCommitted(state, positions, partitionCount) {
|
|
19590
|
+
let recordsLeft = 0;
|
|
19591
|
+
for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
|
|
19592
|
+
if (partitionId >= partitionCount) {
|
|
19593
|
+
state.appliedSeqNoByPartition.delete(partitionId);
|
|
19594
|
+
continue;
|
|
19595
|
+
}
|
|
19596
|
+
recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
|
|
19597
|
+
}
|
|
19598
|
+
state.mutationsSinceCheckpoint = recordsLeft;
|
|
19599
|
+
}
|
|
19600
|
+
|
|
19601
|
+
// src/persistence/durability/snapshot-only/checkpoint.ts
|
|
19602
|
+
function capture2(input, prior, cutoffs) {
|
|
19603
|
+
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
19604
|
+
const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
|
|
19605
|
+
const targets = [];
|
|
19606
|
+
const contents = [];
|
|
19607
|
+
const positions = /* @__PURE__ */ new Map();
|
|
19608
|
+
for (let partitionId = 0; partitionId < input.manager.partitionCount; partitionId += 1) {
|
|
19609
|
+
const lastSeqNo = Math.min(
|
|
19610
|
+
input.appliedSeqNoByPartition.get(partitionId) ?? 0,
|
|
19611
|
+
cutoffs.get(partitionId) ?? Infinity
|
|
19612
|
+
);
|
|
19613
|
+
positions.set(partitionId, lastSeqNo);
|
|
19614
|
+
const unchanged = !input.rewritesEveryPartition && prior !== null && !partitionCountChanged && listedIds.has(partitionId) && checkpointLastSeqNo(prior.checkpoint, partitionId) === lastSeqNo;
|
|
19615
|
+
if (unchanged) continue;
|
|
19616
|
+
targets.push({ partitionId, lastSeqNo, primaryTerm: SINGLE_NODE_PRIMARY_TERM });
|
|
19617
|
+
contents.push(partitionContentsOf(input.manager, partitionId));
|
|
19618
|
+
}
|
|
19619
|
+
const plans = /* @__PURE__ */ new Map();
|
|
19620
|
+
try {
|
|
19621
|
+
for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
|
|
19622
|
+
const listed = prior?.vectors.find((ref) => ref.fieldPath === fieldPath);
|
|
19623
|
+
plans.set(
|
|
19624
|
+
fieldPath,
|
|
19625
|
+
vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key), false, true)
|
|
19626
|
+
);
|
|
19627
|
+
}
|
|
19628
|
+
return {
|
|
19629
|
+
targets,
|
|
19630
|
+
plans,
|
|
19631
|
+
documentCount: input.manager.countDocuments(),
|
|
19632
|
+
contents,
|
|
19633
|
+
positions,
|
|
19634
|
+
partitionCount: input.manager.partitionCount
|
|
19635
|
+
};
|
|
19636
|
+
} catch (error) {
|
|
19637
|
+
for (const plan of plans.values()) plan.release();
|
|
19638
|
+
throw error;
|
|
19639
|
+
}
|
|
19640
|
+
}
|
|
19641
|
+
async function writeCaptured(input, store, prior, captured) {
|
|
18516
19642
|
const priorWithinCount = prior === null ? null : manifestWithinPartitionCount(prior, captured.partitionCount);
|
|
18517
19643
|
const written = await writeCheckpointSegments({
|
|
18518
19644
|
store,
|
|
@@ -18533,7 +19659,8 @@ async function writeCaptured(input, store, prior, captured) {
|
|
|
18533
19659
|
await input.onCommitted({
|
|
18534
19660
|
saved: committed.saved,
|
|
18535
19661
|
documentCount: captured.documentCount,
|
|
18536
|
-
partitionCount: captured.partitionCount
|
|
19662
|
+
partitionCount: captured.partitionCount,
|
|
19663
|
+
positions: captured.positions
|
|
18537
19664
|
});
|
|
18538
19665
|
const garbage = await checkpointGarbage(store, input.indexName, committed.manifest);
|
|
18539
19666
|
await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, input.vectorIndexes, garbage));
|
|
@@ -18555,7 +19682,6 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
18555
19682
|
});
|
|
18556
19683
|
try {
|
|
18557
19684
|
await writeCaptured(input, store, prior, captured);
|
|
18558
|
-
return { positions: captured.positions, partitionCount: captured.partitionCount };
|
|
18559
19685
|
} finally {
|
|
18560
19686
|
for (const plan of captured.plans.values()) plan.release();
|
|
18561
19687
|
}
|
|
@@ -18649,6 +19775,41 @@ function createSnapshotSavedSize(adapter, settleRestore) {
|
|
|
18649
19775
|
};
|
|
18650
19776
|
}
|
|
18651
19777
|
|
|
19778
|
+
// src/persistence/durability/snapshot-only/shutdown-save.ts
|
|
19779
|
+
function asError(error) {
|
|
19780
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
19781
|
+
}
|
|
19782
|
+
async function saveIndex(deps, indexName, indexState) {
|
|
19783
|
+
await indexState.applyChain;
|
|
19784
|
+
await checkpointInTurn(indexState, {
|
|
19785
|
+
joinsInFlight: false,
|
|
19786
|
+
mayStart: () => !indexState.unloading && indexState.mutationsSinceCheckpoint > 0,
|
|
19787
|
+
perform: () => indexState.applyChain.then(() => deps.checkpoint(indexName, indexState)),
|
|
19788
|
+
afterSettling: () => void 0
|
|
19789
|
+
});
|
|
19790
|
+
}
|
|
19791
|
+
async function saveUnsavedIndexes(deps) {
|
|
19792
|
+
let failure = null;
|
|
19793
|
+
for (const [indexName, indexState] of deps.indexes) {
|
|
19794
|
+
try {
|
|
19795
|
+
await saveIndex(deps, indexName, indexState);
|
|
19796
|
+
} catch (error) {
|
|
19797
|
+
const failed = asError(error);
|
|
19798
|
+
deps.onFailure(failed);
|
|
19799
|
+
failure ??= failed;
|
|
19800
|
+
}
|
|
19801
|
+
}
|
|
19802
|
+
if (failure !== null) throw failure;
|
|
19803
|
+
}
|
|
19804
|
+
async function saveAndReleaseIndexes(deps) {
|
|
19805
|
+
try {
|
|
19806
|
+
await saveUnsavedIndexes(deps);
|
|
19807
|
+
} finally {
|
|
19808
|
+
await deps.settleMetadataWrites();
|
|
19809
|
+
deps.release();
|
|
19810
|
+
}
|
|
19811
|
+
}
|
|
19812
|
+
|
|
18652
19813
|
// src/persistence/durability/snapshot-only/index.ts
|
|
18653
19814
|
function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
18654
19815
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
@@ -18698,7 +19859,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
18698
19859
|
const manager = hooks.getManager(indexName);
|
|
18699
19860
|
const metadata = hooks.buildTermsMetadata(indexName);
|
|
18700
19861
|
if (manager === void 0 || metadata === void 0) return;
|
|
18701
|
-
|
|
19862
|
+
await writeSnapshotOnlyCheckpoint({
|
|
18702
19863
|
adapter,
|
|
18703
19864
|
indexName,
|
|
18704
19865
|
metadata,
|
|
@@ -18707,12 +19868,12 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
18707
19868
|
appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
|
|
18708
19869
|
rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName),
|
|
18709
19870
|
onCommitted: async (committed) => {
|
|
19871
|
+
noteSnapshotCommitted(indexState, committed.positions, committed.partitionCount);
|
|
18710
19872
|
savedSize.record(indexName, committed.saved);
|
|
18711
19873
|
hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
|
|
18712
19874
|
await queueMetadataWrite(indexName, () => saveCheckpointMetadata(adapter, hooks, indexName, committed));
|
|
18713
19875
|
}
|
|
18714
19876
|
});
|
|
18715
|
-
noteSnapshotCommitted(indexState, positions, partitionCount);
|
|
18716
19877
|
if (publisher !== void 0) {
|
|
18717
19878
|
const partitionIds = [];
|
|
18718
19879
|
for (let i = 0; i < manager.partitionCount; i += 1) {
|
|
@@ -18803,11 +19964,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
18803
19964
|
}
|
|
18804
19965
|
}
|
|
18805
19966
|
async function recordMutation(record) {
|
|
19967
|
+
if (shuttingDown) throw new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
18806
19968
|
const indexState = getOrCreateIndexState(record.indexName);
|
|
18807
19969
|
const buffered = indexState.applyChain.then(async () => {
|
|
18808
19970
|
await record.apply();
|
|
18809
19971
|
const appliedSeqNo2 = (indexState.appliedSeqNoByPartition.get(record.partitionId) ?? 0) + 1;
|
|
18810
19972
|
indexState.appliedSeqNoByPartition.set(record.partitionId, appliedSeqNo2);
|
|
19973
|
+
indexState.mutationsSinceCheckpoint += 1;
|
|
18811
19974
|
return appliedSeqNo2;
|
|
18812
19975
|
});
|
|
18813
19976
|
indexState.applyChain = buffered.then(
|
|
@@ -18815,7 +19978,6 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
18815
19978
|
() => void 0
|
|
18816
19979
|
);
|
|
18817
19980
|
const appliedSeqNo = await buffered;
|
|
18818
|
-
indexState.mutationsSinceCheckpoint += 1;
|
|
18819
19981
|
startCheckpointTimer();
|
|
18820
19982
|
if (indexState.mutationsSinceCheckpoint >= checkpointMutationThreshold) {
|
|
18821
19983
|
void checkpointIndex(record.indexName).catch((err) => {
|
|
@@ -18950,12 +20112,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
18950
20112
|
checkpointTimer = null;
|
|
18951
20113
|
}
|
|
18952
20114
|
await restores.settleAll();
|
|
18953
|
-
|
|
18954
|
-
|
|
18955
|
-
|
|
18956
|
-
|
|
18957
|
-
|
|
18958
|
-
|
|
20115
|
+
await saveAndReleaseIndexes({
|
|
20116
|
+
indexes,
|
|
20117
|
+
checkpoint: (indexName, indexState) => performCheckpoint(indexName, indexState, false),
|
|
20118
|
+
onFailure: hooks.onFatalError,
|
|
20119
|
+
settleMetadataWrites: () => metadataWrites.settleAll(),
|
|
20120
|
+
release: () => indexes.clear()
|
|
20121
|
+
});
|
|
18959
20122
|
}
|
|
18960
20123
|
};
|
|
18961
20124
|
}
|
|
@@ -19002,7 +20165,11 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
19002
20165
|
function readerOf(indexName) {
|
|
19003
20166
|
return hooks.previousDocumentReader === void 0 ? null : hooks.previousDocumentReader(indexName);
|
|
19004
20167
|
}
|
|
19005
|
-
const manager = tier.kind === "wal" ? createDurabilityManager(
|
|
20168
|
+
const manager = tier.kind === "wal" ? createDurabilityManager(
|
|
20169
|
+
tier.config,
|
|
20170
|
+
managerHooks,
|
|
20171
|
+
tier.indexedDB === void 0 ? void 0 : createIndexedDBDirectory(tier.indexedDB, { relaxedAppends: tier.config.mode === "async" })
|
|
20172
|
+
) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
19006
20173
|
async function recordMutation(indexName, docId, operation, document, apply) {
|
|
19007
20174
|
return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
|
|
19008
20175
|
const partitionId = fnv1a(docId) % partitionCount;
|