@delali/narsil 0.4.1-next.215 → 0.4.2-next.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-DPRCAVC2.mjs} +14 -8
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-WRRXR57N.mjs} +1 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-XBWABEL3.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-Y5CSZDOH.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-FEPCLXXY.mjs} +7 -1
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-HVPPDHSY.mjs} +259 -104
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-DBC4EJWB.mjs → chunk-33ZMFOAR.mjs} +1 -1
- package/dist/{chunk-NCIGNQJE.mjs → chunk-4GCHJ3XD.mjs} +930 -387
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-4WQNWAQE.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-5E3TJ4P3.mjs} +5 -5
- package/dist/{chunk-GXCDWLQI.mjs → chunk-5GYGHPCH.mjs} +7 -1
- package/dist/{chunk-SMQPZHPS.mjs → chunk-ATQ6JR3I.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-BBIAUHAB.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-DA32WXBT.mjs} +2 -2
- package/dist/{chunk-MSJD74GN.mjs → chunk-EALKCTWH.mjs} +5 -5
- package/dist/{chunk-755NM4PL.mjs → chunk-FOGTBAYN.mjs} +183 -21
- package/dist/{chunk-6GHX35ZM.mjs → chunk-GTBJSNN2.mjs} +1 -1
- package/dist/{chunk-DDUXEJXW.mjs → chunk-H6LWGMPQ.mjs} +45 -22
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-IL5DRCNU.mjs → chunk-KQWTS7AM.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-KX3OP6VP.mjs} +2 -2
- package/dist/{chunk-D456ZFPQ.mjs → chunk-MIIJ5O3O.mjs} +2 -2
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-MS746XNY.mjs} +285 -118
- package/dist/{chunk-WM5KJSQV.mjs → chunk-NFA3WHJU.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-OETLM4EH.mjs} +1 -1
- package/dist/{chunk-MYN4K5IS.mjs → chunk-OLXFIUIZ.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-X3EQCQLP.mjs → chunk-Q4NLL66U.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-QJFKL4NK.mjs} +4 -4
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-3IE3A74H.mjs → chunk-SSITKQU6.mjs} +3 -3
- package/dist/{chunk-B3MO37YF.mjs → chunk-TANKI7TG.mjs} +1 -1
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-TI4G3SIG.mjs} +63 -144
- package/dist/{chunk-OGYSPNL2.mjs → chunk-TT3UDDL3.mjs} +7 -7
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-VHIOGDVD.mjs} +1 -1
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-W3JH2IK6.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-WIT6B5OB.mjs} +2 -2
- package/dist/{chunk-MI6UZGUR.mjs → chunk-XGVSG6ZU.mjs} +3 -3
- package/dist/{chunk-6I6OO6B2.mjs → chunk-XWPNRHTG.mjs} +4 -4
- package/dist/{chunk-WTMCGZ74.mjs → chunk-YHZSSPG3.mjs} +2 -2
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-PCBB6FCO.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-4ALYIIC4.mjs} +14 -14
- package/dist/msgpack-leaf-6ZBZWFRD.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-2OK4TOLY.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-Y5OAKNRB.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-UQVQZFIJ.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-4OOUY2K7.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-5MYHYBET.mjs} +286 -118
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AVCMTWET.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-GS6LGXY3.mjs} +1 -1
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-LE4DTMDL.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-VIO5IPF7.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-VRI5XH2M.mjs} +7 -1
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-XGS2UYVI.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-XW6GZQWP.mjs} +4 -4
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-Z44S6WW5.mjs} +2 -2
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-4MIGXWZT.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-LZDHB7FX.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-J7SB263E.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PEKX7TBL.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-I2EC7H6R.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-RQXM7HBO.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-EZGCLSLM.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-JNGH2JYC.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-2OWLLBCV.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-TQ4SU76L.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-W7U2X6V3.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-WGP4FBXF.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-FYWJQSPL.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-GRG44RQA.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-YDJVH73G.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-5UFJV7SC.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-6YLVL2NC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-IP4L7KEW.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -10
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
|
@@ -867,7 +867,7 @@ var loaded = null;
|
|
|
867
867
|
var loading = null;
|
|
868
868
|
function loadMessagePack() {
|
|
869
869
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
870
|
-
loading ??= import('./msgpack-leaf-
|
|
870
|
+
loading ??= import('./msgpack-leaf-6ZBZWFRD.mjs').then(
|
|
871
871
|
(code) => {
|
|
872
872
|
code.bindMsgpackHost({
|
|
873
873
|
BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER,
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { encode, Merger } from './chunk-
|
|
2
|
-
import { DayStore, RestoredHistory, dayOf, sameBytes } from './chunk-
|
|
1
|
+
import { encode, Merger } from './chunk-YHZSSPG3.mjs';
|
|
2
|
+
import { DayStore, RestoredHistory, dayOf, sameBytes } from './chunk-KX3OP6VP.mjs';
|
|
3
3
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
4
|
-
import { isGroup, invalidSetting, historyFilesOf, buildRecordKey, MIN_KEY_BYTES, resolveSetting, isPerson, acceptedPhrase, removeHistoryFolder, normaliseBan, banListFromInput, validateSetting, withState, packEnvelopeWithChecksum, crc32, openChecksummedEnvelope, SCRAMBLED_ID_BYTES, KEY_FINGERPRINT_LABEL, KEY_FINGERPRINT_BYTES, removeRestoreLeftovers, HISTORY_SWEEP_INTERVAL_MS, SCHEDULE_OFF, parseSchedule, nextMatchAfter, latestMatchAtOrBefore, DAY_MS, listHistoryFolder, historyFileOf, MAX_MANUAL_SUGGESTION_COUNT, MAX_SOURCE_INDEX_NAME_LENGTH, DEFAULT_PREVIEW_LIMIT, MAX_PREVIEW_LIMIT, requireFieldsInSchema, MAX_TIMER_DELAY_MS, BUILD_WRITE_BATCH, BYTES_PER_MIB, MS_PER_SECOND, BUILD_LIST_PAGE, unionOfBans } from './chunk-
|
|
5
|
-
import { GROUP_FORMAT, emitEngineEvent, emitWriteEventAfter, createEngineIndex, withOpenIndex, loadDigest, SUGGESTIONS_INDEX_INFIX } from './chunk-
|
|
6
|
-
import { decode } from './chunk-
|
|
4
|
+
import { isGroup, invalidSetting, historyFilesOf, buildRecordKey, MIN_KEY_BYTES, resolveSetting, isPerson, acceptedPhrase, removeHistoryFolder, normaliseBan, banListFromInput, validateSetting, withState, packEnvelopeWithChecksum, crc32, openChecksummedEnvelope, SCRAMBLED_ID_BYTES, KEY_FINGERPRINT_LABEL, KEY_FINGERPRINT_BYTES, removeRestoreLeftovers, HISTORY_SWEEP_INTERVAL_MS, SCHEDULE_OFF, parseSchedule, nextMatchAfter, latestMatchAtOrBefore, DAY_MS, listHistoryFolder, historyFileOf, MAX_MANUAL_SUGGESTION_COUNT, MAX_SOURCE_INDEX_NAME_LENGTH, DEFAULT_PREVIEW_LIMIT, MAX_PREVIEW_LIMIT, requireFieldsInSchema, MAX_TIMER_DELAY_MS, BUILD_WRITE_BATCH, BYTES_PER_MIB, MS_PER_SECOND, BUILD_LIST_PAGE, unionOfBans } from './chunk-WIT6B5OB.mjs';
|
|
5
|
+
import { GROUP_FORMAT, emitEngineEvent, emitWriteEventAfter, createEngineIndex, withOpenIndex, loadDigest, SUGGESTIONS_INDEX_INFIX } from './chunk-MS746XNY.mjs';
|
|
6
|
+
import { decode } from './chunk-5GYGHPCH.mjs';
|
|
7
7
|
import { spawnNodeWorker, resolveWorkerEntry } from './chunk-FJXJZUWF.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
9
9
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
@@ -186,8 +186,14 @@ function decode(bytes) {
|
|
|
186
186
|
sharedDecoder ??= new Decoder();
|
|
187
187
|
return sharedDecoder.decode(bytes);
|
|
188
188
|
}
|
|
189
|
+
function decodeBoundedMessage(bytes) {
|
|
190
|
+
if (scanStrings(bytes, 0, null, Number.POSITIVE_INFINITY) === null) {
|
|
191
|
+
throw new RangeError("Invalid MessagePack value header");
|
|
192
|
+
}
|
|
193
|
+
return decode(bytes);
|
|
194
|
+
}
|
|
189
195
|
function decodeMulti(bytes) {
|
|
190
196
|
return new Decoder().decodeMulti(bytes);
|
|
191
197
|
}
|
|
192
198
|
|
|
193
|
-
export { Decoder, MAP_TAGS, bindMsgpackHost, decode, decodeMulti, headerAt };
|
|
199
|
+
export { Decoder, MAP_TAGS, bindMsgpackHost, decode, decodeBoundedMessage, decodeMulti, headerAt };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { executeUntilAnswered } from './chunk-4H4RG2FR.mjs';
|
|
2
|
-
import { isCompositePartition, encodeSegmentState, freezeSegmentShared } from './chunk-
|
|
2
|
+
import { isCompositePartition, encodeSegmentState, freezeSegmentShared } from './chunk-MS746XNY.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
4
4
|
|
|
5
5
|
// src/core/partition/frozen/share.ts
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { restoredEmbedding, restoredPartitionConfig, readSharedConfigFields } from './chunk-M2Z5WVWW.mjs';
|
|
2
2
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-JDDXV6UF.mjs';
|
|
3
|
-
import { storedStateOf } from './chunk-
|
|
4
|
-
import { getLanguage, generateId, decodeSynonymRules } from './chunk-
|
|
5
|
-
import { decode } from './chunk-
|
|
3
|
+
import { storedStateOf } from './chunk-WIT6B5OB.mjs';
|
|
4
|
+
import { getLanguage, generateId, decodeSynonymRules } from './chunk-MS746XNY.mjs';
|
|
5
|
+
import { decode } from './chunk-5GYGHPCH.mjs';
|
|
6
6
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
7
7
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-3ECIRFS5.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-
|
|
2
|
-
import { getFlatSchema, schemaFieldsOf, getNestedValue } from './chunk-
|
|
1
|
+
import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-MS746XNY.mjs';
|
|
2
|
+
import { getFlatSchema, schemaFieldsOf, getNestedValue } from './chunk-4WQNWAQE.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/partition/pattern-snapshot.ts
|
|
5
5
|
function serializePatternIndexes(state, schema) {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION } from './chunk-
|
|
1
|
+
import { SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION } from './chunk-TANKI7TG.mjs';
|
|
2
2
|
import { encode } from './chunk-S5QMWECL.mjs';
|
|
3
3
|
import { createFixedBuffer, createVectorFileReader } from './chunk-R724DJSG.mjs';
|
|
4
|
-
import { crc32, packChecksummedEnvelope, HEADER_SIZE } from './chunk-
|
|
5
|
-
import { decode } from './chunk-
|
|
4
|
+
import { crc32, packChecksummedEnvelope, HEADER_SIZE } from './chunk-WIT6B5OB.mjs';
|
|
5
|
+
import { decode } from './chunk-5GYGHPCH.mjs';
|
|
6
6
|
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-ASZLNVWW.mjs';
|
|
7
7
|
import { ErrorCodes, NarsilError } from './chunk-ZKOQR7EA.mjs';
|
|
8
8
|
|
|
@@ -13,11 +13,11 @@ function directoryCheckpointStore(directory) {
|
|
|
13
13
|
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
14
14
|
remove: (key) => directory.remove(key),
|
|
15
15
|
list: (prefix) => directory.list(prefix),
|
|
16
|
-
pathOf: (key) => directory.pathOf(key),
|
|
17
16
|
directory
|
|
18
17
|
};
|
|
19
|
-
const byteLength = directory
|
|
18
|
+
const { byteLength, pathOf } = directory;
|
|
20
19
|
if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
|
|
20
|
+
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(directory, key);
|
|
21
21
|
return store;
|
|
22
22
|
}
|
|
23
23
|
function joined(chunks) {
|
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
import { loadVectorField, openColdVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-
|
|
2
|
-
import { engineChangeSource, ChangeCursor, createSemaphore, changePositionList, changesNotEnabled, previousDocumentRequired, invalidPositions, changePositionsOf, requireConfigurationShape, RESTORE_ESTIMATE_MARGIN, persistEpochStates, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-
|
|
1
|
+
import { loadVectorField, openColdVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-QJFKL4NK.mjs';
|
|
2
|
+
import { engineChangeSource, ChangeCursor, createSemaphore, changePositionList, changesNotEnabled, previousDocumentRequired, invalidPositions, changePositionsOf, requireConfigurationShape, RESTORE_ESTIMATE_MARGIN, persistEpochStates, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-SSITKQU6.mjs';
|
|
3
3
|
import { restoredConfigFields, restoredEmbedding } from './chunk-M2Z5WVWW.mjs';
|
|
4
|
-
import { ENTRIES_MESSAGE_LIMIT_BYTES, FEED_HEAD_RETRY_MS, decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-
|
|
5
|
-
import { isPagedSegmentFile, openPagedSegmentScan, countsAsLatestHolder, writeCheckpointSegments, commitCheckpointManifest, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, removeUnusedSegmentFiles, readWalSegments, durableRecordMissing, checkpointLastSeqNo, pagedListingOf, residentSectionsOf, partitionContentsOf, checkPagedSegmentFully, openPagedSnapshot, createPagedFrozenSegment, SEGMENT_HEADER_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
6
|
-
import { decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-
|
|
7
|
-
import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-
|
|
8
|
-
import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-MSIN4NM3.mjs';
|
|
4
|
+
import { ENTRIES_MESSAGE_LIMIT_BYTES, FEED_HEAD_RETRY_MS, decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-XWPNRHTG.mjs';
|
|
5
|
+
import { isPagedSegmentFile, openPagedSegmentScan, countsAsLatestHolder, writeCheckpointSegments, commitCheckpointManifest, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, removeUnusedSegmentFiles, readWalSegments, durableRecordMissing, checkpointLastSeqNo, pagedListingOf, residentSectionsOf, filePathOf, partitionContentsOf, checkPagedSegmentFully, openPagedSnapshot, createPagedFrozenSegment, SEGMENT_HEADER_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-H6LWGMPQ.mjs';
|
|
6
|
+
import { decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-EALKCTWH.mjs';
|
|
7
|
+
import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-TANKI7TG.mjs';
|
|
9
8
|
import { encode } from './chunk-S5QMWECL.mjs';
|
|
10
|
-
import { setNestedValue, restoreVectorFields } from './chunk-
|
|
11
|
-
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-
|
|
9
|
+
import { setNestedValue, restoreVectorFields } from './chunk-OETLM4EH.mjs';
|
|
10
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-VHIOGDVD.mjs';
|
|
12
11
|
import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-R724DJSG.mjs';
|
|
13
12
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
14
|
-
import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-
|
|
15
|
-
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, noteDocumentChurn, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readProcessMemoryUse, prepareConfiguredSynonyms, withOpenIndex, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, decodeStoredDocument, emitSyncChange, ARRAY_POSITION_GAP } from './chunk-
|
|
16
|
-
import { decode } from './chunk-
|
|
13
|
+
import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-WIT6B5OB.mjs';
|
|
14
|
+
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, noteDocumentChurn, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readProcessMemoryUse, prepareConfiguredSynonyms, withOpenIndex, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, decodeStoredDocument, emitSyncChange, ARRAY_POSITION_GAP } from './chunk-MS746XNY.mjs';
|
|
15
|
+
import { decode } from './chunk-5GYGHPCH.mjs';
|
|
17
16
|
import { isSyncEpoch, newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
18
|
-
import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, parseVectorDimension, requireValidFieldTypes, isQuantizationMode, isStorageMode, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-
|
|
19
|
-
import {
|
|
17
|
+
import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, parseVectorDimension, requireValidFieldTypes, isQuantizationMode, isStorageMode, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-4WQNWAQE.mjs';
|
|
18
|
+
import { turnsAreBudgeted, turnIsSpent, yieldTurn, collectAndReturnPages, countWrite } from './chunk-5EI5CXO6.mjs';
|
|
20
19
|
import { DEFAULT_MAX_PARTITIONS, ANALYSIS_CHANGE_MEMBERS, EXPORT_CHUNK_BYTES, MAX_PARTITIONS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-3ECIRFS5.mjs';
|
|
20
|
+
import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-S3JHYPH7.mjs';
|
|
21
21
|
import { VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
|
|
22
22
|
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
23
23
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
@@ -326,6 +326,159 @@ async function checkpointInTurn(turns, turn) {
|
|
|
326
326
|
return run;
|
|
327
327
|
}
|
|
328
328
|
|
|
329
|
+
// src/persistence/durability/mutation-batch.ts
|
|
330
|
+
function shutDownRefusal() {
|
|
331
|
+
return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
332
|
+
}
|
|
333
|
+
function toError(err) {
|
|
334
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
335
|
+
}
|
|
336
|
+
function requireAscendingSeqNo(record, seqNo, previousSeqNo) {
|
|
337
|
+
if (seqNo > previousSeqNo) return;
|
|
338
|
+
throw new NarsilError(
|
|
339
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
340
|
+
`Log record ${seqNo} for partition ${record.partitionId} of index "${record.indexName}" does not follow record ${previousSeqNo}`,
|
|
341
|
+
{ indexName: record.indexName, partitionId: record.partitionId, seqNo, previousSeqNo }
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
345
|
+
for (let i = 0; i < count; i++) {
|
|
346
|
+
const outcome = outcomes[i];
|
|
347
|
+
if (outcome === void 0 || outcome.ok) {
|
|
348
|
+
outcomes[i] = { ok: false, error };
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
function failPartition(partition, address, deps, err) {
|
|
353
|
+
const error = toError(err);
|
|
354
|
+
if (partition.failed !== null) return partition.failed;
|
|
355
|
+
partition.failed = error;
|
|
356
|
+
if (!deps.failsPartitionAlone(address.indexName, address.partitionId, error)) deps.markFatal(error);
|
|
357
|
+
return error;
|
|
358
|
+
}
|
|
359
|
+
async function commitEveryPartition(indexes, deps) {
|
|
360
|
+
for (const [indexName, indexState] of indexes) {
|
|
361
|
+
for (const [partitionId, partition] of indexState.partitions) {
|
|
362
|
+
if (partition.failed !== null || deps.fatalError() !== null) continue;
|
|
363
|
+
try {
|
|
364
|
+
await partition.walWriter.commit();
|
|
365
|
+
} catch (err) {
|
|
366
|
+
failPartition(partition, { indexName, partitionId }, deps, err);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
function partitionFailureFor(indexName, deps) {
|
|
372
|
+
return (partitionId, partition, error) => failPartition(partition, { indexName, partitionId }, deps, error);
|
|
373
|
+
}
|
|
374
|
+
async function recordBatchesByPartition(records, partitionOf, deps) {
|
|
375
|
+
const positionsByPartition = /* @__PURE__ */ new Map();
|
|
376
|
+
for (let i = 0; i < records.length; i++) {
|
|
377
|
+
const partition = partitionOf(records[i]);
|
|
378
|
+
const positions = positionsByPartition.get(partition);
|
|
379
|
+
if (positions === void 0) {
|
|
380
|
+
positionsByPartition.set(partition, [i]);
|
|
381
|
+
} else {
|
|
382
|
+
positions.push(i);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
const outcomes = new Array(records.length);
|
|
386
|
+
await Promise.all(
|
|
387
|
+
[...positionsByPartition].map(async ([partition, positions]) => {
|
|
388
|
+
const partitionOutcomes = await recordPartitionBatch(
|
|
389
|
+
partition,
|
|
390
|
+
positions.map((position) => records[position]),
|
|
391
|
+
deps
|
|
392
|
+
);
|
|
393
|
+
for (let i = 0; i < positions.length; i++) {
|
|
394
|
+
outcomes[positions[i]] = partitionOutcomes[i];
|
|
395
|
+
}
|
|
396
|
+
})
|
|
397
|
+
);
|
|
398
|
+
return outcomes;
|
|
399
|
+
}
|
|
400
|
+
async function recordPartitionBatch(partition, records, deps) {
|
|
401
|
+
const outcomes = new Array(records.length);
|
|
402
|
+
let appendedCount = 0;
|
|
403
|
+
let commitsHere = false;
|
|
404
|
+
const appended = partition.appendChain.then(async () => {
|
|
405
|
+
const fatal = deps.fatalError();
|
|
406
|
+
if (fatal !== null) {
|
|
407
|
+
throw fatal;
|
|
408
|
+
}
|
|
409
|
+
if (partition.failed !== null) {
|
|
410
|
+
throw partition.failed;
|
|
411
|
+
}
|
|
412
|
+
const applied = [];
|
|
413
|
+
for (let i = 0; i < records.length; i++) {
|
|
414
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
415
|
+
try {
|
|
416
|
+
await records[i].apply();
|
|
417
|
+
applied.push(i);
|
|
418
|
+
} catch (err) {
|
|
419
|
+
outcomes[i] = { ok: false, error: err };
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
if (applied.length === 0) {
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
425
|
+
let lastSeqNo = partition.appliedSeqNo;
|
|
426
|
+
const entries = [];
|
|
427
|
+
try {
|
|
428
|
+
for (const i of applied) {
|
|
429
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
430
|
+
let numbered;
|
|
431
|
+
try {
|
|
432
|
+
numbered = deps.numberEntry(records[i]);
|
|
433
|
+
} catch (err) {
|
|
434
|
+
outcomes[i] = { ok: false, error: err };
|
|
435
|
+
continue;
|
|
436
|
+
}
|
|
437
|
+
if (numbered === null) {
|
|
438
|
+
lastSeqNo = partition.seqOwner.next();
|
|
439
|
+
entries.push(deps.buildEntry(records[i], lastSeqNo));
|
|
440
|
+
} else {
|
|
441
|
+
requireAscendingSeqNo(records[i], numbered.seqNo, lastSeqNo);
|
|
442
|
+
partition.seqOwner.advanceTo(numbered.seqNo, numbered.primaryTerm);
|
|
443
|
+
lastSeqNo = numbered.seqNo;
|
|
444
|
+
entries.push(numbered);
|
|
445
|
+
}
|
|
446
|
+
outcomes[i] = { ok: true, seqNo: lastSeqNo };
|
|
447
|
+
if (!deps.defersCommit(records[i])) commitsHere = true;
|
|
448
|
+
}
|
|
449
|
+
if (entries.length > 0) await partition.walWriter.appendAll(entries);
|
|
450
|
+
} catch (err) {
|
|
451
|
+
throw failPartition(partition, records[0], deps, err);
|
|
452
|
+
}
|
|
453
|
+
partition.appliedSeqNo = lastSeqNo;
|
|
454
|
+
appendedCount = entries.length;
|
|
455
|
+
});
|
|
456
|
+
partition.appendChain = appended.catch(() => void 0);
|
|
457
|
+
try {
|
|
458
|
+
await appended;
|
|
459
|
+
if (deps.syncEachBatch && appendedCount > 0 && commitsHere) {
|
|
460
|
+
try {
|
|
461
|
+
await partition.walWriter.commit();
|
|
462
|
+
} catch (err) {
|
|
463
|
+
throw failPartition(partition, records[0], deps, err);
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
} catch (err) {
|
|
467
|
+
failUnlessAlreadyFailed(outcomes, records.length, toError(err));
|
|
468
|
+
}
|
|
469
|
+
return outcomes;
|
|
470
|
+
}
|
|
471
|
+
async function commitAppended(partition, address, deps) {
|
|
472
|
+
await partition.appendChain;
|
|
473
|
+
const failed = partition.failed ?? deps.fatalError();
|
|
474
|
+
if (failed !== null) throw failed;
|
|
475
|
+
try {
|
|
476
|
+
await partition.walWriter.commit();
|
|
477
|
+
} catch (err) {
|
|
478
|
+
throw failPartition(partition, address, deps, err);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
329
482
|
// src/persistence/durability/seq-owner.ts
|
|
330
483
|
var SINGLE_NODE_PRIMARY_TERM = 1;
|
|
331
484
|
function createSeqOwner(startSeqNo, primaryTerm = SINGLE_NODE_PRIMARY_TERM) {
|
|
@@ -407,7 +560,8 @@ async function writeTransition(store, indexName, transition) {
|
|
|
407
560
|
|
|
408
561
|
// src/persistence/durability/layout-transition/index.ts
|
|
409
562
|
async function transitionWalLayout(registry, indexName, manager, replay) {
|
|
410
|
-
const state = registry.
|
|
563
|
+
const state = registry.indexes.get(indexName);
|
|
564
|
+
if (state === void 0) throw shutDownRefusal();
|
|
411
565
|
await switchDurableRouting(
|
|
412
566
|
manager,
|
|
413
567
|
(oldPartitionCount) => checkpointInTurn(state, {
|
|
@@ -1318,7 +1472,7 @@ function termBytesOf(core, indexName) {
|
|
|
1318
1472
|
}
|
|
1319
1473
|
return Math.max(0, bytes);
|
|
1320
1474
|
}
|
|
1321
|
-
function
|
|
1475
|
+
function toError2(value) {
|
|
1322
1476
|
return value instanceof Error ? value : new Error(String(value));
|
|
1323
1477
|
}
|
|
1324
1478
|
async function requireNoUndurableSwitch(core, indexName, entry) {
|
|
@@ -1417,7 +1571,7 @@ async function buildAndSwitch(scope) {
|
|
|
1417
1571
|
}
|
|
1418
1572
|
emit("completed", partitionsRebuilt);
|
|
1419
1573
|
} catch (error) {
|
|
1420
|
-
emit("failed", partitionsRebuilt,
|
|
1574
|
+
emit("failed", partitionsRebuilt, toError2(error));
|
|
1421
1575
|
throw error;
|
|
1422
1576
|
} finally {
|
|
1423
1577
|
if (core.executor.getManager(indexName) === manager) {
|
|
@@ -1538,7 +1692,7 @@ function resumeAnalysisChange(core, indexName) {
|
|
|
1538
1692
|
status: "failed",
|
|
1539
1693
|
partitionsRebuilt: 0,
|
|
1540
1694
|
partitionCount: entry.partitionCount,
|
|
1541
|
-
error:
|
|
1695
|
+
error: toError2(error)
|
|
1542
1696
|
});
|
|
1543
1697
|
return Promise.reject(error);
|
|
1544
1698
|
}
|
|
@@ -2223,7 +2377,10 @@ async function coldFieldOf(checkpoint, field) {
|
|
|
2223
2377
|
const fieldType = checkpoint.manifest.schema[field.fieldPath];
|
|
2224
2378
|
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
2225
2379
|
if (dimension === null) return null;
|
|
2226
|
-
const withPath = async (ref) => ({
|
|
2380
|
+
const withPath = async (ref) => ({
|
|
2381
|
+
ref,
|
|
2382
|
+
path: await filePathOf(directory, ref.key)
|
|
2383
|
+
});
|
|
2227
2384
|
return openColdVectorField({
|
|
2228
2385
|
fieldPath: field.fieldPath,
|
|
2229
2386
|
dimension,
|
|
@@ -2504,7 +2661,7 @@ async function headOf(path, ref) {
|
|
|
2504
2661
|
return decode(head);
|
|
2505
2662
|
}
|
|
2506
2663
|
async function levelsOfPart(directory, ref) {
|
|
2507
|
-
const path = await directory
|
|
2664
|
+
const path = await filePathOf(directory, ref.key);
|
|
2508
2665
|
const head = await headOf(path, ref);
|
|
2509
2666
|
const graphs = head?.graphs;
|
|
2510
2667
|
if (!Array.isArray(graphs) || graphs.length !== 1) return [];
|
|
@@ -4416,6 +4573,11 @@ async function createMarkerWriter(directory, indexName, partitionId) {
|
|
|
4416
4573
|
// src/persistence/durability/recovery.ts
|
|
4417
4574
|
async function removeLeftoverRewriteFiles(directory, names) {
|
|
4418
4575
|
for (const indexName of names) {
|
|
4576
|
+
const prefix = `${indexName}/`;
|
|
4577
|
+
for (const key of await directory.list(prefix)) {
|
|
4578
|
+
const relative = key.slice(prefix.length);
|
|
4579
|
+
if (!relative.includes("/") && /^(meta|manifest)\.tmp-/.test(relative)) await directory.remove(key);
|
|
4580
|
+
}
|
|
4419
4581
|
for (const place of ["rebuilt", "received"]) {
|
|
4420
4582
|
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
4421
4583
|
}
|
|
@@ -4593,4 +4755,4 @@ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeW
|
|
|
4593
4755
|
}
|
|
4594
4756
|
}
|
|
4595
4757
|
|
|
4596
|
-
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, captureAppliedPrefix, changeIndexAnalysis, changesOf, changesStatusOf, checkpointInTurn, closePartitionWriter, configOfNamedSettings, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, documentExportStream, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, followIndexChanges, followSourceChanges, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, openFileVectorReads, pagedPartitionLines, patternIndexesFromWire, readCurrentManifest, readDocumentExport, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireChangesSetting, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, updateChangesOf, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
|
|
4758
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, captureAppliedPrefix, changeIndexAnalysis, changesOf, changesStatusOf, checkpointInTurn, closePartitionWriter, commitAppended, commitEveryPartition, configOfNamedSettings, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, documentExportStream, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, followIndexChanges, followSourceChanges, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, openFileVectorReads, pagedPartitionLines, partitionFailureFor, patternIndexesFromWire, readCurrentManifest, readDocumentExport, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, recordBatchesByPartition, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireChangesSetting, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, shutDownRefusal, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, updateChangesOf, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
2
|
-
import { requireLoadedMessagePack } from './chunk-
|
|
2
|
+
import { requireLoadedMessagePack } from './chunk-4WQNWAQE.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
4
4
|
|
|
5
5
|
// src/sync/device/stores/records.ts
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { PAGED_SEGMENT_ENVELOPE_VERSION, loadFailed, checkEverySectionPage, openPagedFile, sectionNamed, createSectionReader, closePagedFile, readWholeSectionChecked, readCacheOfThisThread, createReadCache, MERGE_READ_CACHE_BYTES, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, requireNativePage, ID_SAMPLE_INTERVAL, SMALLEST_BLOCK_POSTINGS, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, resumePagedFile, suspendPagedFile, cachedFileBytes, LIST_VIEWS_HELD_PER_SEGMENT, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, CACHED_POSITIONS, POSITION_WIDTH_BYTES, CACHED_LIST, POSTING_WIDTH_BYTES, readFileRangeLater, CACHED_DOCUMENT_BLOCK, failNativePage, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, POSTING_WRITE_ROWS, POSTING_WRITE_POSITIONS, SECTION_WRITE_BUFFER_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-
|
|
1
|
+
import { PAGED_SEGMENT_ENVELOPE_VERSION, loadFailed, checkEverySectionPage, openPagedFile, sectionNamed, createSectionReader, closePagedFile, readWholeSectionChecked, readCacheOfThisThread, createReadCache, MERGE_READ_CACHE_BYTES, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, requireNativePage, ID_SAMPLE_INTERVAL, SMALLEST_BLOCK_POSTINGS, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, resumePagedFile, suspendPagedFile, cachedFileBytes, LIST_VIEWS_HELD_PER_SEGMENT, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, CACHED_POSITIONS, POSITION_WIDTH_BYTES, CACHED_LIST, POSTING_WIDTH_BYTES, readFileRangeLater, CACHED_DOCUMENT_BLOCK, failNativePage, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, POSTING_WRITE_ROWS, POSTING_WRITE_POSITIONS, SECTION_WRITE_BUFFER_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-EALKCTWH.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-5FLRXF73.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
4
|
-
import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-
|
|
5
|
-
import { DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_COMPACTION_THRESHOLD, SEGMENT_FILE_FRAMING_SLACK_BYTES, REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-MSIN4NM3.mjs';
|
|
4
|
+
import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-TANKI7TG.mjs';
|
|
6
5
|
import { encode } from './chunk-S5QMWECL.mjs';
|
|
7
|
-
import { restoreVectorFields } from './chunk-
|
|
8
|
-
import { requireWithinFileLimit, HEADER_SIZE, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, readHeader, packChecksummedEnvelope } from './chunk-
|
|
9
|
-
import { decodeStoredDocument, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-
|
|
6
|
+
import { restoreVectorFields } from './chunk-OETLM4EH.mjs';
|
|
7
|
+
import { requireWithinFileLimit, HEADER_SIZE, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, readHeader, packChecksummedEnvelope } from './chunk-WIT6B5OB.mjs';
|
|
8
|
+
import { decodeStoredDocument, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-MS746XNY.mjs';
|
|
10
9
|
import { encodeDocument } from './chunk-2CKDCC65.mjs';
|
|
11
|
-
import { decode } from './chunk-
|
|
12
|
-
import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-
|
|
10
|
+
import { decode } from './chunk-5GYGHPCH.mjs';
|
|
11
|
+
import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-4WQNWAQE.mjs';
|
|
12
|
+
import { DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_COMPACTION_THRESHOLD, SEGMENT_FILE_FRAMING_SLACK_BYTES, REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-S3JHYPH7.mjs';
|
|
13
13
|
import { MAX_FILE_BYTES } from './chunk-4PY4JYMY.mjs';
|
|
14
14
|
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
15
15
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
@@ -84,7 +84,14 @@ async function pagedListingOf(directory, ref) {
|
|
|
84
84
|
listed: ref.bytes
|
|
85
85
|
});
|
|
86
86
|
}
|
|
87
|
-
return { key: ref.key, path: await directory
|
|
87
|
+
return { key: ref.key, path: await filePathOf(directory, ref.key), bytes: ref.bytes, checksum: ref.checksum };
|
|
88
|
+
}
|
|
89
|
+
function filePathOf(directory, key) {
|
|
90
|
+
const { pathOf } = directory;
|
|
91
|
+
if (pathOf === void 0) {
|
|
92
|
+
return loadFailed(`The storage that holds "${key}" keeps no files, so a paged reader cannot open it`, { key });
|
|
93
|
+
}
|
|
94
|
+
return pathOf.call(directory, key);
|
|
88
95
|
}
|
|
89
96
|
|
|
90
97
|
// src/persistence/paged/totals.ts
|
|
@@ -4563,15 +4570,27 @@ function windowFits(segments, start, windowLength, paged) {
|
|
|
4563
4570
|
if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
|
|
4564
4571
|
return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
|
|
4565
4572
|
}
|
|
4566
|
-
function
|
|
4573
|
+
function balanceOf(segments, start, windowLength) {
|
|
4574
|
+
let largest = 0;
|
|
4575
|
+
let bytes = 0;
|
|
4576
|
+
for (let at = start; at < start + windowLength; at++) {
|
|
4577
|
+
largest = Math.max(largest, segments[at].bytes);
|
|
4578
|
+
bytes += segments[at].bytes;
|
|
4579
|
+
}
|
|
4580
|
+
return bytes === 0 ? { skew: 1, bytes: 0 } : { skew: largest / bytes, bytes };
|
|
4581
|
+
}
|
|
4582
|
+
function balancedBefore(balance, other) {
|
|
4583
|
+
if (balance.skew !== other.skew) return balance.skew < other.skew;
|
|
4584
|
+
return balance.bytes < other.bytes;
|
|
4585
|
+
}
|
|
4586
|
+
function mostBalancedFittingWindow(segments, windowLength, paged) {
|
|
4567
4587
|
let best = null;
|
|
4568
|
-
let
|
|
4588
|
+
let bestBalance = null;
|
|
4569
4589
|
for (let start = 0; start + windowLength <= segments.length; start++) {
|
|
4570
4590
|
if (!windowFits(segments, start, windowLength, paged)) continue;
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
bestBytes = bytes;
|
|
4591
|
+
const balance = balanceOf(segments, start, windowLength);
|
|
4592
|
+
if (bestBalance === null || balancedBefore(balance, bestBalance)) {
|
|
4593
|
+
bestBalance = balance;
|
|
4575
4594
|
best = { start, windowLength };
|
|
4576
4595
|
}
|
|
4577
4596
|
}
|
|
@@ -4579,7 +4598,7 @@ function smallestFittingWindow(segments, windowLength, paged) {
|
|
|
4579
4598
|
}
|
|
4580
4599
|
function chooseWindow(segments, wantedLength, paged) {
|
|
4581
4600
|
for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
|
|
4582
|
-
const window =
|
|
4601
|
+
const window = mostBalancedFittingWindow(segments, windowLength, paged);
|
|
4583
4602
|
if (window !== null) return window;
|
|
4584
4603
|
}
|
|
4585
4604
|
return null;
|
|
@@ -4719,8 +4738,14 @@ async function documentsReplacedLater(store, segments) {
|
|
|
4719
4738
|
function exceedsRemovedShare(removed, held) {
|
|
4720
4739
|
return held > 0 && removed > held * REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE;
|
|
4721
4740
|
}
|
|
4722
|
-
function
|
|
4723
|
-
return ref.docCount === 0 ? 0 :
|
|
4741
|
+
function removedShareOf(ref, replacedCount) {
|
|
4742
|
+
return ref.docCount === 0 ? 0 : replacedCount / ref.docCount;
|
|
4743
|
+
}
|
|
4744
|
+
function rewriteOrder(segments, replaced, left, right) {
|
|
4745
|
+
const leftShare = removedShareOf(segments[left], replaced[left].length);
|
|
4746
|
+
const rightShare = removedShareOf(segments[right], replaced[right].length);
|
|
4747
|
+
if (leftShare !== rightShare) return leftShare > rightShare ? -1 : 1;
|
|
4748
|
+
return segments[left].bytes - segments[right].bytes;
|
|
4724
4749
|
}
|
|
4725
4750
|
function chooseRewrites(segments, replaced, budget) {
|
|
4726
4751
|
let held = 0;
|
|
@@ -4731,9 +4756,7 @@ function chooseRewrites(segments, replaced, budget) {
|
|
|
4731
4756
|
}
|
|
4732
4757
|
const candidates = [];
|
|
4733
4758
|
for (let at = 0; at < segments.length; at++) if (replaced[at].length > 0) candidates.push(at);
|
|
4734
|
-
candidates.sort(
|
|
4735
|
-
(left, right) => freedBytesOf(segments[right], replaced[right].length) - freedBytesOf(segments[left], replaced[left].length)
|
|
4736
|
-
);
|
|
4759
|
+
candidates.sort((left, right) => rewriteOrder(segments, replaced, left, right));
|
|
4737
4760
|
const chosen = [];
|
|
4738
4761
|
for (const at of candidates) {
|
|
4739
4762
|
if (!exceedsRemovedShare(removed, held)) break;
|
|
@@ -5005,4 +5028,4 @@ async function removeCheckpointGarbage(store, garbage) {
|
|
|
5005
5028
|
}
|
|
5006
5029
|
}
|
|
5007
5030
|
|
|
5008
|
-
export { DEFAULT_LOG_RETENTION_BYTES, FRAME_CRC_SIZE, INSYNC_CAS_ATTEMPTS, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, NOTHING_SAVED, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, SEGMENT_TAIL_PATTERN, SNAPSHOT_CHUNK_SIZE, WAL_FORMAT_VERSION, WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT, buildEntry, bytesThatSegmentsHoldInMemory, captureWholePartition, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkSegmentHeader, checkpointGarbage, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, countsAsLatestHolder, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPagedFrozenSegment, createPrimaryResignMessage, decodeKeywordSegment, decodeWalEntry, durableRecordMissing, encodeMarkerSlot, entryByteEstimate, frameRecord, isPagedSegmentFile, joinFrames, markerSlotOffset, mergePagedWindow, nextMarkerSlot, nextSegmentKey, openPagedSegmentScan, openPagedSnapshot, optionalIndexUuid, pagedListingOf, pagedTombstonesFor, parseSegmentStartSeqNo, partitionContentsOf, readCommitMarker, readSegmentFile, readWalSegments, removeCheckpointGarbage, removeUnusedSegmentFiles, residentSectionsOf, rewrittenHeaderKey, savedBytesOfManifest, segmentKey2 as segmentKey, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writePagedSegment, writeSegmentHeader };
|
|
5031
|
+
export { DEFAULT_LOG_RETENTION_BYTES, FRAME_CRC_SIZE, INSYNC_CAS_ATTEMPTS, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, NOTHING_SAVED, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, SEGMENT_TAIL_PATTERN, SNAPSHOT_CHUNK_SIZE, WAL_FORMAT_VERSION, WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT, buildEntry, bytesThatSegmentsHoldInMemory, captureWholePartition, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkSegmentHeader, checkpointGarbage, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, countsAsLatestHolder, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPagedFrozenSegment, createPrimaryResignMessage, decodeKeywordSegment, decodeWalEntry, durableRecordMissing, encodeMarkerSlot, entryByteEstimate, filePathOf, frameRecord, isPagedSegmentFile, joinFrames, markerSlotOffset, mergePagedWindow, nextMarkerSlot, nextSegmentKey, openPagedSegmentScan, openPagedSnapshot, optionalIndexUuid, pagedListingOf, pagedTombstonesFor, parseSegmentStartSeqNo, partitionContentsOf, readCommitMarker, readSegmentFile, readWalSegments, removeCheckpointGarbage, removeUnusedSegmentFiles, residentSectionsOf, rewrittenHeaderKey, savedBytesOfManifest, segmentKey2 as segmentKey, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writePagedSegment, writeSegmentHeader };
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// src/distribution/transport/constants.ts
|
|
2
2
|
var MAX_MESSAGE_SIZE_BYTES = 67108864;
|
|
3
|
+
var UNADDRESSABLE_TARGET_DETAIL = "unaddressableTarget";
|
|
3
4
|
var GRPC_ENVELOPE_OVERHEAD_BYTES = 1024;
|
|
4
5
|
var GRPC_KEEPALIVE_TIME_MS = 3e4;
|
|
5
6
|
var GRPC_KEEPALIVE_TIMEOUT_MS = 1e4;
|
|
6
7
|
|
|
7
|
-
export { GRPC_ENVELOPE_OVERHEAD_BYTES, GRPC_KEEPALIVE_TIMEOUT_MS, GRPC_KEEPALIVE_TIME_MS, MAX_MESSAGE_SIZE_BYTES };
|
|
8
|
+
export { GRPC_ENVELOPE_OVERHEAD_BYTES, GRPC_KEEPALIVE_TIMEOUT_MS, GRPC_KEEPALIVE_TIME_MS, MAX_MESSAGE_SIZE_BYTES, UNADDRESSABLE_TARGET_DETAIL };
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { getFs, getPath } from './chunk-HLPRHOSO.mjs';
|
|
2
|
-
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-MSIN4NM3.mjs';
|
|
3
1
|
import { nodeThreadId } from './chunk-FJXJZUWF.mjs';
|
|
2
|
+
import { getFs, getPath, renameReplacing, renameUnlessMissing } from './chunk-ZYEEOZTT.mjs';
|
|
3
|
+
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-S3JHYPH7.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
5
5
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
6
6
|
import { openSync, close, readFileSync } from 'fs';
|
|
7
|
-
import {
|
|
7
|
+
import { readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
|
|
8
8
|
import { endianness } from 'os';
|
|
9
9
|
import { dirname } from 'path';
|
|
10
10
|
|
|
@@ -128,7 +128,7 @@ function holdDirectoryOpen(directory) {
|
|
|
128
128
|
}
|
|
129
129
|
async function removeStaleLock(lockPath, asidePath, isStale) {
|
|
130
130
|
try {
|
|
131
|
-
await
|
|
131
|
+
await renameUnlessMissing(lockPath, asidePath);
|
|
132
132
|
} catch (err) {
|
|
133
133
|
if (errnoOf(err) === "ENOENT") return;
|
|
134
134
|
throw err;
|
|
@@ -534,7 +534,7 @@ function createDurableDirectory(root) {
|
|
|
534
534
|
}
|
|
535
535
|
await handle.close();
|
|
536
536
|
handle = null;
|
|
537
|
-
await
|
|
537
|
+
await renameReplacing(tempPath, filePath);
|
|
538
538
|
await fsyncDirectory(dir, fs);
|
|
539
539
|
} catch (err) {
|
|
540
540
|
if (handle !== null) {
|
|
@@ -621,7 +621,7 @@ function createDurableDirectory(root) {
|
|
|
621
621
|
const target = await resolve(to);
|
|
622
622
|
const fs = await getFs();
|
|
623
623
|
await ensureDir(target.path, target.pathMod, fs);
|
|
624
|
-
await
|
|
624
|
+
await renameReplacing(source.path, target.path);
|
|
625
625
|
const directories = new Set([target.path, source.path].map((path) => target.pathMod.dirname(path)));
|
|
626
626
|
for (const movedIn of directories) await fsyncDirectory(movedIn, fs);
|
|
627
627
|
},
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-
|
|
1
|
+
import { requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-4WQNWAQE.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
3
3
|
import { codePointCount, exceedsCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
4
4
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-
|
|
2
|
-
import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-
|
|
1
|
+
import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-WIT6B5OB.mjs';
|
|
2
|
+
import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-MS746XNY.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
4
4
|
|
|
5
5
|
// src/query-suggestions/engine-hooks.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-
|
|
1
|
+
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-HZ3OAUTL.mjs';
|
|
2
2
|
import { TransportError, TransportErrorCodes } from './chunk-5FLRXF73.mjs';
|
|
3
|
-
import { decode } from './chunk-
|
|
3
|
+
import { decode } from './chunk-5GYGHPCH.mjs';
|
|
4
4
|
import { encode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
6
6
|
// src/distribution/transport/tcp/types.ts
|