@delali/narsil 0.3.1-next.140 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, writeMetadataEnvelope, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, metadataConfigOf, durableMetadataConfigOf, buildEntry, readCurrentManifest, captureWholePartition } from './chunk-KTOK275N.mjs';
|
|
2
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, DIRECTORY_LOCK_ATTEMPTS, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, MAX_SEGMENTS_PER_PARTITION, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-MVM33ZUL.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-TCLEOZ4P.mjs';
|
|
3
4
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
4
|
-
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-
|
|
5
|
-
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
6
|
-
import './chunk-26A5JFT2.mjs';
|
|
7
|
-
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-4MFH226Q.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-LVY2LI35.mjs';
|
|
6
|
+
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment, createRequestId } from './chunk-5DB2VDGI.mjs';
|
|
8
7
|
import './chunk-JPTQZWJQ.mjs';
|
|
8
|
+
import './chunk-26A5JFT2.mjs';
|
|
9
|
+
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-D33A5VMU.mjs';
|
|
9
10
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-MZI3TEOW.mjs';
|
|
10
11
|
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
11
12
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -183,6 +184,29 @@ async function lockDirectory(root) {
|
|
|
183
184
|
};
|
|
184
185
|
}
|
|
185
186
|
|
|
187
|
+
// src/persistence/durability/list-recursive.ts
|
|
188
|
+
async function listRecursive(dir, base, pathMod, fs) {
|
|
189
|
+
const results = [];
|
|
190
|
+
let entries;
|
|
191
|
+
try {
|
|
192
|
+
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
193
|
+
} catch (err) {
|
|
194
|
+
if (err.code === "ENOENT") {
|
|
195
|
+
return results;
|
|
196
|
+
}
|
|
197
|
+
throw err;
|
|
198
|
+
}
|
|
199
|
+
for (const entry of entries) {
|
|
200
|
+
const full = pathMod.join(dir, entry.name);
|
|
201
|
+
if (entry.isDirectory()) {
|
|
202
|
+
results.push(...await listRecursive(full, base, pathMod, fs));
|
|
203
|
+
} else {
|
|
204
|
+
results.push(pathMod.relative(base, full).split(pathMod.sep).join("/"));
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return results;
|
|
208
|
+
}
|
|
209
|
+
|
|
186
210
|
// src/persistence/durability/durable-filesystem.ts
|
|
187
211
|
function validateRelativeKey(key) {
|
|
188
212
|
if (key.length === 0) {
|
|
@@ -480,8 +504,8 @@ function createDurableDirectory(root) {
|
|
|
480
504
|
const fs = await getFs();
|
|
481
505
|
await ensureDir(target.path, target.pathMod, fs);
|
|
482
506
|
await fs.rename(source.path, target.path);
|
|
483
|
-
|
|
484
|
-
await fsyncDirectory(
|
|
507
|
+
const directories = new Set([target.path, source.path].map((path) => target.pathMod.dirname(path)));
|
|
508
|
+
for (const movedIn of directories) await fsyncDirectory(movedIn, fs);
|
|
485
509
|
},
|
|
486
510
|
async list(prefix) {
|
|
487
511
|
const fs = await getFs();
|
|
@@ -496,27 +520,6 @@ function createDurableDirectory(root) {
|
|
|
496
520
|
}
|
|
497
521
|
};
|
|
498
522
|
}
|
|
499
|
-
async function listRecursive(dir, base, pathMod, fs) {
|
|
500
|
-
const results = [];
|
|
501
|
-
let entries;
|
|
502
|
-
try {
|
|
503
|
-
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
504
|
-
} catch (err) {
|
|
505
|
-
if (err.code === "ENOENT") {
|
|
506
|
-
return results;
|
|
507
|
-
}
|
|
508
|
-
throw err;
|
|
509
|
-
}
|
|
510
|
-
for (const entry of entries) {
|
|
511
|
-
const full = pathMod.join(dir, entry.name);
|
|
512
|
-
if (entry.isDirectory()) {
|
|
513
|
-
results.push(...await listRecursive(full, base, pathMod, fs));
|
|
514
|
-
} else {
|
|
515
|
-
results.push(pathMod.relative(base, full).split(pathMod.sep).join("/"));
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
return results;
|
|
519
|
-
}
|
|
520
523
|
|
|
521
524
|
// src/persistence/durability/sha256.ts
|
|
522
525
|
var ROUND_CONSTANTS = new Uint32Array([
|
|
@@ -785,6 +788,7 @@ function storageFailed(code, message, key, error) {
|
|
|
785
788
|
}
|
|
786
789
|
function directoryRestoreStorage(directory) {
|
|
787
790
|
return {
|
|
791
|
+
directory,
|
|
788
792
|
read: (key) => directory.read(key),
|
|
789
793
|
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
790
794
|
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
@@ -829,6 +833,25 @@ function pathUnder(path, root) {
|
|
|
829
833
|
const next = path.charAt(root.length);
|
|
830
834
|
return next === "" || next === "/" || next === "\\";
|
|
831
835
|
}
|
|
836
|
+
function stagedDirectory(directory, staged, unstaged) {
|
|
837
|
+
const { readRange, byteLength } = directory;
|
|
838
|
+
return {
|
|
839
|
+
root: directory.root,
|
|
840
|
+
appendHandle: (key) => directory.appendHandle(staged(key)),
|
|
841
|
+
markerHandle: (key) => directory.markerHandle(staged(key)),
|
|
842
|
+
atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
|
|
843
|
+
syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
|
|
844
|
+
read: (key) => directory.read(staged(key)),
|
|
845
|
+
remove: (key) => directory.remove(staged(key)),
|
|
846
|
+
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
847
|
+
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
848
|
+
pathOf: (key) => directory.pathOf(staged(key)),
|
|
849
|
+
...readRange === void 0 ? {} : {
|
|
850
|
+
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
851
|
+
},
|
|
852
|
+
...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
|
|
853
|
+
};
|
|
854
|
+
}
|
|
832
855
|
function createRestoreStaging(storage, indexName, options = {}) {
|
|
833
856
|
const stagingPrefix = newStagingPrefix();
|
|
834
857
|
const indexRoot = `${indexName}/`;
|
|
@@ -850,6 +873,7 @@ function createRestoreStaging(storage, indexName, options = {}) {
|
|
|
850
873
|
};
|
|
851
874
|
const pathOf = storage.pathOf;
|
|
852
875
|
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
876
|
+
if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
|
|
853
877
|
let retiredPrefix = null;
|
|
854
878
|
async function finishOnStorage() {
|
|
855
879
|
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
@@ -969,7 +993,8 @@ function createArchiveCheckpointAccess(deps) {
|
|
|
969
993
|
const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
|
|
970
994
|
return whole.slice(offset, offset + length);
|
|
971
995
|
},
|
|
972
|
-
release
|
|
996
|
+
release,
|
|
997
|
+
directory: deps.directory
|
|
973
998
|
};
|
|
974
999
|
} catch (error) {
|
|
975
1000
|
release();
|
|
@@ -1062,6 +1087,13 @@ function formatSegmentStart(startSeqNo) {
|
|
|
1062
1087
|
}
|
|
1063
1088
|
|
|
1064
1089
|
// src/persistence/durability/recovery.ts
|
|
1090
|
+
async function removeLeftoverRewriteFiles(directory, names) {
|
|
1091
|
+
for (const indexName of names) {
|
|
1092
|
+
for (const place of ["rebuilt", "received"]) {
|
|
1093
|
+
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1065
1097
|
async function listPersistedIndexes(directory) {
|
|
1066
1098
|
const metaKeys = await directory.list("");
|
|
1067
1099
|
const names = [];
|
|
@@ -1081,6 +1113,7 @@ async function loadMetadata(directory, indexName) {
|
|
|
1081
1113
|
return metadata;
|
|
1082
1114
|
}
|
|
1083
1115
|
async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
1116
|
+
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
1084
1117
|
const read = await readWalSegments(directory, indexName, partitionId);
|
|
1085
1118
|
if (read === null) {
|
|
1086
1119
|
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
@@ -1185,6 +1218,22 @@ async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
|
|
|
1185
1218
|
}
|
|
1186
1219
|
}
|
|
1187
1220
|
}
|
|
1221
|
+
async function openNewLogSegments(indexState, failPartition2) {
|
|
1222
|
+
const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
|
|
1223
|
+
try {
|
|
1224
|
+
if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
|
|
1225
|
+
} catch (error) {
|
|
1226
|
+
failPartition2(partitionId, partition, error);
|
|
1227
|
+
return;
|
|
1228
|
+
}
|
|
1229
|
+
const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
|
|
1230
|
+
failPartition2(partitionId, partition, error);
|
|
1231
|
+
});
|
|
1232
|
+
partition.appendChain = rolled;
|
|
1233
|
+
await rolled;
|
|
1234
|
+
});
|
|
1235
|
+
await Promise.all(rolls);
|
|
1236
|
+
}
|
|
1188
1237
|
function aTargetFailed(indexState, targets) {
|
|
1189
1238
|
return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
|
|
1190
1239
|
}
|
|
@@ -1217,9 +1266,10 @@ function wholePartitionsWhereMostChanged(manager, rules) {
|
|
|
1217
1266
|
return whole;
|
|
1218
1267
|
};
|
|
1219
1268
|
}
|
|
1220
|
-
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf) {
|
|
1269
|
+
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
|
|
1221
1270
|
for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
1222
1271
|
return whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
1272
|
+
onCaptured?.();
|
|
1223
1273
|
const documentCount = manager.countDocuments();
|
|
1224
1274
|
const targets = [];
|
|
1225
1275
|
let recordsLeftInLog = 0;
|
|
@@ -1575,23 +1625,68 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1575
1625
|
const writeRecords = input.hooks.writeRecords ?? null;
|
|
1576
1626
|
const { indexName } = input;
|
|
1577
1627
|
const priorManifest = await readCurrentManifest(store, input.indexName);
|
|
1628
|
+
const textOnDisk = metadata.textStorage === "disk";
|
|
1629
|
+
const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
|
|
1630
|
+
priorCheckpoint: priorManifest?.checkpoint ?? [],
|
|
1631
|
+
priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
|
|
1632
|
+
everyPartition: fromMemory,
|
|
1633
|
+
aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
|
|
1634
|
+
...writeRecords === null ? {} : {
|
|
1635
|
+
captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
|
|
1636
|
+
}
|
|
1637
|
+
});
|
|
1638
|
+
let changesSinceCapture = null;
|
|
1578
1639
|
const capture2 = await captureCheckpoint(
|
|
1579
1640
|
input.indexState,
|
|
1580
1641
|
manager,
|
|
1581
1642
|
vectorIndexes,
|
|
1582
|
-
|
|
1583
|
-
priorCheckpoint: priorManifest?.checkpoint ?? [],
|
|
1584
|
-
priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
|
|
1585
|
-
everyPartition: fromMemory,
|
|
1586
|
-
aWorkerCanSerialise: input.canOffload && checkpointWorkerIsUsable(),
|
|
1587
|
-
...writeRecords === null ? {} : {
|
|
1588
|
-
captureRecords: (partitionId) => (segments2) => writeRecords.captureSegments(indexName, partitionId, segments2)
|
|
1589
|
-
}
|
|
1590
|
-
}),
|
|
1643
|
+
textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
|
|
1591
1644
|
fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
|
|
1592
1645
|
priorManifest?.vectors ?? [],
|
|
1593
|
-
writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo)
|
|
1646
|
+
writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
|
|
1647
|
+
textOnDisk ? () => {
|
|
1648
|
+
changesSinceCapture = manager.openCheckpointChangeLog();
|
|
1649
|
+
} : void 0
|
|
1594
1650
|
);
|
|
1651
|
+
try {
|
|
1652
|
+
await openNewLogSegments(input.indexState, input.failPartition);
|
|
1653
|
+
return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
|
|
1654
|
+
return changesSinceCapture;
|
|
1655
|
+
});
|
|
1656
|
+
} finally {
|
|
1657
|
+
changesSinceCapture?.close();
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
async function adoptWrittenSegments(input, manager, partitions, changes) {
|
|
1661
|
+
const { indexName, indexState, hooks } = input;
|
|
1662
|
+
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-BUFNZMEU.mjs');
|
|
1663
|
+
const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
|
|
1664
|
+
if (adoption === null) return [];
|
|
1665
|
+
const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
1666
|
+
if (changes.partitionsReplaced) return false;
|
|
1667
|
+
const changed = changes.takeWrittenDocIds();
|
|
1668
|
+
swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
|
|
1669
|
+
hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
|
|
1670
|
+
});
|
|
1671
|
+
hooks.queueDiskAdoption?.({
|
|
1672
|
+
type: "adoptDiskSegments",
|
|
1673
|
+
indexName,
|
|
1674
|
+
partitions: actionPartitionsOf(adoption),
|
|
1675
|
+
changed: [...changed],
|
|
1676
|
+
requestId: createRequestId()
|
|
1677
|
+
});
|
|
1678
|
+
return true;
|
|
1679
|
+
});
|
|
1680
|
+
if (!adopted) {
|
|
1681
|
+
closeUnusedSegments(adoption);
|
|
1682
|
+
return [];
|
|
1683
|
+
}
|
|
1684
|
+
await hooks.awaitCopiesOfIndex?.(indexName);
|
|
1685
|
+
return adoption.droppedKeys;
|
|
1686
|
+
}
|
|
1687
|
+
async function finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, changesSinceCapture) {
|
|
1688
|
+
const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
|
|
1689
|
+
const writeRecords = input.hooks.writeRecords ?? null;
|
|
1595
1690
|
const { targets, documentCount } = capture2;
|
|
1596
1691
|
await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
|
|
1597
1692
|
if (aTargetFailed(input.indexState, targets)) return 0;
|
|
@@ -1615,8 +1710,15 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1615
1710
|
input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
|
|
1616
1711
|
recordWrittenVectors(vectorIndexes, liveVectors.written);
|
|
1617
1712
|
await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
|
|
1713
|
+
const changes = changesSinceCapture();
|
|
1714
|
+
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
1618
1715
|
if (!input.holdsCheckpointFiles()) {
|
|
1619
|
-
|
|
1716
|
+
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
|
|
1717
|
+
const stillRead = metadata.textStorage === "disk" ? (await import('./module-BUFNZMEU.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
1718
|
+
await removeCheckpointGarbage(
|
|
1719
|
+
store,
|
|
1720
|
+
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
|
|
1721
|
+
);
|
|
1620
1722
|
}
|
|
1621
1723
|
const checkpointDocumentCount = documentCount;
|
|
1622
1724
|
await input.queueMetadataWrite(input.indexName, async () => {
|
|
@@ -2129,6 +2231,8 @@ function parseSegmentStartSeqNo2(key) {
|
|
|
2129
2231
|
function createWalWriter(directory, config) {
|
|
2130
2232
|
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
2131
2233
|
const commitKey = `${config.indexName}/wal/${config.partitionId}/commit`;
|
|
2234
|
+
const preparedKey = nextSegmentKey(config.indexName, config.partitionId);
|
|
2235
|
+
let nextSegmentPrepared = false;
|
|
2132
2236
|
let handle = null;
|
|
2133
2237
|
let activeKey = null;
|
|
2134
2238
|
let activeStartSeqNo = 0;
|
|
@@ -2207,16 +2311,49 @@ function createWalWriter(directory, config) {
|
|
|
2207
2311
|
await withDurabilityLock(flushActiveSegment);
|
|
2208
2312
|
}
|
|
2209
2313
|
}
|
|
2210
|
-
async function
|
|
2211
|
-
|
|
2212
|
-
if (handle !== null) {
|
|
2213
|
-
await flushActiveSegment();
|
|
2214
|
-
await handle.close();
|
|
2215
|
-
handle = null;
|
|
2216
|
-
}
|
|
2217
|
-
await openSegmentByKey(segmentKey(config.indexName, config.partitionId, startSeqNo));
|
|
2314
|
+
async function rollUnderLock(startSeqNo) {
|
|
2315
|
+
if (handle !== null) {
|
|
2218
2316
|
await flushActiveSegment();
|
|
2317
|
+
await handle.close();
|
|
2318
|
+
handle = null;
|
|
2319
|
+
}
|
|
2320
|
+
await openSegmentByKey(segmentKey(config.indexName, config.partitionId, startSeqNo));
|
|
2321
|
+
await flushActiveSegment();
|
|
2322
|
+
}
|
|
2323
|
+
async function rollToNewSegment(startSeqNo) {
|
|
2324
|
+
await withDurabilityLock(() => rollUnderLock(startSeqNo));
|
|
2325
|
+
}
|
|
2326
|
+
function activeSegmentHoldsRecords() {
|
|
2327
|
+
return handle !== null && activeBytes > SEGMENT_HEADER_SIZE && highestAppendedSeqNo >= activeStartSeqNo;
|
|
2328
|
+
}
|
|
2329
|
+
async function prepareNextSegment() {
|
|
2330
|
+
const prepared = await directory.appendHandle(preparedKey);
|
|
2331
|
+
try {
|
|
2332
|
+
if (await prepared.size() > 0) await prepared.truncate(0);
|
|
2333
|
+
await prepared.append(writeSegmentHeader());
|
|
2334
|
+
await prepared.sync();
|
|
2335
|
+
nextSegmentPrepared = true;
|
|
2336
|
+
} finally {
|
|
2337
|
+
await prepared.close();
|
|
2338
|
+
}
|
|
2339
|
+
}
|
|
2340
|
+
async function swapToPreparedSegment(sealed) {
|
|
2341
|
+
const writer = await getMarkerWriter();
|
|
2342
|
+
const startSeqNo = highestAppendedSeqNo + 1;
|
|
2343
|
+
const durableSeqNo = Math.max(highestAppendedSeqNo, highestDurableSeqNo);
|
|
2344
|
+
const key = segmentKey(config.indexName, config.partitionId, startSeqNo);
|
|
2345
|
+
await Promise.all([sealed.sync(), directory.move(preparedKey, key)]);
|
|
2346
|
+
handle = await directory.appendHandle(key);
|
|
2347
|
+
activeKey = key;
|
|
2348
|
+
activeStartSeqNo = startSeqNo;
|
|
2349
|
+
activeBytes = SEGMENT_HEADER_SIZE;
|
|
2350
|
+
await writer.commit({
|
|
2351
|
+
activeSegmentSeqNo: startSeqNo,
|
|
2352
|
+
durableByteLength: SEGMENT_HEADER_SIZE,
|
|
2353
|
+
highestDurableSeqNo: durableSeqNo
|
|
2219
2354
|
});
|
|
2355
|
+
highestDurableSeqNo = durableSeqNo;
|
|
2356
|
+
await sealed.close();
|
|
2220
2357
|
}
|
|
2221
2358
|
async function maybeRoll(nextSeqNo, appendedBytes) {
|
|
2222
2359
|
if (handle === null || activeBytes <= SEGMENT_HEADER_SIZE) {
|
|
@@ -2302,6 +2439,19 @@ function createWalWriter(directory, config) {
|
|
|
2302
2439
|
await coordinator.commit();
|
|
2303
2440
|
},
|
|
2304
2441
|
rollToNewSegment,
|
|
2442
|
+
async prepareCheckpointRoll() {
|
|
2443
|
+
if (activeSegmentHoldsRecords()) await prepareNextSegment();
|
|
2444
|
+
},
|
|
2445
|
+
async rollForCheckpoint() {
|
|
2446
|
+
await withDurabilityLock(async () => {
|
|
2447
|
+
const sealed = handle;
|
|
2448
|
+
const prepared = nextSegmentPrepared;
|
|
2449
|
+
nextSegmentPrepared = false;
|
|
2450
|
+
if (sealed === null || !activeSegmentHoldsRecords()) return;
|
|
2451
|
+
if (prepared) await swapToPreparedSegment(sealed);
|
|
2452
|
+
else await rollUnderLock(highestAppendedSeqNo + 1);
|
|
2453
|
+
});
|
|
2454
|
+
},
|
|
2305
2455
|
async close() {
|
|
2306
2456
|
await withDurabilityLock(async () => {
|
|
2307
2457
|
if (handle !== null) {
|
|
@@ -2776,9 +2926,33 @@ async function readEarlierSegmentDocuments(store, key) {
|
|
|
2776
2926
|
|
|
2777
2927
|
// src/persistence/durability/checkpoint-count.ts
|
|
2778
2928
|
var decoder = new TextDecoder();
|
|
2929
|
+
async function countPagedPartition(store, partition) {
|
|
2930
|
+
const { directory } = store;
|
|
2931
|
+
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
2932
|
+
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-BUFNZMEU.mjs');
|
|
2933
|
+
for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
2934
|
+
const scans = [];
|
|
2935
|
+
try {
|
|
2936
|
+
for (const ref of partition.segments) scans.push(await openPagedSegmentScan(directory, ref));
|
|
2937
|
+
let live = 0;
|
|
2938
|
+
for (let at = 0; at < scans.length; at++) {
|
|
2939
|
+
for (let ordinal = 0; ordinal < scans[at].count; ordinal++) {
|
|
2940
|
+
if (countsAsLatestHolder(scans, at, scans[at].idAt(ordinal))) live += 1;
|
|
2941
|
+
}
|
|
2942
|
+
}
|
|
2943
|
+
return live;
|
|
2944
|
+
} finally {
|
|
2945
|
+
for (const scan of scans) scan.close();
|
|
2946
|
+
}
|
|
2947
|
+
}
|
|
2779
2948
|
async function countCurrentLayout(store, manifest) {
|
|
2780
2949
|
let total = 0;
|
|
2781
2950
|
for (const partition of manifest.partitions) {
|
|
2951
|
+
const paged = await countPagedPartition(store, partition);
|
|
2952
|
+
if (paged !== null) {
|
|
2953
|
+
total += paged;
|
|
2954
|
+
continue;
|
|
2955
|
+
}
|
|
2782
2956
|
const live = /* @__PURE__ */ new Set();
|
|
2783
2957
|
for (const ref of partition.segments) {
|
|
2784
2958
|
const contents = await readSegmentFile(store, ref);
|
|
@@ -3186,7 +3360,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3186
3360
|
|
|
3187
3361
|
// src/persistence/durability/manager.ts
|
|
3188
3362
|
function createDurabilityManager(config, hooks, directoryOverride) {
|
|
3189
|
-
const directory = createDurableDirectory(config.directory ?? "");
|
|
3363
|
+
const directory = directoryOverride ?? createDurableDirectory(config.directory ?? "");
|
|
3190
3364
|
const canOffloadCheckpoint = directoryOverride === void 0;
|
|
3191
3365
|
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
3192
3366
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
@@ -3369,6 +3543,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3369
3543
|
await directoryWatch.claim();
|
|
3370
3544
|
await recoverDirectoryRestores(directory);
|
|
3371
3545
|
const names = await listPersistedIndexes(directory);
|
|
3546
|
+
await removeLeftoverRewriteFiles(directory, names);
|
|
3372
3547
|
for (const indexName of names) {
|
|
3373
3548
|
await recoverIndex(indexName, metadataOnly, false);
|
|
3374
3549
|
}
|
|
@@ -3392,6 +3567,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3392
3567
|
keepWriteRecordsWith(keeper) {
|
|
3393
3568
|
hooks.writeRecords = keeper;
|
|
3394
3569
|
},
|
|
3570
|
+
rewriteStorage: () => ({ directory, writeRecords: hooks.writeRecords ?? null }),
|
|
3395
3571
|
heldCheckpoints: () => heldCheckpoints.report(indexes),
|
|
3396
3572
|
async commitPartition(indexName, partitionId) {
|
|
3397
3573
|
const partition = indexes.get(indexName)?.partitions.get(partitionId);
|
|
@@ -3930,6 +4106,8 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
3930
4106
|
},
|
|
3931
4107
|
mustRewriteTerms: hooks.mustRewriteTerms,
|
|
3932
4108
|
onManifestCommitted: hooks.onManifestCommitted,
|
|
4109
|
+
...hooks.queueDiskAdoption === void 0 ? {} : { queueDiskAdoption: hooks.queueDiskAdoption },
|
|
4110
|
+
...hooks.awaitCopiesOfIndex === void 0 ? {} : { awaitCopiesOfIndex: hooks.awaitCopiesOfIndex },
|
|
3933
4111
|
createIndexFromMetadata: hooks.createIndexFromMetadata,
|
|
3934
4112
|
recordCheckpoint: hooks.recordCheckpoint,
|
|
3935
4113
|
onLayoutUpgrade: hooks.onLayoutUpgrade,
|
|
@@ -4012,6 +4190,8 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
4012
4190
|
const entry = wiring.indexRegistry.get(indexName);
|
|
4013
4191
|
if (entry !== void 0 && entry.analysisGeneration === analysisGeneration) entry.durableAnalysis = null;
|
|
4014
4192
|
},
|
|
4193
|
+
queueDiskAdoption: wiring.queueDiskAdoption,
|
|
4194
|
+
awaitCopiesOfIndex: wiring.awaitCopiesOfIndex,
|
|
4015
4195
|
createIndexFromMetadata: wiring.createIndexFromMetadata,
|
|
4016
4196
|
recordCheckpoint: wiring.recordCheckpoint,
|
|
4017
4197
|
onLayoutUpgrade: wiring.emitLayoutUpgrade,
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import './chunk-
|
|
5
|
-
import './chunk-4MFH226Q.mjs';
|
|
1
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport } from './chunk-VFYMPCOV.mjs';
|
|
2
|
+
import './chunk-ZEV26AQ6.mjs';
|
|
3
|
+
import './chunk-WCFCHIBB.mjs';
|
|
4
|
+
import './chunk-5DB2VDGI.mjs';
|
|
6
5
|
import './chunk-JPTQZWJQ.mjs';
|
|
6
|
+
import './chunk-26A5JFT2.mjs';
|
|
7
|
+
import './chunk-D33A5VMU.mjs';
|
|
7
8
|
import './chunk-MZI3TEOW.mjs';
|
|
8
9
|
import './chunk-35B42UFM.mjs';
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-MWUK3DA5.mjs';
|
|
2
|
+
import './chunk-MVM33ZUL.mjs';
|
|
3
|
+
import './chunk-CCOAOO73.mjs';
|
|
4
|
+
import './chunk-TCLEOZ4P.mjs';
|
|
5
|
+
import './chunk-WCFCHIBB.mjs';
|
|
6
|
+
import './chunk-3ZBJYAS4.mjs';
|
|
7
|
+
import './chunk-5DB2VDGI.mjs';
|
|
8
|
+
import './chunk-JPTQZWJQ.mjs';
|
|
9
|
+
import './chunk-26A5JFT2.mjs';
|
|
10
|
+
import './chunk-D33A5VMU.mjs';
|
|
11
|
+
import './chunk-MZI3TEOW.mjs';
|
|
12
|
+
import './chunk-35B42UFM.mjs';
|