@delali/narsil 0.4.1 → 0.4.2-next.273
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-56UNHLAF.mjs} +1 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-EKISQBV2.mjs} +283 -244
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-VAPCFSMX.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-VIVG3OXM.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-76Q5SZ5S.mjs} +23 -10
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-3HKIVZFU.mjs} +295 -106
- package/dist/browser/{segment-codec-leaf-A427QJUX.mjs → segment-codec-leaf-GXSRX5VT.mjs} +47 -30
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-35RWLUVY.mjs} +63 -144
- package/dist/{chunk-6GHX35ZM.mjs → chunk-43T3P3BE.mjs} +1 -1
- package/dist/{chunk-IL5DRCNU.mjs → chunk-4JLHHEBN.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-4YIVY3PB.mjs} +2 -2
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-5SHWA6PE.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-6LVQ3GLT.mjs} +3 -3
- package/dist/{chunk-DDUXEJXW.mjs → chunk-6SIEVWMG.mjs} +45 -22
- package/dist/{chunk-WM5KJSQV.mjs → chunk-ACNHJOGC.mjs} +1 -1
- package/dist/{chunk-755NM4PL.mjs → chunk-BWPVDIRN.mjs} +183 -21
- package/dist/{chunk-SMQPZHPS.mjs → chunk-C57GWAXN.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-DRZP5MSG.mjs} +5 -5
- package/dist/{chunk-MYN4K5IS.mjs → chunk-DZMD4OMD.mjs} +1 -1
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-EY5U4P5V.mjs} +1 -1
- package/dist/{chunk-X3EQCQLP.mjs → chunk-EZJ6HBG7.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-FE2XILWN.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-G3KVEHIW.mjs} +2 -2
- package/dist/{chunk-B3MO37YF.mjs → chunk-GTGQLCEI.mjs} +1 -1
- package/dist/{chunk-MSJD74GN.mjs → chunk-H23FB267.mjs} +5 -5
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-WTMCGZ74.mjs → chunk-KXA77DPZ.mjs} +2 -2
- package/dist/{chunk-6I6OO6B2.mjs → chunk-LRIJ7U3T.mjs} +4 -4
- package/dist/{chunk-3IE3A74H.mjs → chunk-PNFP5THV.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-PQWK7KXS.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-OGYSPNL2.mjs → chunk-QNZJYMCF.mjs} +7 -7
- package/dist/{chunk-DBC4EJWB.mjs → chunk-R3PFPFME.mjs} +1 -1
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-MI6UZGUR.mjs → chunk-SPA7GXST.mjs} +3 -3
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-WYH3QHBK.mjs} +698 -451
- package/dist/{chunk-D456ZFPQ.mjs → chunk-X5V25BPN.mjs} +2 -2
- package/dist/{chunk-GXCDWLQI.mjs → chunk-Y7GXGJFZ.mjs} +23 -10
- package/dist/{chunk-NCIGNQJE.mjs → chunk-YGRJLI2Z.mjs} +930 -387
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-ZO7VLAIT.mjs} +2 -2
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-ZTB7AWTW.mjs} +4 -4
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-7G57AYXH.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-KPN7NSOC.mjs} +14 -14
- package/dist/msgpack-leaf-LIT7TKZW.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-CSULLISE.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-BKQADXOT.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-HTSJKZRU.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AOLLV37A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-DN4FT62B.mjs} +23 -10
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-EF4Y5C3Y.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-ESVBC3AT.mjs} +1 -1
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-H6LK5WCF.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-JEYZQA4Z.mjs} +699 -451
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-KJQ3BUVM.mjs} +4 -4
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-QYVRD6C5.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-X3VY2YT4.mjs} +2 -2
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-ZF3VSGEI.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-FOUPTT7J.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-MGFFGKYQ.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-MVJTP3E4.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PSWHL55A.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-WOJWSDLW.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-ZUN6Y44X.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-K5XG46FV.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-KDPM3P27.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-FVFMBBH6.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-3QEJNMFH.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-Q76Q4S35.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-BSDDR7DQ.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-6TT66HC5.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-R5JQ2X2M.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-NCGETZYY.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-TX3GTD6S.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-VLVPAUWC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-NZSBQZKG.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -12
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
|
@@ -242,13 +242,14 @@ function bindEngineNames(host) {
|
|
|
242
242
|
|
|
243
243
|
// src/persistence/storage-host/msgpack.ts
|
|
244
244
|
var decode;
|
|
245
|
+
var decodeBoundedMessage;
|
|
245
246
|
var decodeMulti;
|
|
246
247
|
var encode;
|
|
247
248
|
var encodeDocument;
|
|
248
249
|
var restoreVectorFields;
|
|
249
250
|
var setNestedValue;
|
|
250
251
|
function bindMsgpackNames(msgpack) {
|
|
251
|
-
({ decode, decodeMulti, encode, encodeDocument, restoreVectorFields, setNestedValue } = msgpack);
|
|
252
|
+
({ decode, decodeBoundedMessage, decodeMulti, encode, encodeDocument, restoreVectorFields, setNestedValue } = msgpack);
|
|
252
253
|
}
|
|
253
254
|
|
|
254
255
|
// src/persistence/storage-host/segment-codec.ts
|
|
@@ -1696,6 +1697,159 @@ async function checkpointInTurn(turns, turn) {
|
|
|
1696
1697
|
return run;
|
|
1697
1698
|
}
|
|
1698
1699
|
|
|
1700
|
+
// src/persistence/durability/mutation-batch.ts
|
|
1701
|
+
function shutDownRefusal() {
|
|
1702
|
+
return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
1703
|
+
}
|
|
1704
|
+
function toError2(err) {
|
|
1705
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
1706
|
+
}
|
|
1707
|
+
function requireAscendingSeqNo(record, seqNo, previousSeqNo) {
|
|
1708
|
+
if (seqNo > previousSeqNo) return;
|
|
1709
|
+
throw new NarsilError(
|
|
1710
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1711
|
+
`Log record ${seqNo} for partition ${record.partitionId} of index "${record.indexName}" does not follow record ${previousSeqNo}`,
|
|
1712
|
+
{ indexName: record.indexName, partitionId: record.partitionId, seqNo, previousSeqNo }
|
|
1713
|
+
);
|
|
1714
|
+
}
|
|
1715
|
+
function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
1716
|
+
for (let i = 0; i < count; i++) {
|
|
1717
|
+
const outcome = outcomes[i];
|
|
1718
|
+
if (outcome === void 0 || outcome.ok) {
|
|
1719
|
+
outcomes[i] = { ok: false, error };
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
}
|
|
1723
|
+
function failPartition(partition, address, deps, err) {
|
|
1724
|
+
const error = toError2(err);
|
|
1725
|
+
if (partition.failed !== null) return partition.failed;
|
|
1726
|
+
partition.failed = error;
|
|
1727
|
+
if (!deps.failsPartitionAlone(address.indexName, address.partitionId, error)) deps.markFatal(error);
|
|
1728
|
+
return error;
|
|
1729
|
+
}
|
|
1730
|
+
async function commitEveryPartition(indexes, deps) {
|
|
1731
|
+
for (const [indexName, indexState] of indexes) {
|
|
1732
|
+
for (const [partitionId, partition] of indexState.partitions) {
|
|
1733
|
+
if (partition.failed !== null || deps.fatalError() !== null) continue;
|
|
1734
|
+
try {
|
|
1735
|
+
await partition.walWriter.commit();
|
|
1736
|
+
} catch (err) {
|
|
1737
|
+
failPartition(partition, { indexName, partitionId }, deps, err);
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1741
|
+
}
|
|
1742
|
+
function partitionFailureFor(indexName, deps) {
|
|
1743
|
+
return (partitionId, partition, error) => failPartition(partition, { indexName, partitionId }, deps, error);
|
|
1744
|
+
}
|
|
1745
|
+
async function recordBatchesByPartition(records, partitionOf, deps) {
|
|
1746
|
+
const positionsByPartition = /* @__PURE__ */ new Map();
|
|
1747
|
+
for (let i = 0; i < records.length; i++) {
|
|
1748
|
+
const partition = partitionOf(records[i]);
|
|
1749
|
+
const positions = positionsByPartition.get(partition);
|
|
1750
|
+
if (positions === void 0) {
|
|
1751
|
+
positionsByPartition.set(partition, [i]);
|
|
1752
|
+
} else {
|
|
1753
|
+
positions.push(i);
|
|
1754
|
+
}
|
|
1755
|
+
}
|
|
1756
|
+
const outcomes = new Array(records.length);
|
|
1757
|
+
await Promise.all(
|
|
1758
|
+
[...positionsByPartition].map(async ([partition, positions]) => {
|
|
1759
|
+
const partitionOutcomes = await recordPartitionBatch(
|
|
1760
|
+
partition,
|
|
1761
|
+
positions.map((position) => records[position]),
|
|
1762
|
+
deps
|
|
1763
|
+
);
|
|
1764
|
+
for (let i = 0; i < positions.length; i++) {
|
|
1765
|
+
outcomes[positions[i]] = partitionOutcomes[i];
|
|
1766
|
+
}
|
|
1767
|
+
})
|
|
1768
|
+
);
|
|
1769
|
+
return outcomes;
|
|
1770
|
+
}
|
|
1771
|
+
async function recordPartitionBatch(partition, records, deps) {
|
|
1772
|
+
const outcomes = new Array(records.length);
|
|
1773
|
+
let appendedCount = 0;
|
|
1774
|
+
let commitsHere = false;
|
|
1775
|
+
const appended = partition.appendChain.then(async () => {
|
|
1776
|
+
const fatal = deps.fatalError();
|
|
1777
|
+
if (fatal !== null) {
|
|
1778
|
+
throw fatal;
|
|
1779
|
+
}
|
|
1780
|
+
if (partition.failed !== null) {
|
|
1781
|
+
throw partition.failed;
|
|
1782
|
+
}
|
|
1783
|
+
const applied = [];
|
|
1784
|
+
for (let i = 0; i < records.length; i++) {
|
|
1785
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
1786
|
+
try {
|
|
1787
|
+
await records[i].apply();
|
|
1788
|
+
applied.push(i);
|
|
1789
|
+
} catch (err) {
|
|
1790
|
+
outcomes[i] = { ok: false, error: err };
|
|
1791
|
+
}
|
|
1792
|
+
}
|
|
1793
|
+
if (applied.length === 0) {
|
|
1794
|
+
return;
|
|
1795
|
+
}
|
|
1796
|
+
let lastSeqNo = partition.appliedSeqNo;
|
|
1797
|
+
const entries = [];
|
|
1798
|
+
try {
|
|
1799
|
+
for (const i of applied) {
|
|
1800
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
1801
|
+
let numbered;
|
|
1802
|
+
try {
|
|
1803
|
+
numbered = deps.numberEntry(records[i]);
|
|
1804
|
+
} catch (err) {
|
|
1805
|
+
outcomes[i] = { ok: false, error: err };
|
|
1806
|
+
continue;
|
|
1807
|
+
}
|
|
1808
|
+
if (numbered === null) {
|
|
1809
|
+
lastSeqNo = partition.seqOwner.next();
|
|
1810
|
+
entries.push(deps.buildEntry(records[i], lastSeqNo));
|
|
1811
|
+
} else {
|
|
1812
|
+
requireAscendingSeqNo(records[i], numbered.seqNo, lastSeqNo);
|
|
1813
|
+
partition.seqOwner.advanceTo(numbered.seqNo, numbered.primaryTerm);
|
|
1814
|
+
lastSeqNo = numbered.seqNo;
|
|
1815
|
+
entries.push(numbered);
|
|
1816
|
+
}
|
|
1817
|
+
outcomes[i] = { ok: true, seqNo: lastSeqNo };
|
|
1818
|
+
if (!deps.defersCommit(records[i])) commitsHere = true;
|
|
1819
|
+
}
|
|
1820
|
+
if (entries.length > 0) await partition.walWriter.appendAll(entries);
|
|
1821
|
+
} catch (err) {
|
|
1822
|
+
throw failPartition(partition, records[0], deps, err);
|
|
1823
|
+
}
|
|
1824
|
+
partition.appliedSeqNo = lastSeqNo;
|
|
1825
|
+
appendedCount = entries.length;
|
|
1826
|
+
});
|
|
1827
|
+
partition.appendChain = appended.catch(() => void 0);
|
|
1828
|
+
try {
|
|
1829
|
+
await appended;
|
|
1830
|
+
if (deps.syncEachBatch && appendedCount > 0 && commitsHere) {
|
|
1831
|
+
try {
|
|
1832
|
+
await partition.walWriter.commit();
|
|
1833
|
+
} catch (err) {
|
|
1834
|
+
throw failPartition(partition, records[0], deps, err);
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
} catch (err) {
|
|
1838
|
+
failUnlessAlreadyFailed(outcomes, records.length, toError2(err));
|
|
1839
|
+
}
|
|
1840
|
+
return outcomes;
|
|
1841
|
+
}
|
|
1842
|
+
async function commitAppended(partition, address, deps) {
|
|
1843
|
+
await partition.appendChain;
|
|
1844
|
+
const failed = partition.failed ?? deps.fatalError();
|
|
1845
|
+
if (failed !== null) throw failed;
|
|
1846
|
+
try {
|
|
1847
|
+
await partition.walWriter.commit();
|
|
1848
|
+
} catch (err) {
|
|
1849
|
+
throw failPartition(partition, address, deps, err);
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1699
1853
|
// src/persistence/durability/segment/store.ts
|
|
1700
1854
|
function directoryCheckpointStore(directory) {
|
|
1701
1855
|
const store = {
|
|
@@ -1703,11 +1857,11 @@ function directoryCheckpointStore(directory) {
|
|
|
1703
1857
|
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
1704
1858
|
remove: (key) => directory.remove(key),
|
|
1705
1859
|
list: (prefix) => directory.list(prefix),
|
|
1706
|
-
pathOf: (key) => directory.pathOf(key),
|
|
1707
1860
|
directory
|
|
1708
1861
|
};
|
|
1709
|
-
const byteLength = directory
|
|
1862
|
+
const { byteLength, pathOf } = directory;
|
|
1710
1863
|
if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
|
|
1864
|
+
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(directory, key);
|
|
1711
1865
|
return store;
|
|
1712
1866
|
}
|
|
1713
1867
|
function joined(chunks) {
|
|
@@ -1934,7 +2088,8 @@ async function writeTransition(store, indexName, transition) {
|
|
|
1934
2088
|
|
|
1935
2089
|
// src/persistence/durability/layout-transition/index.ts
|
|
1936
2090
|
async function transitionWalLayout(registry, indexName, manager, replay) {
|
|
1937
|
-
const state = registry.
|
|
2091
|
+
const state = registry.indexes.get(indexName);
|
|
2092
|
+
if (state === void 0) throw shutDownRefusal();
|
|
1938
2093
|
await switchDurableRouting(
|
|
1939
2094
|
manager,
|
|
1940
2095
|
(oldPartitionCount) => checkpointInTurn(state, {
|
|
@@ -2274,7 +2429,7 @@ function termBytesOf(core, indexName) {
|
|
|
2274
2429
|
}
|
|
2275
2430
|
return Math.max(0, bytes);
|
|
2276
2431
|
}
|
|
2277
|
-
function
|
|
2432
|
+
function toError3(value) {
|
|
2278
2433
|
return value instanceof Error ? value : new Error(String(value));
|
|
2279
2434
|
}
|
|
2280
2435
|
async function requireNoUndurableSwitch(core, indexName, entry) {
|
|
@@ -2373,7 +2528,7 @@ async function buildAndSwitch(scope) {
|
|
|
2373
2528
|
}
|
|
2374
2529
|
emit("completed", partitionsRebuilt);
|
|
2375
2530
|
} catch (error) {
|
|
2376
|
-
emit("failed", partitionsRebuilt,
|
|
2531
|
+
emit("failed", partitionsRebuilt, toError3(error));
|
|
2377
2532
|
throw error;
|
|
2378
2533
|
} finally {
|
|
2379
2534
|
if (core.executor.getManager(indexName) === manager) {
|
|
@@ -2494,7 +2649,7 @@ function resumeAnalysisChange(core, indexName) {
|
|
|
2494
2649
|
status: "failed",
|
|
2495
2650
|
partitionsRebuilt: 0,
|
|
2496
2651
|
partitionCount: entry.partitionCount,
|
|
2497
|
-
error:
|
|
2652
|
+
error: toError3(error)
|
|
2498
2653
|
});
|
|
2499
2654
|
return Promise.reject(error);
|
|
2500
2655
|
}
|
|
@@ -5968,7 +6123,14 @@ async function pagedListingOf(directory, ref) {
|
|
|
5968
6123
|
listed: ref.bytes
|
|
5969
6124
|
});
|
|
5970
6125
|
}
|
|
5971
|
-
return { key: ref.key, path: await directory
|
|
6126
|
+
return { key: ref.key, path: await filePathOf(directory, ref.key), bytes: ref.bytes, checksum: ref.checksum };
|
|
6127
|
+
}
|
|
6128
|
+
function filePathOf(directory, key) {
|
|
6129
|
+
const { pathOf } = directory;
|
|
6130
|
+
if (pathOf === void 0) {
|
|
6131
|
+
return loadFailed(`The storage that holds "${key}" keeps no files, so a paged reader cannot open it`, { key });
|
|
6132
|
+
}
|
|
6133
|
+
return pathOf.call(directory, key);
|
|
5972
6134
|
}
|
|
5973
6135
|
|
|
5974
6136
|
// src/persistence/paged/check-contents.ts
|
|
@@ -7006,7 +7168,10 @@ async function coldFieldOf(checkpoint, field) {
|
|
|
7006
7168
|
const fieldType = checkpoint.manifest.schema[field.fieldPath];
|
|
7007
7169
|
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
7008
7170
|
if (dimension === null) return null;
|
|
7009
|
-
const withPath = async (ref) => ({
|
|
7171
|
+
const withPath = async (ref) => ({
|
|
7172
|
+
ref,
|
|
7173
|
+
path: await filePathOf(directory, ref.key)
|
|
7174
|
+
});
|
|
7010
7175
|
return openColdVectorField({
|
|
7011
7176
|
fieldPath: field.fieldPath,
|
|
7012
7177
|
dimension,
|
|
@@ -7257,6 +7422,8 @@ var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
|
|
|
7257
7422
|
var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
|
|
7258
7423
|
var DIRECTORY_LOCK_ATTEMPTS = 3;
|
|
7259
7424
|
var DIRECTORY_LOCK_SETTLE_MS = 50;
|
|
7425
|
+
var INDEXEDDB_BLOCK_BYTES = 32768;
|
|
7426
|
+
var INDEXEDDB_CACHED_BLOCKS = 64;
|
|
7260
7427
|
var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
|
|
7261
7428
|
var DEFAULT_COMPACTION_THRESHOLD = 12;
|
|
7262
7429
|
var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
|
|
@@ -10449,15 +10616,27 @@ function windowFits(segments, start, windowLength, paged) {
|
|
|
10449
10616
|
if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
|
|
10450
10617
|
return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
|
|
10451
10618
|
}
|
|
10452
|
-
function
|
|
10619
|
+
function balanceOf(segments, start, windowLength) {
|
|
10620
|
+
let largest = 0;
|
|
10621
|
+
let bytes = 0;
|
|
10622
|
+
for (let at = start; at < start + windowLength; at++) {
|
|
10623
|
+
largest = Math.max(largest, segments[at].bytes);
|
|
10624
|
+
bytes += segments[at].bytes;
|
|
10625
|
+
}
|
|
10626
|
+
return bytes === 0 ? { skew: 1, bytes: 0 } : { skew: largest / bytes, bytes };
|
|
10627
|
+
}
|
|
10628
|
+
function balancedBefore(balance, other) {
|
|
10629
|
+
if (balance.skew !== other.skew) return balance.skew < other.skew;
|
|
10630
|
+
return balance.bytes < other.bytes;
|
|
10631
|
+
}
|
|
10632
|
+
function mostBalancedFittingWindow(segments, windowLength, paged) {
|
|
10453
10633
|
let best = null;
|
|
10454
|
-
let
|
|
10634
|
+
let bestBalance = null;
|
|
10455
10635
|
for (let start = 0; start + windowLength <= segments.length; start++) {
|
|
10456
10636
|
if (!windowFits(segments, start, windowLength, paged)) continue;
|
|
10457
|
-
|
|
10458
|
-
|
|
10459
|
-
|
|
10460
|
-
bestBytes = bytes;
|
|
10637
|
+
const balance = balanceOf(segments, start, windowLength);
|
|
10638
|
+
if (bestBalance === null || balancedBefore(balance, bestBalance)) {
|
|
10639
|
+
bestBalance = balance;
|
|
10461
10640
|
best = { start, windowLength };
|
|
10462
10641
|
}
|
|
10463
10642
|
}
|
|
@@ -10465,7 +10644,7 @@ function smallestFittingWindow(segments, windowLength, paged) {
|
|
|
10465
10644
|
}
|
|
10466
10645
|
function chooseWindow(segments, wantedLength, paged) {
|
|
10467
10646
|
for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
|
|
10468
|
-
const window =
|
|
10647
|
+
const window = mostBalancedFittingWindow(segments, windowLength, paged);
|
|
10469
10648
|
if (window !== null) return window;
|
|
10470
10649
|
}
|
|
10471
10650
|
return null;
|
|
@@ -10651,8 +10830,14 @@ async function documentsReplacedLater(store, segments) {
|
|
|
10651
10830
|
function exceedsRemovedShare(removed, held) {
|
|
10652
10831
|
return held > 0 && removed > held * REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE;
|
|
10653
10832
|
}
|
|
10654
|
-
function
|
|
10655
|
-
return ref.docCount === 0 ? 0 :
|
|
10833
|
+
function removedShareOf(ref, replacedCount) {
|
|
10834
|
+
return ref.docCount === 0 ? 0 : replacedCount / ref.docCount;
|
|
10835
|
+
}
|
|
10836
|
+
function rewriteOrder(segments, replaced, left, right) {
|
|
10837
|
+
const leftShare = removedShareOf(segments[left], replaced[left].length);
|
|
10838
|
+
const rightShare = removedShareOf(segments[right], replaced[right].length);
|
|
10839
|
+
if (leftShare !== rightShare) return leftShare > rightShare ? -1 : 1;
|
|
10840
|
+
return segments[left].bytes - segments[right].bytes;
|
|
10656
10841
|
}
|
|
10657
10842
|
function chooseRewrites(segments, replaced, budget) {
|
|
10658
10843
|
let held = 0;
|
|
@@ -10663,9 +10848,7 @@ function chooseRewrites(segments, replaced, budget) {
|
|
|
10663
10848
|
}
|
|
10664
10849
|
const candidates = [];
|
|
10665
10850
|
for (let at = 0; at < segments.length; at++) if (replaced[at].length > 0) candidates.push(at);
|
|
10666
|
-
candidates.sort(
|
|
10667
|
-
(left, right) => freedBytesOf(segments[right], replaced[right].length) - freedBytesOf(segments[left], replaced[left].length)
|
|
10668
|
-
);
|
|
10851
|
+
candidates.sort((left, right) => rewriteOrder(segments, replaced, left, right));
|
|
10669
10852
|
const chosen = [];
|
|
10670
10853
|
for (const at of candidates) {
|
|
10671
10854
|
if (!exceedsRemovedShare(removed, held)) break;
|
|
@@ -11792,7 +11975,7 @@ async function headOf(path, ref) {
|
|
|
11792
11975
|
return decode(head);
|
|
11793
11976
|
}
|
|
11794
11977
|
async function levelsOfPart(directory, ref) {
|
|
11795
|
-
const path = await directory
|
|
11978
|
+
const path = await filePathOf(directory, ref.key);
|
|
11796
11979
|
const head = await headOf(path, ref);
|
|
11797
11980
|
const graphs = head?.graphs;
|
|
11798
11981
|
if (!Array.isArray(graphs) || graphs.length !== 1) return [];
|
|
@@ -13762,6 +13945,10 @@ async function getFs() {
|
|
|
13762
13945
|
async function getPath() {
|
|
13763
13946
|
return filesystemUnavailable();
|
|
13764
13947
|
}
|
|
13948
|
+
async function renameReplacing(_from, _to) {
|
|
13949
|
+
return filesystemUnavailable();
|
|
13950
|
+
}
|
|
13951
|
+
var renameUnlessMissing = renameReplacing;
|
|
13765
13952
|
|
|
13766
13953
|
// src/persistence/durability/lock-holder.browser.ts
|
|
13767
13954
|
async function holderIsLive(_holder, _self, _lockPath) {
|
|
@@ -14154,7 +14341,7 @@ function createDurableDirectory(root) {
|
|
|
14154
14341
|
}
|
|
14155
14342
|
await handle.close();
|
|
14156
14343
|
handle = null;
|
|
14157
|
-
await
|
|
14344
|
+
await renameReplacing(tempPath, filePath);
|
|
14158
14345
|
await fsyncDirectory(dir, fs);
|
|
14159
14346
|
} catch (err) {
|
|
14160
14347
|
if (handle !== null) {
|
|
@@ -14241,7 +14428,7 @@ function createDurableDirectory(root) {
|
|
|
14241
14428
|
const target = await resolve(to);
|
|
14242
14429
|
const fs = await getFs();
|
|
14243
14430
|
await ensureDir(target.path, target.pathMod, fs);
|
|
14244
|
-
await
|
|
14431
|
+
await renameReplacing(source.path, target.path);
|
|
14245
14432
|
const directories = new Set([target.path, source.path].map((path) => target.pathMod.dirname(path)));
|
|
14246
14433
|
for (const movedIn of directories) await fsyncDirectory(movedIn, fs);
|
|
14247
14434
|
},
|
|
@@ -14664,6 +14851,11 @@ async function createMarkerWriter(directory, indexName, partitionId) {
|
|
|
14664
14851
|
// src/persistence/durability/recovery.ts
|
|
14665
14852
|
async function removeLeftoverRewriteFiles(directory, names) {
|
|
14666
14853
|
for (const indexName of names) {
|
|
14854
|
+
const prefix = `${indexName}/`;
|
|
14855
|
+
for (const key of await directory.list(prefix)) {
|
|
14856
|
+
const relative = key.slice(prefix.length);
|
|
14857
|
+
if (!relative.includes("/") && /^(meta|manifest)\.tmp-/.test(relative)) await directory.remove(key);
|
|
14858
|
+
}
|
|
14667
14859
|
for (const place of ["rebuilt", "received"]) {
|
|
14668
14860
|
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
14669
14861
|
}
|
|
@@ -15809,10 +16001,10 @@ function createLogBytesLedger(base, sizes) {
|
|
|
15809
16001
|
const moved = untrack(from);
|
|
15810
16002
|
if (loggedFileOf(to) !== null) track(to, moved?.bytes ?? await currentLength(to));
|
|
15811
16003
|
},
|
|
15812
|
-
list: (prefix) => base.list(prefix)
|
|
15813
|
-
pathOf: (key) => base.pathOf(key)
|
|
16004
|
+
list: (prefix) => base.list(prefix)
|
|
15814
16005
|
};
|
|
15815
|
-
const { readRange, byteLength, identity, lock } = base;
|
|
16006
|
+
const { readRange, byteLength, identity, lock, pathOf } = base;
|
|
16007
|
+
if (pathOf !== void 0) directory.pathOf = (key) => pathOf.call(base, key);
|
|
15816
16008
|
if (readRange !== void 0) directory.readRange = (key, offset, length) => readRange.call(base, key, offset, length);
|
|
15817
16009
|
if (byteLength !== void 0) directory.byteLength = (key) => byteLength.call(base, key);
|
|
15818
16010
|
if (identity !== void 0) directory.identity = () => identity.call(base);
|
|
@@ -16000,21 +16192,27 @@ function storageFailed(code, message, key, error) {
|
|
|
16000
16192
|
return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
|
|
16001
16193
|
}
|
|
16002
16194
|
function directoryRestoreStorage(directory) {
|
|
16003
|
-
|
|
16195
|
+
const storage = {
|
|
16004
16196
|
directory,
|
|
16005
16197
|
read: (key) => directory.read(key),
|
|
16006
16198
|
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
16007
16199
|
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
16008
16200
|
remove: (key) => directory.remove(key),
|
|
16009
16201
|
list: (prefix) => directory.list(prefix),
|
|
16010
|
-
moveKey: (from, to) => directory.move(from, to)
|
|
16011
|
-
|
|
16202
|
+
moveKey: (from, to) => directory.move(from, to)
|
|
16203
|
+
};
|
|
16204
|
+
const { pathOf } = directory;
|
|
16205
|
+
if (pathOf === void 0) return storage;
|
|
16206
|
+
return { ...storage, ...fileTreeOperations(directory, (key) => pathOf.call(directory, key)) };
|
|
16207
|
+
}
|
|
16208
|
+
function fileTreeOperations(directory, pathOf) {
|
|
16209
|
+
return {
|
|
16210
|
+
pathOf,
|
|
16012
16211
|
async moveTree(from, to) {
|
|
16013
|
-
const source = await
|
|
16014
|
-
const target = await
|
|
16015
|
-
const fs = await getFs();
|
|
16212
|
+
const source = await pathOf(from);
|
|
16213
|
+
const target = await pathOf(to);
|
|
16016
16214
|
try {
|
|
16017
|
-
await
|
|
16215
|
+
await renameUnlessMissing(source, target);
|
|
16018
16216
|
} catch (error) {
|
|
16019
16217
|
if (missing(error)) return;
|
|
16020
16218
|
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
@@ -16025,7 +16223,7 @@ function directoryRestoreStorage(directory) {
|
|
|
16025
16223
|
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
16026
16224
|
const fs = await getFs();
|
|
16027
16225
|
try {
|
|
16028
|
-
await fs.rm(await
|
|
16226
|
+
await fs.rm(await pathOf(key), { recursive: true, force: true });
|
|
16029
16227
|
} catch (error) {
|
|
16030
16228
|
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
16031
16229
|
}
|
|
@@ -16038,7 +16236,7 @@ function recoverDirectoryRestores(directory) {
|
|
|
16038
16236
|
}
|
|
16039
16237
|
|
|
16040
16238
|
// src/persistence/durability/restore-staging.ts
|
|
16041
|
-
function
|
|
16239
|
+
function toError4(error) {
|
|
16042
16240
|
return error instanceof Error ? error : new Error(String(error));
|
|
16043
16241
|
}
|
|
16044
16242
|
function pathUnder(path, root) {
|
|
@@ -16047,7 +16245,7 @@ function pathUnder(path, root) {
|
|
|
16047
16245
|
return next === "" || next === "/" || next === "\\";
|
|
16048
16246
|
}
|
|
16049
16247
|
function stagedDirectory(directory, staged, unstaged) {
|
|
16050
|
-
const { readRange, byteLength } = directory;
|
|
16248
|
+
const { readRange, byteLength, pathOf } = directory;
|
|
16051
16249
|
return {
|
|
16052
16250
|
root: directory.root,
|
|
16053
16251
|
appendHandle: (key) => directory.appendHandle(staged(key)),
|
|
@@ -16058,7 +16256,7 @@ function stagedDirectory(directory, staged, unstaged) {
|
|
|
16058
16256
|
remove: (key) => directory.remove(staged(key)),
|
|
16059
16257
|
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
16060
16258
|
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
16061
|
-
pathOf: (key) =>
|
|
16259
|
+
...pathOf === void 0 ? {} : { pathOf: (key) => pathOf.call(directory, staged(key)) },
|
|
16062
16260
|
...readRange === void 0 ? {} : {
|
|
16063
16261
|
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
16064
16262
|
},
|
|
@@ -16107,7 +16305,7 @@ function createRestoreStaging(storage, indexName, options = {}) {
|
|
|
16107
16305
|
await storage.remove(journalKey(indexName));
|
|
16108
16306
|
} catch (removal) {
|
|
16109
16307
|
committed = true;
|
|
16110
|
-
options.onStorageLost?.(
|
|
16308
|
+
options.onStorageLost?.(toError4(removal));
|
|
16111
16309
|
}
|
|
16112
16310
|
throw error;
|
|
16113
16311
|
}
|
|
@@ -16117,7 +16315,7 @@ function createRestoreStaging(storage, indexName, options = {}) {
|
|
|
16117
16315
|
try {
|
|
16118
16316
|
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
16119
16317
|
} catch (error) {
|
|
16120
|
-
options.onStorageLost?.(
|
|
16318
|
+
options.onStorageLost?.(toError4(error));
|
|
16121
16319
|
}
|
|
16122
16320
|
},
|
|
16123
16321
|
async removeRetired() {
|
|
@@ -16149,7 +16347,7 @@ function adapterRestoreStorage(adapter) {
|
|
|
16149
16347
|
}
|
|
16150
16348
|
|
|
16151
16349
|
// src/persistence/durability/archive-checkpoint.ts
|
|
16152
|
-
function
|
|
16350
|
+
function toError5(error) {
|
|
16153
16351
|
return error instanceof Error ? error : new Error(String(error));
|
|
16154
16352
|
}
|
|
16155
16353
|
function stageDirectoryRestore(deps, indexName) {
|
|
@@ -16158,7 +16356,7 @@ function stageDirectoryRestore(deps, indexName) {
|
|
|
16158
16356
|
try {
|
|
16159
16357
|
await run();
|
|
16160
16358
|
} catch (error) {
|
|
16161
|
-
deps.markFatal(
|
|
16359
|
+
deps.markFatal(toError5(error));
|
|
16162
16360
|
throw error;
|
|
16163
16361
|
}
|
|
16164
16362
|
},
|
|
@@ -16815,6 +17013,18 @@ function createIndexStorageAccess(deps) {
|
|
|
16815
17013
|
}
|
|
16816
17014
|
|
|
16817
17015
|
// src/persistence/durability/metadata-write-queue.ts
|
|
17016
|
+
async function writeIndexMetadataFile(directory, indexName, bytes) {
|
|
17017
|
+
try {
|
|
17018
|
+
await directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
17019
|
+
} catch (error) {
|
|
17020
|
+
if (error instanceof NarsilError) throw error;
|
|
17021
|
+
throw new NarsilError(
|
|
17022
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
17023
|
+
`The engine cannot write the metadata of index "${indexName}" to the durability directory "${directory.root}"`,
|
|
17024
|
+
{ indexName, directory: directory.root, cause: error instanceof Error ? error.message : String(error) }
|
|
17025
|
+
);
|
|
17026
|
+
}
|
|
17027
|
+
}
|
|
16818
17028
|
function createMetadataWriteQueue() {
|
|
16819
17029
|
const writes = /* @__PURE__ */ new Map();
|
|
16820
17030
|
return {
|
|
@@ -16830,201 +17040,59 @@ function createMetadataWriteQueue() {
|
|
|
16830
17040
|
pending: (indexName) => writes.get(indexName),
|
|
16831
17041
|
forget(indexName) {
|
|
16832
17042
|
writes.delete(indexName);
|
|
17043
|
+
},
|
|
17044
|
+
async settleAll() {
|
|
17045
|
+
let waiting = [...writes.values()];
|
|
17046
|
+
while (waiting.length > 0) {
|
|
17047
|
+
await Promise.all(waiting);
|
|
17048
|
+
const settled = waiting;
|
|
17049
|
+
waiting = [...writes.values()].filter((write) => !settled.includes(write));
|
|
17050
|
+
}
|
|
16833
17051
|
}
|
|
16834
17052
|
};
|
|
16835
17053
|
}
|
|
16836
17054
|
|
|
16837
|
-
// src/persistence/durability/mutation-
|
|
16838
|
-
function
|
|
16839
|
-
return
|
|
16840
|
-
|
|
16841
|
-
|
|
16842
|
-
|
|
16843
|
-
|
|
16844
|
-
|
|
16845
|
-
|
|
16846
|
-
|
|
16847
|
-
|
|
17055
|
+
// src/persistence/durability/mutation-entry.ts
|
|
17056
|
+
function buildMutationEntry(record, seqNo) {
|
|
17057
|
+
return buildEntry({
|
|
17058
|
+
seqNo,
|
|
17059
|
+
primaryTerm: SINGLE_NODE_PRIMARY_TERM,
|
|
17060
|
+
operation: record.operation,
|
|
17061
|
+
partitionId: record.partitionId,
|
|
17062
|
+
indexName: record.indexName,
|
|
17063
|
+
documentId: record.documentId,
|
|
17064
|
+
document: record.document,
|
|
17065
|
+
previousDocument: record.previousDocument
|
|
17066
|
+
});
|
|
16848
17067
|
}
|
|
16849
|
-
|
|
16850
|
-
|
|
16851
|
-
|
|
16852
|
-
|
|
16853
|
-
|
|
17068
|
+
|
|
17069
|
+
// src/persistence/durability/partition-failure.ts
|
|
17070
|
+
function createPartitionFailureRouting() {
|
|
17071
|
+
let onFailed = null;
|
|
17072
|
+
return {
|
|
17073
|
+
isolate(handler) {
|
|
17074
|
+
onFailed = handler;
|
|
17075
|
+
},
|
|
17076
|
+
failsAlone(indexName, partitionId, error) {
|
|
17077
|
+
if (onFailed === null) return false;
|
|
17078
|
+
onFailed(indexName, partitionId, error);
|
|
17079
|
+
return true;
|
|
16854
17080
|
}
|
|
16855
|
-
}
|
|
16856
|
-
}
|
|
16857
|
-
function failPartition(partition, address, deps, err) {
|
|
16858
|
-
const error = toError5(err);
|
|
16859
|
-
if (partition.failed !== null) return partition.failed;
|
|
16860
|
-
partition.failed = error;
|
|
16861
|
-
if (!deps.failsPartitionAlone(address.indexName, address.partitionId, error)) deps.markFatal(error);
|
|
16862
|
-
return error;
|
|
17081
|
+
};
|
|
16863
17082
|
}
|
|
16864
|
-
|
|
16865
|
-
|
|
16866
|
-
|
|
16867
|
-
|
|
16868
|
-
|
|
16869
|
-
|
|
16870
|
-
|
|
16871
|
-
|
|
16872
|
-
|
|
16873
|
-
|
|
16874
|
-
|
|
16875
|
-
}
|
|
16876
|
-
|
|
16877
|
-
return (partitionId, partition, error) => failPartition(partition, { indexName, partitionId }, deps, error);
|
|
16878
|
-
}
|
|
16879
|
-
async function recordBatchesByPartition(records, partitionOf, deps) {
|
|
16880
|
-
const positionsByPartition = /* @__PURE__ */ new Map();
|
|
16881
|
-
for (let i = 0; i < records.length; i++) {
|
|
16882
|
-
const partition = partitionOf(records[i]);
|
|
16883
|
-
const positions = positionsByPartition.get(partition);
|
|
16884
|
-
if (positions === void 0) {
|
|
16885
|
-
positionsByPartition.set(partition, [i]);
|
|
16886
|
-
} else {
|
|
16887
|
-
positions.push(i);
|
|
16888
|
-
}
|
|
16889
|
-
}
|
|
16890
|
-
const outcomes = new Array(records.length);
|
|
16891
|
-
await Promise.all(
|
|
16892
|
-
[...positionsByPartition].map(async ([partition, positions]) => {
|
|
16893
|
-
const partitionOutcomes = await recordPartitionBatch(
|
|
16894
|
-
partition,
|
|
16895
|
-
positions.map((position) => records[position]),
|
|
16896
|
-
deps
|
|
16897
|
-
);
|
|
16898
|
-
for (let i = 0; i < positions.length; i++) {
|
|
16899
|
-
outcomes[positions[i]] = partitionOutcomes[i];
|
|
16900
|
-
}
|
|
16901
|
-
})
|
|
16902
|
-
);
|
|
16903
|
-
return outcomes;
|
|
16904
|
-
}
|
|
16905
|
-
async function recordPartitionBatch(partition, records, deps) {
|
|
16906
|
-
const outcomes = new Array(records.length);
|
|
16907
|
-
let appendedCount = 0;
|
|
16908
|
-
let commitsHere = false;
|
|
16909
|
-
const appended = partition.appendChain.then(async () => {
|
|
16910
|
-
const fatal = deps.fatalError();
|
|
16911
|
-
if (fatal !== null) {
|
|
16912
|
-
throw fatal;
|
|
16913
|
-
}
|
|
16914
|
-
if (partition.failed !== null) {
|
|
16915
|
-
throw partition.failed;
|
|
16916
|
-
}
|
|
16917
|
-
const applied = [];
|
|
16918
|
-
for (let i = 0; i < records.length; i++) {
|
|
16919
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
16920
|
-
try {
|
|
16921
|
-
await records[i].apply();
|
|
16922
|
-
applied.push(i);
|
|
16923
|
-
} catch (err) {
|
|
16924
|
-
outcomes[i] = { ok: false, error: err };
|
|
16925
|
-
}
|
|
16926
|
-
}
|
|
16927
|
-
if (applied.length === 0) {
|
|
16928
|
-
return;
|
|
16929
|
-
}
|
|
16930
|
-
let lastSeqNo = partition.appliedSeqNo;
|
|
16931
|
-
const entries = [];
|
|
16932
|
-
try {
|
|
16933
|
-
for (const i of applied) {
|
|
16934
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
16935
|
-
let numbered;
|
|
16936
|
-
try {
|
|
16937
|
-
numbered = deps.numberEntry(records[i]);
|
|
16938
|
-
} catch (err) {
|
|
16939
|
-
outcomes[i] = { ok: false, error: err };
|
|
16940
|
-
continue;
|
|
16941
|
-
}
|
|
16942
|
-
if (numbered === null) {
|
|
16943
|
-
lastSeqNo = partition.seqOwner.next();
|
|
16944
|
-
entries.push(deps.buildEntry(records[i], lastSeqNo));
|
|
16945
|
-
} else {
|
|
16946
|
-
requireAscendingSeqNo(records[i], numbered.seqNo, lastSeqNo);
|
|
16947
|
-
partition.seqOwner.advanceTo(numbered.seqNo, numbered.primaryTerm);
|
|
16948
|
-
lastSeqNo = numbered.seqNo;
|
|
16949
|
-
entries.push(numbered);
|
|
16950
|
-
}
|
|
16951
|
-
outcomes[i] = { ok: true, seqNo: lastSeqNo };
|
|
16952
|
-
if (!deps.defersCommit(records[i])) commitsHere = true;
|
|
16953
|
-
}
|
|
16954
|
-
if (entries.length > 0) await partition.walWriter.appendAll(entries);
|
|
16955
|
-
} catch (err) {
|
|
16956
|
-
throw failPartition(partition, records[0], deps, err);
|
|
16957
|
-
}
|
|
16958
|
-
partition.appliedSeqNo = lastSeqNo;
|
|
16959
|
-
appendedCount = entries.length;
|
|
16960
|
-
});
|
|
16961
|
-
partition.appendChain = appended.catch(() => void 0);
|
|
16962
|
-
try {
|
|
16963
|
-
await appended;
|
|
16964
|
-
if (deps.syncEachBatch && appendedCount > 0 && commitsHere) {
|
|
16965
|
-
try {
|
|
16966
|
-
await partition.walWriter.commit();
|
|
16967
|
-
} catch (err) {
|
|
16968
|
-
throw failPartition(partition, records[0], deps, err);
|
|
16969
|
-
}
|
|
16970
|
-
}
|
|
16971
|
-
} catch (err) {
|
|
16972
|
-
failUnlessAlreadyFailed(outcomes, records.length, toError5(err));
|
|
16973
|
-
}
|
|
16974
|
-
return outcomes;
|
|
16975
|
-
}
|
|
16976
|
-
async function commitAppended(partition, address, deps) {
|
|
16977
|
-
await partition.appendChain;
|
|
16978
|
-
const failed = partition.failed ?? deps.fatalError();
|
|
16979
|
-
if (failed !== null) throw failed;
|
|
16980
|
-
try {
|
|
16981
|
-
await partition.walWriter.commit();
|
|
16982
|
-
} catch (err) {
|
|
16983
|
-
throw failPartition(partition, address, deps, err);
|
|
16984
|
-
}
|
|
16985
|
-
}
|
|
16986
|
-
|
|
16987
|
-
// src/persistence/durability/mutation-entry.ts
|
|
16988
|
-
function buildMutationEntry(record, seqNo) {
|
|
16989
|
-
return buildEntry({
|
|
16990
|
-
seqNo,
|
|
16991
|
-
primaryTerm: SINGLE_NODE_PRIMARY_TERM,
|
|
16992
|
-
operation: record.operation,
|
|
16993
|
-
partitionId: record.partitionId,
|
|
16994
|
-
indexName: record.indexName,
|
|
16995
|
-
documentId: record.documentId,
|
|
16996
|
-
document: record.document,
|
|
16997
|
-
previousDocument: record.previousDocument
|
|
16998
|
-
});
|
|
16999
|
-
}
|
|
17000
|
-
|
|
17001
|
-
// src/persistence/durability/partition-failure.ts
|
|
17002
|
-
function createPartitionFailureRouting() {
|
|
17003
|
-
let onFailed = null;
|
|
17004
|
-
return {
|
|
17005
|
-
isolate(handler) {
|
|
17006
|
-
onFailed = handler;
|
|
17007
|
-
},
|
|
17008
|
-
failsAlone(indexName, partitionId, error) {
|
|
17009
|
-
if (onFailed === null) return false;
|
|
17010
|
-
onFailed(indexName, partitionId, error);
|
|
17011
|
-
return true;
|
|
17012
|
-
}
|
|
17013
|
-
};
|
|
17014
|
-
}
|
|
17015
|
-
|
|
17016
|
-
// src/persistence/durability/group-commit.ts
|
|
17017
|
-
function createGroupCommitCoordinator(sync) {
|
|
17018
|
-
let inFlight = null;
|
|
17019
|
-
let next = null;
|
|
17020
|
-
function makeBatch() {
|
|
17021
|
-
let resolve = () => void 0;
|
|
17022
|
-
let reject = () => void 0;
|
|
17023
|
-
const promise = new Promise((res, rej) => {
|
|
17024
|
-
resolve = res;
|
|
17025
|
-
reject = rej;
|
|
17026
|
-
});
|
|
17027
|
-
return { promise, resolve, reject, waiters: 0 };
|
|
17083
|
+
|
|
17084
|
+
// src/persistence/durability/group-commit.ts
|
|
17085
|
+
function createGroupCommitCoordinator(sync) {
|
|
17086
|
+
let inFlight = null;
|
|
17087
|
+
let next = null;
|
|
17088
|
+
function makeBatch() {
|
|
17089
|
+
let resolve = () => void 0;
|
|
17090
|
+
let reject = () => void 0;
|
|
17091
|
+
const promise = new Promise((res, rej) => {
|
|
17092
|
+
resolve = res;
|
|
17093
|
+
reject = rej;
|
|
17094
|
+
});
|
|
17095
|
+
return { promise, resolve, reject, waiters: 0 };
|
|
17028
17096
|
}
|
|
17029
17097
|
function drain() {
|
|
17030
17098
|
if (inFlight !== null || next === null) {
|
|
@@ -18349,7 +18417,7 @@ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkp
|
|
|
18349
18417
|
// src/persistence/durability/manager.ts
|
|
18350
18418
|
function createDurabilityManager(config, hooks, directoryOverride) {
|
|
18351
18419
|
const sizes = createStoredSizesRecord(true);
|
|
18352
|
-
const logBytes = createLogBytesLedger(createDurableDirectory(config.directory ?? ""), sizes);
|
|
18420
|
+
const logBytes = createLogBytesLedger(directoryOverride ?? createDurableDirectory(config.directory ?? ""), sizes);
|
|
18353
18421
|
const { directory } = logBytes;
|
|
18354
18422
|
const canOffloadCheckpoint = directoryOverride === void 0;
|
|
18355
18423
|
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
@@ -18473,10 +18541,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
18473
18541
|
defersCommit: (record) => commitDeferral?.(record) === true
|
|
18474
18542
|
};
|
|
18475
18543
|
async function recordMutations(records) {
|
|
18476
|
-
|
|
18477
|
-
|
|
18478
|
-
return records.map(() => ({ ok: false, error }));
|
|
18479
|
-
}
|
|
18544
|
+
const refusal = fatalError ?? (shuttingDown ? shutDownRefusal() : null);
|
|
18545
|
+
if (refusal !== null) return records.map(() => ({ ok: false, error: refusal }));
|
|
18480
18546
|
const outcomes = await recordBatchesByPartition(
|
|
18481
18547
|
records,
|
|
18482
18548
|
(record) => getOrCreatePartition(record.indexName, record.partitionId, 0),
|
|
@@ -18591,7 +18657,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
18591
18657
|
return;
|
|
18592
18658
|
}
|
|
18593
18659
|
const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
|
|
18594
|
-
await directory
|
|
18660
|
+
await writeIndexMetadataFile(directory, indexName, bytes);
|
|
18595
18661
|
getOrCreateIndexState(indexName);
|
|
18596
18662
|
});
|
|
18597
18663
|
},
|
|
@@ -18628,14 +18694,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
18628
18694
|
async shutdown() {
|
|
18629
18695
|
shuttingDown = true;
|
|
18630
18696
|
stopTimers();
|
|
18697
|
+
for (const indexState of indexes.values()) indexState.unloading = true;
|
|
18631
18698
|
for (const indexState of indexes.values()) {
|
|
18632
18699
|
if (indexState.checkpointInFlight !== null) {
|
|
18633
18700
|
await indexState.checkpointInFlight.catch(() => void 0);
|
|
18634
18701
|
}
|
|
18635
18702
|
for (const partition of indexState.partitions.values()) {
|
|
18703
|
+
await partition.appendChain;
|
|
18636
18704
|
await closeWriterReportingFailure(closePartitionWriter(partition));
|
|
18637
18705
|
}
|
|
18638
18706
|
}
|
|
18707
|
+
await metadataWrites.settleAll();
|
|
18639
18708
|
indexes.clear();
|
|
18640
18709
|
terminateCheckpointWorker();
|
|
18641
18710
|
await directoryWatch.release();
|
|
@@ -18643,69 +18712,1127 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
18643
18712
|
};
|
|
18644
18713
|
}
|
|
18645
18714
|
|
|
18646
|
-
// src/persistence/
|
|
18647
|
-
|
|
18648
|
-
|
|
18649
|
-
|
|
18650
|
-
|
|
18651
|
-
|
|
18652
|
-
|
|
18653
|
-
|
|
18654
|
-
};
|
|
18655
|
-
}
|
|
18656
|
-
function metadataKey(indexName) {
|
|
18657
|
-
return `${indexName}/meta`;
|
|
18715
|
+
// src/persistence/constants.ts
|
|
18716
|
+
var INDEXEDDB_BLOCKED_TIMEOUT_MS = 5e3;
|
|
18717
|
+
|
|
18718
|
+
// src/persistence/durability/indexeddb-directory/connection.ts
|
|
18719
|
+
function requestResult(request) {
|
|
18720
|
+
return new Promise((resolve, reject) => {
|
|
18721
|
+
request.onsuccess = () => resolve(request.result);
|
|
18722
|
+
request.onerror = () => reject(request.error);
|
|
18723
|
+
});
|
|
18658
18724
|
}
|
|
18659
|
-
function
|
|
18660
|
-
|
|
18661
|
-
|
|
18662
|
-
|
|
18663
|
-
|
|
18664
|
-
|
|
18665
|
-
|
|
18666
|
-
|
|
18725
|
+
function causeOf(error) {
|
|
18726
|
+
if (error instanceof Error || typeof error === "object" && error !== null && "message" in error) {
|
|
18727
|
+
return String(error.message);
|
|
18728
|
+
}
|
|
18729
|
+
return String(error);
|
|
18730
|
+
}
|
|
18731
|
+
function rootOf(location) {
|
|
18732
|
+
return `indexeddb:${location.databaseName}/${location.storeName}`;
|
|
18733
|
+
}
|
|
18734
|
+
function openLogConnection(location) {
|
|
18735
|
+
const root = rootOf(location);
|
|
18736
|
+
let opening = null;
|
|
18737
|
+
let lostReason = null;
|
|
18738
|
+
let closedByEngine = false;
|
|
18739
|
+
const token = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
18740
|
+
function storeFailed(failure, cause) {
|
|
18741
|
+
return new NarsilError(failure.code, `IndexedDB could not ${failure.action} in ${root}: ${cause}`, {
|
|
18742
|
+
directory: root,
|
|
18743
|
+
cause
|
|
18744
|
+
});
|
|
18667
18745
|
}
|
|
18668
|
-
|
|
18669
|
-
|
|
18670
|
-
|
|
18671
|
-
|
|
18672
|
-
|
|
18673
|
-
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
18674
|
-
const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
|
|
18675
|
-
const targets = [];
|
|
18676
|
-
const contents = [];
|
|
18677
|
-
const positions = /* @__PURE__ */ new Map();
|
|
18678
|
-
for (let partitionId = 0; partitionId < input.manager.partitionCount; partitionId += 1) {
|
|
18679
|
-
const lastSeqNo = Math.min(
|
|
18680
|
-
input.appliedSeqNoByPartition.get(partitionId) ?? 0,
|
|
18681
|
-
cutoffs.get(partitionId) ?? Infinity
|
|
18746
|
+
function replaced() {
|
|
18747
|
+
return new NarsilError(
|
|
18748
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
18749
|
+
`The engine treats ${root} as replaced, because ${lostReason ?? "the engine closed it"}, so it acknowledges no later write`,
|
|
18750
|
+
{ directory: root }
|
|
18682
18751
|
);
|
|
18683
|
-
positions.set(partitionId, lastSeqNo);
|
|
18684
|
-
const unchanged = !input.rewritesEveryPartition && prior !== null && !partitionCountChanged && listedIds.has(partitionId) && checkpointLastSeqNo(prior.checkpoint, partitionId) === lastSeqNo;
|
|
18685
|
-
if (unchanged) continue;
|
|
18686
|
-
targets.push({ partitionId, lastSeqNo, primaryTerm: SINGLE_NODE_PRIMARY_TERM });
|
|
18687
|
-
contents.push(partitionContentsOf(input.manager, partitionId));
|
|
18688
18752
|
}
|
|
18689
|
-
|
|
18690
|
-
|
|
18691
|
-
|
|
18692
|
-
|
|
18693
|
-
|
|
18694
|
-
|
|
18695
|
-
|
|
18753
|
+
function watch(database2) {
|
|
18754
|
+
database2.onclose = () => {
|
|
18755
|
+
lostReason ??= "the browser closed the connection";
|
|
18756
|
+
};
|
|
18757
|
+
database2.onversionchange = () => {
|
|
18758
|
+
lostReason ??= "another connection asked for a version change of the database";
|
|
18759
|
+
database2.close();
|
|
18760
|
+
};
|
|
18761
|
+
}
|
|
18762
|
+
function open() {
|
|
18763
|
+
const factory = globalThis.indexedDB;
|
|
18764
|
+
if (factory === void 0) {
|
|
18765
|
+
return Promise.reject(
|
|
18766
|
+
new NarsilError(ErrorCodes.CONFIG_INVALID, "IndexedDB is not available, so the write-ahead log cannot open")
|
|
18696
18767
|
);
|
|
18697
18768
|
}
|
|
18698
|
-
return {
|
|
18699
|
-
|
|
18700
|
-
|
|
18701
|
-
|
|
18702
|
-
|
|
18703
|
-
|
|
18704
|
-
|
|
18705
|
-
|
|
18706
|
-
|
|
18707
|
-
|
|
18708
|
-
|
|
18769
|
+
return new Promise((resolve, reject) => {
|
|
18770
|
+
const request = factory.open(location.databaseName);
|
|
18771
|
+
let blockedTimer = null;
|
|
18772
|
+
const settle = () => {
|
|
18773
|
+
if (blockedTimer !== null) clearTimeout(blockedTimer);
|
|
18774
|
+
};
|
|
18775
|
+
request.onblocked = () => {
|
|
18776
|
+
blockedTimer = setTimeout(
|
|
18777
|
+
() => reject(storeFailed({ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open" }, "the open is blocked")),
|
|
18778
|
+
INDEXEDDB_BLOCKED_TIMEOUT_MS
|
|
18779
|
+
);
|
|
18780
|
+
};
|
|
18781
|
+
request.onupgradeneeded = () => {
|
|
18782
|
+
const database2 = request.result;
|
|
18783
|
+
if (!database2.objectStoreNames.contains(location.storeName)) database2.createObjectStore(location.storeName);
|
|
18784
|
+
};
|
|
18785
|
+
request.onsuccess = () => {
|
|
18786
|
+
settle();
|
|
18787
|
+
const database2 = request.result;
|
|
18788
|
+
if (!database2.objectStoreNames.contains(location.storeName)) {
|
|
18789
|
+
database2.close();
|
|
18790
|
+
reject(
|
|
18791
|
+
storeFailed(
|
|
18792
|
+
{ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open" },
|
|
18793
|
+
`the database holds no object store named "${location.storeName}"`
|
|
18794
|
+
)
|
|
18795
|
+
);
|
|
18796
|
+
return;
|
|
18797
|
+
}
|
|
18798
|
+
watch(database2);
|
|
18799
|
+
resolve(database2);
|
|
18800
|
+
};
|
|
18801
|
+
request.onerror = () => {
|
|
18802
|
+
settle();
|
|
18803
|
+
reject(storeFailed({ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open" }, causeOf(request.error)));
|
|
18804
|
+
};
|
|
18805
|
+
});
|
|
18806
|
+
}
|
|
18807
|
+
async function database() {
|
|
18808
|
+
if (closedByEngine || lostReason !== null) throw replaced();
|
|
18809
|
+
if (opening === null) {
|
|
18810
|
+
opening = open();
|
|
18811
|
+
opening.catch(() => {
|
|
18812
|
+
opening = null;
|
|
18813
|
+
});
|
|
18814
|
+
}
|
|
18815
|
+
const opened = await opening;
|
|
18816
|
+
if (lostReason !== null || closedByEngine) throw replaced();
|
|
18817
|
+
return opened;
|
|
18818
|
+
}
|
|
18819
|
+
async function transact(mode, durability, failure, body) {
|
|
18820
|
+
const opened = await database();
|
|
18821
|
+
let transaction;
|
|
18822
|
+
try {
|
|
18823
|
+
transaction = durability === null ? opened.transaction(location.storeName, mode) : opened.transaction(location.storeName, mode, { durability });
|
|
18824
|
+
} catch (error) {
|
|
18825
|
+
if (lostReason === null && error.name === "InvalidStateError") {
|
|
18826
|
+
lostReason = "the browser closed the connection";
|
|
18827
|
+
}
|
|
18828
|
+
throw lostReason === null ? storeFailed(failure, causeOf(error)) : replaced();
|
|
18829
|
+
}
|
|
18830
|
+
const committed = new Promise((resolve, reject) => {
|
|
18831
|
+
transaction.oncomplete = () => resolve();
|
|
18832
|
+
transaction.onabort = () => reject(transaction.error ?? new Error("the transaction aborted"));
|
|
18833
|
+
});
|
|
18834
|
+
let result;
|
|
18835
|
+
try {
|
|
18836
|
+
result = await body(transaction.objectStore(location.storeName));
|
|
18837
|
+
} catch (error) {
|
|
18838
|
+
try {
|
|
18839
|
+
transaction.abort();
|
|
18840
|
+
} catch {
|
|
18841
|
+
}
|
|
18842
|
+
await committed.catch(() => void 0);
|
|
18843
|
+
throw transactionFailed(transaction, failure, error);
|
|
18844
|
+
}
|
|
18845
|
+
try {
|
|
18846
|
+
await committed;
|
|
18847
|
+
} catch (error) {
|
|
18848
|
+
throw transactionFailed(transaction, failure, error);
|
|
18849
|
+
}
|
|
18850
|
+
if (lostReason !== null || closedByEngine) throw replaced();
|
|
18851
|
+
return result;
|
|
18852
|
+
}
|
|
18853
|
+
function transactionFailed(transaction, failure, error) {
|
|
18854
|
+
const aborted = transaction.error?.name === "AbortError" || error instanceof Error && error.name === "AbortError";
|
|
18855
|
+
if (lostReason === null && aborted) {
|
|
18856
|
+
lostReason = "the browser closed the connection";
|
|
18857
|
+
}
|
|
18858
|
+
if (lostReason !== null || closedByEngine) return replaced();
|
|
18859
|
+
return error instanceof NarsilError ? error : storeFailed(failure, causeOf(error));
|
|
18860
|
+
}
|
|
18861
|
+
return {
|
|
18862
|
+
root,
|
|
18863
|
+
read: (failure, body) => transact("readonly", null, failure, body),
|
|
18864
|
+
write: (durability, failure, body) => transact("readwrite", durability, failure, body),
|
|
18865
|
+
async identity() {
|
|
18866
|
+
await database().catch((error) => {
|
|
18867
|
+
if (lostReason === null && !closedByEngine) throw error;
|
|
18868
|
+
});
|
|
18869
|
+
return lostReason === null && !closedByEngine ? token : null;
|
|
18870
|
+
},
|
|
18871
|
+
close() {
|
|
18872
|
+
closedByEngine = true;
|
|
18873
|
+
const pending = opening;
|
|
18874
|
+
opening = null;
|
|
18875
|
+
void pending?.then(
|
|
18876
|
+
(opened) => opened.close(),
|
|
18877
|
+
() => void 0
|
|
18878
|
+
);
|
|
18879
|
+
}
|
|
18880
|
+
};
|
|
18881
|
+
}
|
|
18882
|
+
|
|
18883
|
+
// src/persistence/durability/indexeddb-directory/records.ts
|
|
18884
|
+
function appendKey(key, n) {
|
|
18885
|
+
return [key, n];
|
|
18886
|
+
}
|
|
18887
|
+
function appendRecordsOf(key) {
|
|
18888
|
+
return IDBKeyRange.bound([key], [key, []], true, true);
|
|
18889
|
+
}
|
|
18890
|
+
function compactBytes(bytes) {
|
|
18891
|
+
return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength ? bytes : bytes.slice();
|
|
18892
|
+
}
|
|
18893
|
+
function joinBytes(chunks) {
|
|
18894
|
+
if (chunks.length === 1) return compactBytes(chunks[0]);
|
|
18895
|
+
let total = 0;
|
|
18896
|
+
for (const chunk of chunks) total += chunk.byteLength;
|
|
18897
|
+
const joined4 = new Uint8Array(total);
|
|
18898
|
+
let offset = 0;
|
|
18899
|
+
for (const chunk of chunks) {
|
|
18900
|
+
joined4.set(chunk, offset);
|
|
18901
|
+
offset += chunk.byteLength;
|
|
18902
|
+
}
|
|
18903
|
+
return joined4;
|
|
18904
|
+
}
|
|
18905
|
+
function requireBytes2(key, value) {
|
|
18906
|
+
if (value instanceof Uint8Array) return value;
|
|
18907
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The IndexedDB record of "${key}" holds no bytes`, { key });
|
|
18908
|
+
}
|
|
18909
|
+
function requireAppendNumber(key, recordKey) {
|
|
18910
|
+
const n = Array.isArray(recordKey) && recordKey.length === 2 && recordKey[0] === key ? recordKey[1] : void 0;
|
|
18911
|
+
if (typeof n === "number" && Number.isSafeInteger(n) && n >= 1) return n;
|
|
18912
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `An append record of "${key}" carries no valid position`, {
|
|
18913
|
+
key
|
|
18914
|
+
});
|
|
18915
|
+
}
|
|
18916
|
+
async function readRecords(store, key) {
|
|
18917
|
+
const first = await requestResult(store.get(key));
|
|
18918
|
+
if (first === void 0 && await requestResult(store.getKey(key)) === void 0) return null;
|
|
18919
|
+
const bytes = requireBytes2(key, first);
|
|
18920
|
+
const range = appendRecordsOf(key);
|
|
18921
|
+
const keys = await requestResult(store.getAllKeys(range));
|
|
18922
|
+
const values = await requestResult(store.getAll(range));
|
|
18923
|
+
if (keys.length !== values.length) {
|
|
18924
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The append records of "${key}" changed during a read`, {
|
|
18925
|
+
key
|
|
18926
|
+
});
|
|
18927
|
+
}
|
|
18928
|
+
const appended = [];
|
|
18929
|
+
for (let at = 0; at < keys.length; at++) {
|
|
18930
|
+
appended.push({ n: requireAppendNumber(key, keys[at]), bytes: requireBytes2(key, values[at]) });
|
|
18931
|
+
}
|
|
18932
|
+
return { first: bytes, appended };
|
|
18933
|
+
}
|
|
18934
|
+
function lengthsOf(records) {
|
|
18935
|
+
return [
|
|
18936
|
+
{ n: 0, length: records.first.byteLength },
|
|
18937
|
+
...records.appended.map((record) => ({ n: record.n, length: record.bytes.byteLength }))
|
|
18938
|
+
];
|
|
18939
|
+
}
|
|
18940
|
+
function walkCursor(store, key, range, visit) {
|
|
18941
|
+
return new Promise((resolve, reject) => {
|
|
18942
|
+
const request = store.openCursor(range);
|
|
18943
|
+
request.onsuccess = () => {
|
|
18944
|
+
const cursor = request.result;
|
|
18945
|
+
if (cursor === null) {
|
|
18946
|
+
resolve(true);
|
|
18947
|
+
return;
|
|
18948
|
+
}
|
|
18949
|
+
let enough;
|
|
18950
|
+
try {
|
|
18951
|
+
enough = visit(requireAppendNumber(key, cursor.key), requireBytes2(key, cursor.value));
|
|
18952
|
+
} catch (error) {
|
|
18953
|
+
reject(error);
|
|
18954
|
+
return;
|
|
18955
|
+
}
|
|
18956
|
+
if (enough) resolve(false);
|
|
18957
|
+
else cursor.continue();
|
|
18958
|
+
};
|
|
18959
|
+
request.onerror = () => reject(request.error);
|
|
18960
|
+
});
|
|
18961
|
+
}
|
|
18962
|
+
async function walkBlocks(store, key, afterN, visit) {
|
|
18963
|
+
if (afterN === null) {
|
|
18964
|
+
const first = await requestResult(store.get(key));
|
|
18965
|
+
if (first === void 0 && await requestResult(store.getKey(key)) === void 0) return null;
|
|
18966
|
+
if (visit(0, requireBytes2(key, first))) return false;
|
|
18967
|
+
return walkCursor(store, key, appendRecordsOf(key), visit);
|
|
18968
|
+
}
|
|
18969
|
+
return walkCursor(store, key, IDBKeyRange.bound(appendKey(key, afterN), [key, []], true, true), visit);
|
|
18970
|
+
}
|
|
18971
|
+
function blocksChanged(key) {
|
|
18972
|
+
return new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The blocks of "${key}" changed during a read`, { key });
|
|
18973
|
+
}
|
|
18974
|
+
async function readBlocks(store, key, wanted) {
|
|
18975
|
+
const found = /* @__PURE__ */ new Map();
|
|
18976
|
+
const appended = wanted.filter((block) => block.n > 0);
|
|
18977
|
+
const reads = [];
|
|
18978
|
+
if (appended.length < wanted.length) {
|
|
18979
|
+
reads.push(
|
|
18980
|
+
requestResult(store.get(key)).then((value) => {
|
|
18981
|
+
found.set(0, requireBytes2(key, value));
|
|
18982
|
+
})
|
|
18983
|
+
);
|
|
18984
|
+
}
|
|
18985
|
+
const lowest = appended[0];
|
|
18986
|
+
const highest = appended[appended.length - 1];
|
|
18987
|
+
if (lowest !== void 0 && highest !== void 0) {
|
|
18988
|
+
const range = IDBKeyRange.bound(appendKey(key, lowest.n), appendKey(key, highest.n));
|
|
18989
|
+
reads.push(
|
|
18990
|
+
Promise.all([requestResult(store.getAllKeys(range)), requestResult(store.getAll(range))]).then(
|
|
18991
|
+
([keys, values]) => {
|
|
18992
|
+
if (keys.length !== values.length) throw blocksChanged(key);
|
|
18993
|
+
for (let at = 0; at < keys.length; at++) {
|
|
18994
|
+
found.set(requireAppendNumber(key, keys[at]), requireBytes2(key, values[at]));
|
|
18995
|
+
}
|
|
18996
|
+
}
|
|
18997
|
+
)
|
|
18998
|
+
);
|
|
18999
|
+
}
|
|
19000
|
+
await Promise.all(reads);
|
|
19001
|
+
for (const block of wanted) {
|
|
19002
|
+
if (found.get(block.n)?.byteLength !== block.length) throw blocksChanged(key);
|
|
19003
|
+
}
|
|
19004
|
+
return found;
|
|
19005
|
+
}
|
|
19006
|
+
function splitIntoBlocks(chunks, blockBytes) {
|
|
19007
|
+
let total = 0;
|
|
19008
|
+
for (const chunk of chunks) total += chunk.byteLength;
|
|
19009
|
+
const blocks = [];
|
|
19010
|
+
let block = new Uint8Array(Math.min(blockBytes, total));
|
|
19011
|
+
let filled = 0;
|
|
19012
|
+
for (const chunk of chunks) {
|
|
19013
|
+
let read = 0;
|
|
19014
|
+
while (read < chunk.byteLength) {
|
|
19015
|
+
const taken = Math.min(block.byteLength - filled, chunk.byteLength - read);
|
|
19016
|
+
block.set(chunk.subarray(read, read + taken), filled);
|
|
19017
|
+
filled += taken;
|
|
19018
|
+
read += taken;
|
|
19019
|
+
total -= taken;
|
|
19020
|
+
if (filled === block.byteLength && total > 0) {
|
|
19021
|
+
blocks.push(block);
|
|
19022
|
+
block = new Uint8Array(Math.min(blockBytes, total));
|
|
19023
|
+
filled = 0;
|
|
19024
|
+
}
|
|
19025
|
+
}
|
|
19026
|
+
}
|
|
19027
|
+
blocks.push(block);
|
|
19028
|
+
return blocks;
|
|
19029
|
+
}
|
|
19030
|
+
function contentOf(records) {
|
|
19031
|
+
if (records.appended.length === 0) return records.first;
|
|
19032
|
+
return joinBytes([records.first, ...records.appended.map((record) => record.bytes)]);
|
|
19033
|
+
}
|
|
19034
|
+
async function removeRecords(store, key) {
|
|
19035
|
+
await requestResult(store.delete(appendRecordsOf(key)));
|
|
19036
|
+
await requestResult(store.delete(key));
|
|
19037
|
+
}
|
|
19038
|
+
async function replaceRecords(store, key, records) {
|
|
19039
|
+
await removeRecords(store, key);
|
|
19040
|
+
await requestResult(store.put(records.first, key));
|
|
19041
|
+
for (const record of records.appended) await requestResult(store.put(record.bytes, appendKey(key, record.n)));
|
|
19042
|
+
}
|
|
19043
|
+
function listKeys(store, prefix) {
|
|
19044
|
+
return new Promise((resolve, reject) => {
|
|
19045
|
+
const keys = [];
|
|
19046
|
+
const request = store.openKeyCursor(IDBKeyRange.lowerBound(prefix));
|
|
19047
|
+
request.onsuccess = () => {
|
|
19048
|
+
const cursor = request.result;
|
|
19049
|
+
if (cursor === null || typeof cursor.key !== "string" || !cursor.key.startsWith(prefix)) {
|
|
19050
|
+
resolve(keys);
|
|
19051
|
+
return;
|
|
19052
|
+
}
|
|
19053
|
+
keys.push(cursor.key);
|
|
19054
|
+
cursor.continue();
|
|
19055
|
+
};
|
|
19056
|
+
request.onerror = () => reject(request.error);
|
|
19057
|
+
});
|
|
19058
|
+
}
|
|
19059
|
+
|
|
19060
|
+
// src/persistence/durability/indexeddb-directory/append-handle.ts
|
|
19061
|
+
async function openFileRecords(connection, key) {
|
|
19062
|
+
const parts = await connection.write(
|
|
19063
|
+
"strict",
|
|
19064
|
+
{ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open a log file" },
|
|
19065
|
+
async (store) => {
|
|
19066
|
+
const records = await readRecords(store, key);
|
|
19067
|
+
if (records !== null) return lengthsOf(records);
|
|
19068
|
+
await requestResult(store.put(new Uint8Array(0), key));
|
|
19069
|
+
return [{ n: 0, length: 0 }];
|
|
19070
|
+
}
|
|
19071
|
+
);
|
|
19072
|
+
const nextN = parts[parts.length - 1].n + 1;
|
|
19073
|
+
return { key, parts, unsynced: [], nextN, handles: 0, detached: false, failure: null, moving: null };
|
|
19074
|
+
}
|
|
19075
|
+
async function waitForMove(file) {
|
|
19076
|
+
while (file.moving !== null) await file.moving;
|
|
19077
|
+
}
|
|
19078
|
+
function lengthsOfRecords(records) {
|
|
19079
|
+
return records.map((record) => ({ n: record.n, length: record.bytes.byteLength }));
|
|
19080
|
+
}
|
|
19081
|
+
function sizeOf(file) {
|
|
19082
|
+
let size = 0;
|
|
19083
|
+
for (const part of file.parts) size += part.length;
|
|
19084
|
+
return size;
|
|
19085
|
+
}
|
|
19086
|
+
function recordKeyOf(key, n) {
|
|
19087
|
+
return n === 0 ? key : appendKey(key, n);
|
|
19088
|
+
}
|
|
19089
|
+
function putRecords(store, key, records) {
|
|
19090
|
+
return Promise.all(records.map((record) => requestResult(store.put(record.bytes, recordKeyOf(key, record.n)))));
|
|
19091
|
+
}
|
|
19092
|
+
function holdsNoBytes(file) {
|
|
19093
|
+
return file.parts.length === 1 && file.parts[0].length === 0;
|
|
19094
|
+
}
|
|
19095
|
+
function createAppendHandle(file, deps) {
|
|
19096
|
+
const { connection, blocks } = deps;
|
|
19097
|
+
const appendFailure = { code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "append to a log file" };
|
|
19098
|
+
const strictCommitFailure = {
|
|
19099
|
+
code: ErrorCodes.PERSISTENCE_FSYNC_FAILED,
|
|
19100
|
+
action: "commit a log file with strict durability"
|
|
19101
|
+
};
|
|
19102
|
+
let closed = false;
|
|
19103
|
+
file.handles += 1;
|
|
19104
|
+
function requireUsable() {
|
|
19105
|
+
if (closed) throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The log file "${file.key}" is closed`);
|
|
19106
|
+
if (file.failure !== null) throw file.failure;
|
|
19107
|
+
if (file.detached) {
|
|
19108
|
+
throw new NarsilError(
|
|
19109
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
19110
|
+
`The log file "${file.key}" was removed or replaced while a handle held it open`,
|
|
19111
|
+
{ key: file.key }
|
|
19112
|
+
);
|
|
19113
|
+
}
|
|
19114
|
+
}
|
|
19115
|
+
function latch(error) {
|
|
19116
|
+
const failed = error instanceof Error ? error : new Error(String(error));
|
|
19117
|
+
file.failure ??= failed;
|
|
19118
|
+
return file.failure;
|
|
19119
|
+
}
|
|
19120
|
+
async function putCommitted(durability, failure, records) {
|
|
19121
|
+
const key = file.key;
|
|
19122
|
+
const epoch = blocks.appendEpochOf(key);
|
|
19123
|
+
try {
|
|
19124
|
+
await connection.write(durability, failure, (store) => putRecords(store, key, records));
|
|
19125
|
+
} catch (error) {
|
|
19126
|
+
throw latch(error);
|
|
19127
|
+
}
|
|
19128
|
+
blocks.appended(key, epoch, lengthsOfRecords(records));
|
|
19129
|
+
}
|
|
19130
|
+
async function writeUnsynced(durability) {
|
|
19131
|
+
if (file.moving !== null) await waitForMove(file);
|
|
19132
|
+
const batch = file.unsynced.slice();
|
|
19133
|
+
if (batch.length === 0) return;
|
|
19134
|
+
await putCommitted(durability, durability === "strict" ? strictCommitFailure : appendFailure, batch);
|
|
19135
|
+
file.unsynced.splice(0, batch.length);
|
|
19136
|
+
}
|
|
19137
|
+
async function truncateTo(byteLength) {
|
|
19138
|
+
if (file.moving !== null) await waitForMove(file);
|
|
19139
|
+
const key = file.key;
|
|
19140
|
+
const batch = file.unsynced.slice();
|
|
19141
|
+
const kept = [];
|
|
19142
|
+
let start = 0;
|
|
19143
|
+
for (const part of file.parts) {
|
|
19144
|
+
if (start >= byteLength && part.n !== 0) break;
|
|
19145
|
+
kept.push({ ...part, length: Math.min(part.length, byteLength - start) });
|
|
19146
|
+
start += part.length;
|
|
19147
|
+
}
|
|
19148
|
+
const dropped = file.parts.slice(kept.length);
|
|
19149
|
+
const cut = kept[kept.length - 1];
|
|
19150
|
+
const cutLength = cut.length;
|
|
19151
|
+
const cutIsPartial = cutLength < (file.parts[kept.length - 1]?.length ?? 0);
|
|
19152
|
+
const epoch = blocks.begin(key);
|
|
19153
|
+
try {
|
|
19154
|
+
await connection.write("strict", strictCommitFailure, async (store) => {
|
|
19155
|
+
await putRecords(store, key, batch);
|
|
19156
|
+
for (const part of dropped) await requestResult(store.delete(recordKeyOf(key, part.n)));
|
|
19157
|
+
if (!cutIsPartial) return;
|
|
19158
|
+
const recordKey = recordKeyOf(key, cut.n);
|
|
19159
|
+
const stored = await requestResult(store.get(recordKey));
|
|
19160
|
+
if (!(stored instanceof Uint8Array) || stored.byteLength < cutLength) {
|
|
19161
|
+
throw new NarsilError(
|
|
19162
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
19163
|
+
`A record of "${key}" holds fewer bytes than its log`,
|
|
19164
|
+
{
|
|
19165
|
+
key
|
|
19166
|
+
}
|
|
19167
|
+
);
|
|
19168
|
+
}
|
|
19169
|
+
await requestResult(store.put(stored.slice(0, cutLength), recordKey));
|
|
19170
|
+
});
|
|
19171
|
+
} catch (error) {
|
|
19172
|
+
blocks.settle(key, epoch, void 0);
|
|
19173
|
+
throw error;
|
|
19174
|
+
}
|
|
19175
|
+
blocks.settle(key, epoch, kept);
|
|
19176
|
+
file.parts = kept;
|
|
19177
|
+
file.unsynced = [];
|
|
19178
|
+
file.nextN = kept[kept.length - 1].n + 1;
|
|
19179
|
+
}
|
|
19180
|
+
return {
|
|
19181
|
+
async append(bytes) {
|
|
19182
|
+
requireUsable();
|
|
19183
|
+
if (bytes.byteLength === 0) return;
|
|
19184
|
+
const records = [];
|
|
19185
|
+
for (const piece of splitIntoBlocks([bytes], INDEXEDDB_BLOCK_BYTES)) {
|
|
19186
|
+
if (holdsNoBytes(file)) {
|
|
19187
|
+
file.parts[0] = { n: 0, length: piece.byteLength };
|
|
19188
|
+
records.push({ n: 0, bytes: piece });
|
|
19189
|
+
continue;
|
|
19190
|
+
}
|
|
19191
|
+
records.push({ n: file.nextN, bytes: piece });
|
|
19192
|
+
file.parts.push({ n: file.nextN, length: piece.byteLength });
|
|
19193
|
+
file.nextN += 1;
|
|
19194
|
+
}
|
|
19195
|
+
file.unsynced.push(...records);
|
|
19196
|
+
if (!deps.relaxedAppends) return;
|
|
19197
|
+
if (file.moving !== null) await waitForMove(file);
|
|
19198
|
+
await putCommitted("relaxed", appendFailure, records);
|
|
19199
|
+
},
|
|
19200
|
+
async sync() {
|
|
19201
|
+
requireUsable();
|
|
19202
|
+
await writeUnsynced("strict");
|
|
19203
|
+
},
|
|
19204
|
+
async truncate(byteLength) {
|
|
19205
|
+
requireUsable();
|
|
19206
|
+
if (!Number.isSafeInteger(byteLength) || byteLength < 0 || byteLength > sizeOf(file)) {
|
|
19207
|
+
throw new NarsilError(
|
|
19208
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
19209
|
+
`The log file "${file.key}" cannot be cut to ${byteLength} bytes, because it holds ${sizeOf(file)}`,
|
|
19210
|
+
{ key: file.key, byteLength }
|
|
19211
|
+
);
|
|
19212
|
+
}
|
|
19213
|
+
try {
|
|
19214
|
+
await truncateTo(byteLength);
|
|
19215
|
+
} catch (error) {
|
|
19216
|
+
throw latch(error);
|
|
19217
|
+
}
|
|
19218
|
+
},
|
|
19219
|
+
async size() {
|
|
19220
|
+
if (closed) throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The log file "${file.key}" is closed`);
|
|
19221
|
+
return sizeOf(file);
|
|
19222
|
+
},
|
|
19223
|
+
async close() {
|
|
19224
|
+
if (closed) return;
|
|
19225
|
+
closed = true;
|
|
19226
|
+
file.handles -= 1;
|
|
19227
|
+
try {
|
|
19228
|
+
if (!deps.relaxedAppends && file.failure === null && !file.detached) await writeUnsynced("relaxed");
|
|
19229
|
+
} finally {
|
|
19230
|
+
if (file.handles === 0) deps.release(file);
|
|
19231
|
+
}
|
|
19232
|
+
}
|
|
19233
|
+
};
|
|
19234
|
+
}
|
|
19235
|
+
|
|
19236
|
+
// src/persistence/durability/indexeddb-directory/block-map.ts
|
|
19237
|
+
function laidOut(lengths) {
|
|
19238
|
+
const blocks = [];
|
|
19239
|
+
let start = 0;
|
|
19240
|
+
for (const { n, length } of lengths) {
|
|
19241
|
+
blocks.push({ n, length, start });
|
|
19242
|
+
start += length;
|
|
19243
|
+
}
|
|
19244
|
+
return blocks;
|
|
19245
|
+
}
|
|
19246
|
+
function sizeOfBlocks(blocks) {
|
|
19247
|
+
const last = blocks[blocks.length - 1];
|
|
19248
|
+
return last === void 0 ? 0 : last.start + last.length;
|
|
19249
|
+
}
|
|
19250
|
+
function firstBlockAt(blocks, offset) {
|
|
19251
|
+
let low = 0;
|
|
19252
|
+
let high = blocks.length - 1;
|
|
19253
|
+
while (low < high) {
|
|
19254
|
+
const middle = low + high + 1 >>> 1;
|
|
19255
|
+
if (blocks[middle].start <= offset) low = middle;
|
|
19256
|
+
else high = middle - 1;
|
|
19257
|
+
}
|
|
19258
|
+
return low;
|
|
19259
|
+
}
|
|
19260
|
+
function blockNumbered(blocks, n) {
|
|
19261
|
+
let low = 0;
|
|
19262
|
+
let high = blocks.length - 1;
|
|
19263
|
+
while (low <= high) {
|
|
19264
|
+
const middle = low + high >>> 1;
|
|
19265
|
+
const block = blocks[middle];
|
|
19266
|
+
if (block.n === n) return block;
|
|
19267
|
+
if (block.n < n) low = middle + 1;
|
|
19268
|
+
else high = middle - 1;
|
|
19269
|
+
}
|
|
19270
|
+
return void 0;
|
|
19271
|
+
}
|
|
19272
|
+
function mergedBlocks(known, complete, lengths) {
|
|
19273
|
+
let merged = known;
|
|
19274
|
+
for (const { n, length } of lengths) {
|
|
19275
|
+
const last = merged[merged.length - 1];
|
|
19276
|
+
if (last === void 0) return void 0;
|
|
19277
|
+
if (n > last.n) {
|
|
19278
|
+
if (complete) merged.push({ n, length, start: last.start + last.length });
|
|
19279
|
+
continue;
|
|
19280
|
+
}
|
|
19281
|
+
if (n === 0 && merged.length === 1 && last.length === 0) {
|
|
19282
|
+
if (!complete) return void 0;
|
|
19283
|
+
merged = [{ n: 0, length, start: 0 }];
|
|
19284
|
+
continue;
|
|
19285
|
+
}
|
|
19286
|
+
const existing = blockNumbered(merged, n);
|
|
19287
|
+
if (existing === void 0 || existing.length !== length) return void 0;
|
|
19288
|
+
}
|
|
19289
|
+
return merged;
|
|
19290
|
+
}
|
|
19291
|
+
function createBlockMap() {
|
|
19292
|
+
const states = /* @__PURE__ */ new Map();
|
|
19293
|
+
const cache = /* @__PURE__ */ new Map();
|
|
19294
|
+
let counter = 0;
|
|
19295
|
+
function next() {
|
|
19296
|
+
counter += 1;
|
|
19297
|
+
return counter;
|
|
19298
|
+
}
|
|
19299
|
+
function stateOf2(key) {
|
|
19300
|
+
const known = states.get(key);
|
|
19301
|
+
if (known !== void 0) return known;
|
|
19302
|
+
const created = { generation: next(), epoch: next(), changes: 0, blocks: void 0, complete: false };
|
|
19303
|
+
states.set(key, created);
|
|
19304
|
+
return created;
|
|
19305
|
+
}
|
|
19306
|
+
function cacheKeyOf(key, n) {
|
|
19307
|
+
return `${n}\0${key}`;
|
|
19308
|
+
}
|
|
19309
|
+
function invalidate(state) {
|
|
19310
|
+
state.generation = next();
|
|
19311
|
+
state.changes += 1;
|
|
19312
|
+
state.blocks = void 0;
|
|
19313
|
+
state.complete = false;
|
|
19314
|
+
}
|
|
19315
|
+
return {
|
|
19316
|
+
viewOf(key) {
|
|
19317
|
+
const state = stateOf2(key);
|
|
19318
|
+
const layout = state.blocks === void 0 ? void 0 : { blocks: state.blocks, complete: state.complete };
|
|
19319
|
+
return { generation: state.generation, changes: state.changes, layout };
|
|
19320
|
+
},
|
|
19321
|
+
generationOf: (key) => stateOf2(key).generation,
|
|
19322
|
+
install(key, view, blocks, reachedEnd) {
|
|
19323
|
+
const state = stateOf2(key);
|
|
19324
|
+
if (state.generation !== view.generation) return;
|
|
19325
|
+
const complete = reachedEnd && state.changes === view.changes;
|
|
19326
|
+
const current = state.blocks;
|
|
19327
|
+
if (current !== void 0 && (state.complete || !complete && blocks.length <= current.length)) return;
|
|
19328
|
+
state.blocks = blocks;
|
|
19329
|
+
state.complete = complete;
|
|
19330
|
+
},
|
|
19331
|
+
forget: (key) => invalidate(stateOf2(key)),
|
|
19332
|
+
begin(key) {
|
|
19333
|
+
const state = stateOf2(key);
|
|
19334
|
+
invalidate(state);
|
|
19335
|
+
state.epoch = next();
|
|
19336
|
+
return state.epoch;
|
|
19337
|
+
},
|
|
19338
|
+
settle(key, epoch, lengths) {
|
|
19339
|
+
const state = stateOf2(key);
|
|
19340
|
+
if (state.epoch !== epoch) return;
|
|
19341
|
+
invalidate(state);
|
|
19342
|
+
if (lengths === void 0) return;
|
|
19343
|
+
state.blocks = laidOut(lengths);
|
|
19344
|
+
state.complete = true;
|
|
19345
|
+
},
|
|
19346
|
+
appendEpochOf: (key) => stateOf2(key).epoch,
|
|
19347
|
+
appended(key, epoch, lengths) {
|
|
19348
|
+
const state = stateOf2(key);
|
|
19349
|
+
if (state.epoch !== epoch) return;
|
|
19350
|
+
state.changes += 1;
|
|
19351
|
+
if (state.blocks === void 0) return;
|
|
19352
|
+
const merged = mergedBlocks(state.blocks, state.complete, lengths);
|
|
19353
|
+
if (merged === void 0) invalidate(state);
|
|
19354
|
+
else state.blocks = merged;
|
|
19355
|
+
},
|
|
19356
|
+
covering(key, generation, blocks, offset, length) {
|
|
19357
|
+
const covered = [];
|
|
19358
|
+
const end = offset + length;
|
|
19359
|
+
for (let at = firstBlockAt(blocks, offset); at < blocks.length; at++) {
|
|
19360
|
+
const block = blocks[at];
|
|
19361
|
+
if (block.start >= end && covered.length > 0) break;
|
|
19362
|
+
const cacheKey = cacheKeyOf(key, block.n);
|
|
19363
|
+
const entry = cache.get(cacheKey);
|
|
19364
|
+
const cached = entry !== void 0 && entry.generation === generation && entry.bytes.byteLength === block.length ? entry.bytes : void 0;
|
|
19365
|
+
if (cached !== void 0) {
|
|
19366
|
+
cache.delete(cacheKey);
|
|
19367
|
+
cache.set(cacheKey, { generation, bytes: cached });
|
|
19368
|
+
}
|
|
19369
|
+
covered.push({ ...block, cached });
|
|
19370
|
+
if (block.start + block.length >= end) break;
|
|
19371
|
+
}
|
|
19372
|
+
return covered;
|
|
19373
|
+
},
|
|
19374
|
+
keep(key, generation, n, bytes) {
|
|
19375
|
+
if (bytes.byteLength > INDEXEDDB_BLOCK_BYTES || stateOf2(key).generation !== generation) return;
|
|
19376
|
+
const cacheKey = cacheKeyOf(key, n);
|
|
19377
|
+
cache.delete(cacheKey);
|
|
19378
|
+
cache.set(cacheKey, { generation, bytes });
|
|
19379
|
+
while (cache.size > INDEXEDDB_CACHED_BLOCKS) {
|
|
19380
|
+
const oldest = cache.keys().next();
|
|
19381
|
+
if (oldest.done === true) break;
|
|
19382
|
+
cache.delete(oldest.value);
|
|
19383
|
+
}
|
|
19384
|
+
}
|
|
19385
|
+
};
|
|
19386
|
+
}
|
|
19387
|
+
|
|
19388
|
+
// src/persistence/durability/indexeddb-directory/lock.ts
|
|
19389
|
+
var WEB_LOCK_NAME_PREFIX = "narsil-wal";
|
|
19390
|
+
var heldWithoutWebLocks = /* @__PURE__ */ new Set();
|
|
19391
|
+
function webLockNameOf(location) {
|
|
19392
|
+
return `${WEB_LOCK_NAME_PREFIX}${JSON.stringify([location.databaseName, location.storeName])}`;
|
|
19393
|
+
}
|
|
19394
|
+
function heldElsewhere2(location) {
|
|
19395
|
+
return new NarsilError(
|
|
19396
|
+
ErrorCodes.CONFIG_INVALID,
|
|
19397
|
+
`Another tab or engine holds the write-ahead log of IndexedDB database "${location.databaseName}" store "${location.storeName}". Several tabs that share one store need an invalidation adapter or durability.tier "snapshot"`,
|
|
19398
|
+
{ databaseName: location.databaseName, storeName: location.storeName }
|
|
19399
|
+
);
|
|
19400
|
+
}
|
|
19401
|
+
function lockWithinProcess(name, location) {
|
|
19402
|
+
if (heldWithoutWebLocks.has(name)) throw heldElsewhere2(location);
|
|
19403
|
+
heldWithoutWebLocks.add(name);
|
|
19404
|
+
let held = true;
|
|
19405
|
+
return {
|
|
19406
|
+
holds: async () => held,
|
|
19407
|
+
async release() {
|
|
19408
|
+
if (!held) return;
|
|
19409
|
+
held = false;
|
|
19410
|
+
heldWithoutWebLocks.delete(name);
|
|
19411
|
+
}
|
|
19412
|
+
};
|
|
19413
|
+
}
|
|
19414
|
+
function lockAcrossTabs(locks, name, location) {
|
|
19415
|
+
return new Promise((resolve, reject) => {
|
|
19416
|
+
let held = false;
|
|
19417
|
+
let releaseLock = () => void 0;
|
|
19418
|
+
const releasing = new Promise((settle) => {
|
|
19419
|
+
releaseLock = settle;
|
|
19420
|
+
});
|
|
19421
|
+
locks.request(name, { mode: "exclusive", ifAvailable: true }, (granted) => {
|
|
19422
|
+
if (granted === null) {
|
|
19423
|
+
reject(heldElsewhere2(location));
|
|
19424
|
+
return void 0;
|
|
19425
|
+
}
|
|
19426
|
+
held = true;
|
|
19427
|
+
resolve({
|
|
19428
|
+
holds: async () => held,
|
|
19429
|
+
async release() {
|
|
19430
|
+
held = false;
|
|
19431
|
+
releaseLock();
|
|
19432
|
+
}
|
|
19433
|
+
});
|
|
19434
|
+
return releasing;
|
|
19435
|
+
}).then(
|
|
19436
|
+
() => {
|
|
19437
|
+
held = false;
|
|
19438
|
+
},
|
|
19439
|
+
(error) => {
|
|
19440
|
+
held = false;
|
|
19441
|
+
reject(error);
|
|
19442
|
+
}
|
|
19443
|
+
);
|
|
19444
|
+
});
|
|
19445
|
+
}
|
|
19446
|
+
function lockIndexedDBStore(location) {
|
|
19447
|
+
const name = webLockNameOf(location);
|
|
19448
|
+
const locks = globalThis.navigator?.locks;
|
|
19449
|
+
if (locks === void 0) return Promise.resolve().then(() => lockWithinProcess(name, location));
|
|
19450
|
+
return lockAcrossTabs(locks, name, location);
|
|
19451
|
+
}
|
|
19452
|
+
|
|
19453
|
+
// src/persistence/durability/indexeddb-directory/range-read.ts
|
|
19454
|
+
function endsBefore(key, offset, length) {
|
|
19455
|
+
return new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `"${key}" ends before the bytes a read asked for`, {
|
|
19456
|
+
key,
|
|
19457
|
+
offset,
|
|
19458
|
+
length
|
|
19459
|
+
});
|
|
19460
|
+
}
|
|
19461
|
+
function isLoadFailure(error) {
|
|
19462
|
+
return error instanceof NarsilError && error.code === ErrorCodes.PERSISTENCE_LOAD_FAILED;
|
|
19463
|
+
}
|
|
19464
|
+
function assembled(key, covered, fetched, offset, length) {
|
|
19465
|
+
const bytes = new Uint8Array(length);
|
|
19466
|
+
for (const block of covered) {
|
|
19467
|
+
if (block.length === 0) continue;
|
|
19468
|
+
const source = block.cached ?? fetched.get(block.n);
|
|
19469
|
+
if (source === void 0) throw endsBefore(key, offset, length);
|
|
19470
|
+
const from = Math.max(offset, block.start);
|
|
19471
|
+
const to = Math.min(offset + length, block.start + block.length);
|
|
19472
|
+
bytes.set(source.subarray(from - block.start, to - block.start), from - offset);
|
|
19473
|
+
}
|
|
19474
|
+
return bytes;
|
|
19475
|
+
}
|
|
19476
|
+
function createRangeReads(connection, blocks, readContent) {
|
|
19477
|
+
async function extendTo(key, view, offset, end) {
|
|
19478
|
+
const known = view.layout === void 0 ? [] : view.layout.blocks.slice();
|
|
19479
|
+
const fetched = /* @__PURE__ */ new Map();
|
|
19480
|
+
let size = sizeOfBlocks(known);
|
|
19481
|
+
const last = known[known.length - 1];
|
|
19482
|
+
const reachedEnd = await connection.read(
|
|
19483
|
+
{ code: ErrorCodes.PERSISTENCE_LOAD_FAILED, action: `read the blocks of "${key}"` },
|
|
19484
|
+
(store) => walkBlocks(store, key, last === void 0 ? null : last.n, (n, bytes) => {
|
|
19485
|
+
const start = size;
|
|
19486
|
+
known.push({ n, length: bytes.byteLength, start });
|
|
19487
|
+
size += bytes.byteLength;
|
|
19488
|
+
if (bytes.byteLength > 0 && start < end && size > offset) fetched.set(n, bytes);
|
|
19489
|
+
return size >= end;
|
|
19490
|
+
})
|
|
19491
|
+
);
|
|
19492
|
+
if (reachedEnd === null) return null;
|
|
19493
|
+
blocks.install(key, view, known, reachedEnd);
|
|
19494
|
+
return { blocks: known, fetched };
|
|
19495
|
+
}
|
|
19496
|
+
async function wholeFileRange(key, offset, length) {
|
|
19497
|
+
const content = await readContent(key);
|
|
19498
|
+
if (content === null || offset + length > content.byteLength) throw endsBefore(key, offset, length);
|
|
19499
|
+
return content.slice(offset, offset + length);
|
|
19500
|
+
}
|
|
19501
|
+
async function blockRange(key, offset, length) {
|
|
19502
|
+
const end = offset + length;
|
|
19503
|
+
const view = blocks.viewOf(key);
|
|
19504
|
+
let layout = view.layout?.blocks;
|
|
19505
|
+
let fetched = /* @__PURE__ */ new Map();
|
|
19506
|
+
if (view.layout === void 0 || !view.layout.complete && sizeOfBlocks(view.layout.blocks) < end) {
|
|
19507
|
+
const extended = await extendTo(key, view, offset, end);
|
|
19508
|
+
layout = extended?.blocks;
|
|
19509
|
+
if (extended !== null) fetched = extended.fetched;
|
|
19510
|
+
}
|
|
19511
|
+
if (layout === void 0 || sizeOfBlocks(layout) < end) {
|
|
19512
|
+
if (blocks.generationOf(key) !== view.generation) return wholeFileRange(key, offset, length);
|
|
19513
|
+
throw endsBefore(key, offset, length);
|
|
19514
|
+
}
|
|
19515
|
+
if (length === 0) return new Uint8Array(0);
|
|
19516
|
+
const covered = blocks.covering(key, view.generation, layout, offset, length);
|
|
19517
|
+
const missing2 = covered.filter((block) => block.length > 0 && block.cached === void 0 && !fetched.has(block.n));
|
|
19518
|
+
if (missing2.length > 0) {
|
|
19519
|
+
try {
|
|
19520
|
+
const read = await connection.read(
|
|
19521
|
+
{ code: ErrorCodes.PERSISTENCE_LOAD_FAILED, action: `read "${key}"` },
|
|
19522
|
+
(store) => readBlocks(store, key, missing2)
|
|
19523
|
+
);
|
|
19524
|
+
for (const [n, bytes] of read) fetched.set(n, bytes);
|
|
19525
|
+
} catch (error) {
|
|
19526
|
+
if (!isLoadFailure(error)) throw error;
|
|
19527
|
+
blocks.forget(key);
|
|
19528
|
+
return wholeFileRange(key, offset, length);
|
|
19529
|
+
}
|
|
19530
|
+
}
|
|
19531
|
+
if (blocks.generationOf(key) !== view.generation) return wholeFileRange(key, offset, length);
|
|
19532
|
+
for (const [n, bytes] of fetched) blocks.keep(key, view.generation, n, bytes);
|
|
19533
|
+
return assembled(key, covered, fetched, offset, length);
|
|
19534
|
+
}
|
|
19535
|
+
return {
|
|
19536
|
+
async readRange(key, offset, length) {
|
|
19537
|
+
if (!Number.isSafeInteger(offset) || !Number.isSafeInteger(length) || offset < 0 || length < 0) {
|
|
19538
|
+
throw endsBefore(key, offset, length);
|
|
19539
|
+
}
|
|
19540
|
+
return blockRange(key, offset, length);
|
|
19541
|
+
},
|
|
19542
|
+
async byteLength(key) {
|
|
19543
|
+
const view = blocks.viewOf(key);
|
|
19544
|
+
if (view.layout?.complete === true) return sizeOfBlocks(view.layout.blocks);
|
|
19545
|
+
const extended = await extendTo(key, view, Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY);
|
|
19546
|
+
if (blocks.generationOf(key) !== view.generation) return (await readContent(key))?.byteLength ?? null;
|
|
19547
|
+
return extended === null ? null : sizeOfBlocks(extended.blocks);
|
|
19548
|
+
}
|
|
19549
|
+
};
|
|
19550
|
+
}
|
|
19551
|
+
|
|
19552
|
+
// src/persistence/durability/indexeddb-directory/index.ts
|
|
19553
|
+
function noValue() {
|
|
19554
|
+
return void 0;
|
|
19555
|
+
}
|
|
19556
|
+
function requireKey(key) {
|
|
19557
|
+
if (typeof key !== "string" || key.length === 0) {
|
|
19558
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: empty");
|
|
19559
|
+
}
|
|
19560
|
+
if (key.includes("\0")) {
|
|
19561
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: null byte detected", { key });
|
|
19562
|
+
}
|
|
19563
|
+
}
|
|
19564
|
+
function createIndexedDBDirectory(location, options) {
|
|
19565
|
+
const saveFailed = ErrorCodes.PERSISTENCE_SAVE_FAILED;
|
|
19566
|
+
const loadFailed2 = ErrorCodes.PERSISTENCE_LOAD_FAILED;
|
|
19567
|
+
const connection = openLogConnection(location);
|
|
19568
|
+
const blocks = createBlockMap();
|
|
19569
|
+
const openFiles2 = /* @__PURE__ */ new Map();
|
|
19570
|
+
const opening = /* @__PURE__ */ new Map();
|
|
19571
|
+
async function settledWrite(key, write, lengthsAfter) {
|
|
19572
|
+
const epoch = blocks.begin(key);
|
|
19573
|
+
let result;
|
|
19574
|
+
try {
|
|
19575
|
+
result = await write();
|
|
19576
|
+
} catch (error) {
|
|
19577
|
+
blocks.settle(key, epoch, void 0);
|
|
19578
|
+
throw error;
|
|
19579
|
+
}
|
|
19580
|
+
blocks.settle(key, epoch, lengthsAfter(result));
|
|
19581
|
+
return result;
|
|
19582
|
+
}
|
|
19583
|
+
function detach(key) {
|
|
19584
|
+
const file = openFiles2.get(key);
|
|
19585
|
+
if (file === void 0) return;
|
|
19586
|
+
file.detached = true;
|
|
19587
|
+
openFiles2.delete(key);
|
|
19588
|
+
}
|
|
19589
|
+
function release(file) {
|
|
19590
|
+
if (openFiles2.get(file.key) === file) openFiles2.delete(file.key);
|
|
19591
|
+
}
|
|
19592
|
+
async function openFile(key) {
|
|
19593
|
+
const existing = openFiles2.get(key);
|
|
19594
|
+
if (existing !== void 0) return existing;
|
|
19595
|
+
const pending = opening.get(key);
|
|
19596
|
+
if (pending !== void 0) return pending;
|
|
19597
|
+
const opened = settledWrite(
|
|
19598
|
+
key,
|
|
19599
|
+
() => openFileRecords(connection, key),
|
|
19600
|
+
(file) => file.parts
|
|
19601
|
+
).then(
|
|
19602
|
+
(file) => {
|
|
19603
|
+
opening.delete(key);
|
|
19604
|
+
openFiles2.set(key, file);
|
|
19605
|
+
return file;
|
|
19606
|
+
},
|
|
19607
|
+
(error) => {
|
|
19608
|
+
opening.delete(key);
|
|
19609
|
+
throw error;
|
|
19610
|
+
}
|
|
19611
|
+
);
|
|
19612
|
+
opening.set(key, opened);
|
|
19613
|
+
return opened;
|
|
19614
|
+
}
|
|
19615
|
+
async function readContent(key) {
|
|
19616
|
+
requireKey(key);
|
|
19617
|
+
const view = blocks.viewOf(key);
|
|
19618
|
+
const records = await connection.read(
|
|
19619
|
+
{ code: loadFailed2, action: `read "${key}"` },
|
|
19620
|
+
(store) => readRecords(store, key)
|
|
19621
|
+
);
|
|
19622
|
+
if (records === null) return null;
|
|
19623
|
+
blocks.install(key, view, laidOut(lengthsOf(records)), true);
|
|
19624
|
+
return contentOf(records);
|
|
19625
|
+
}
|
|
19626
|
+
const ranges = createRangeReads(connection, blocks, readContent);
|
|
19627
|
+
function renameAfterMove(moving, from, to) {
|
|
19628
|
+
moving.key = to;
|
|
19629
|
+
if (openFiles2.get(from) !== moving) return;
|
|
19630
|
+
openFiles2.delete(from);
|
|
19631
|
+
detach(to);
|
|
19632
|
+
openFiles2.set(to, moving);
|
|
19633
|
+
}
|
|
19634
|
+
async function moveRecords(from, to) {
|
|
19635
|
+
const fromEpoch = blocks.begin(from);
|
|
19636
|
+
const moved = settledWrite(
|
|
19637
|
+
to,
|
|
19638
|
+
() => connection.write("strict", { code: saveFailed, action: `move "${from}" to "${to}"` }, async (store) => {
|
|
19639
|
+
const records = await readRecords(store, from);
|
|
19640
|
+
if (records === null) {
|
|
19641
|
+
throw new NarsilError(saveFailed, `Failed to move "${from}" to "${to}": "${from}" holds nothing`, {
|
|
19642
|
+
from,
|
|
19643
|
+
to
|
|
19644
|
+
});
|
|
19645
|
+
}
|
|
19646
|
+
await replaceRecords(store, to, records);
|
|
19647
|
+
await removeRecords(store, from);
|
|
19648
|
+
return lengthsOf(records);
|
|
19649
|
+
}),
|
|
19650
|
+
(lengths) => lengths
|
|
19651
|
+
);
|
|
19652
|
+
const moving = openFiles2.get(from);
|
|
19653
|
+
if (moving !== void 0) moving.moving = moved.then(noValue, noValue);
|
|
19654
|
+
try {
|
|
19655
|
+
await moved;
|
|
19656
|
+
if (moving !== void 0) renameAfterMove(moving, from, to);
|
|
19657
|
+
} finally {
|
|
19658
|
+
blocks.settle(from, fromEpoch, void 0);
|
|
19659
|
+
if (moving !== void 0) moving.moving = null;
|
|
19660
|
+
}
|
|
19661
|
+
}
|
|
19662
|
+
return {
|
|
19663
|
+
root: connection.root,
|
|
19664
|
+
async appendHandle(key) {
|
|
19665
|
+
requireKey(key);
|
|
19666
|
+
const file = await openFile(key);
|
|
19667
|
+
return createAppendHandle(file, { connection, blocks, relaxedAppends: options.relaxedAppends, release });
|
|
19668
|
+
},
|
|
19669
|
+
async markerHandle(key) {
|
|
19670
|
+
requireKey(key);
|
|
19671
|
+
let current = await settledWrite(
|
|
19672
|
+
key,
|
|
19673
|
+
() => connection.write("strict", { code: saveFailed, action: `open "${key}"` }, async (store) => {
|
|
19674
|
+
const records = await readRecords(store, key);
|
|
19675
|
+
if (records !== null) return contentOf(records);
|
|
19676
|
+
const empty = new Uint8Array(0);
|
|
19677
|
+
await requestResult(store.put(empty, key));
|
|
19678
|
+
return empty;
|
|
19679
|
+
}),
|
|
19680
|
+
(bytes) => [{ n: 0, length: bytes.byteLength }]
|
|
19681
|
+
);
|
|
19682
|
+
return {
|
|
19683
|
+
read: async () => current.slice(),
|
|
19684
|
+
async writeSlot(offset, slot) {
|
|
19685
|
+
const next = new Uint8Array(Math.max(current.byteLength, offset + slot.byteLength));
|
|
19686
|
+
next.set(current);
|
|
19687
|
+
next.set(slot, offset);
|
|
19688
|
+
await settledWrite(
|
|
19689
|
+
key,
|
|
19690
|
+
() => connection.write(
|
|
19691
|
+
"strict",
|
|
19692
|
+
{ code: ErrorCodes.PERSISTENCE_FSYNC_FAILED, action: `commit "${key}"` },
|
|
19693
|
+
(store) => replaceRecords(store, key, { first: next, appended: [] })
|
|
19694
|
+
),
|
|
19695
|
+
() => [{ n: 0, length: next.byteLength }]
|
|
19696
|
+
);
|
|
19697
|
+
current = next;
|
|
19698
|
+
},
|
|
19699
|
+
close: async () => void 0
|
|
19700
|
+
};
|
|
19701
|
+
},
|
|
19702
|
+
async atomicWrite(key, data) {
|
|
19703
|
+
requireKey(key);
|
|
19704
|
+
const pieces = splitIntoBlocks(Array.isArray(data) ? data : [data], INDEXEDDB_BLOCK_BYTES);
|
|
19705
|
+
const records = {
|
|
19706
|
+
first: pieces[0],
|
|
19707
|
+
appended: pieces.slice(1).map((bytes, at) => ({ n: at + 1, bytes }))
|
|
19708
|
+
};
|
|
19709
|
+
detach(key);
|
|
19710
|
+
await settledWrite(
|
|
19711
|
+
key,
|
|
19712
|
+
() => connection.write(
|
|
19713
|
+
"strict",
|
|
19714
|
+
{ code: saveFailed, action: `write "${key}"` },
|
|
19715
|
+
(store) => replaceRecords(store, key, records)
|
|
19716
|
+
),
|
|
19717
|
+
() => lengthsOf(records)
|
|
19718
|
+
);
|
|
19719
|
+
},
|
|
19720
|
+
async syncDirectoryOf(key) {
|
|
19721
|
+
requireKey(key);
|
|
19722
|
+
},
|
|
19723
|
+
read: readContent,
|
|
19724
|
+
async readRange(key, offset, length) {
|
|
19725
|
+
requireKey(key);
|
|
19726
|
+
return ranges.readRange(key, offset, length);
|
|
19727
|
+
},
|
|
19728
|
+
async byteLength(key) {
|
|
19729
|
+
requireKey(key);
|
|
19730
|
+
return ranges.byteLength(key);
|
|
19731
|
+
},
|
|
19732
|
+
async remove(key) {
|
|
19733
|
+
requireKey(key);
|
|
19734
|
+
detach(key);
|
|
19735
|
+
await settledWrite(
|
|
19736
|
+
key,
|
|
19737
|
+
() => connection.write(
|
|
19738
|
+
"strict",
|
|
19739
|
+
{ code: ErrorCodes.PERSISTENCE_DELETE_FAILED, action: `delete "${key}"` },
|
|
19740
|
+
(store) => removeRecords(store, key)
|
|
19741
|
+
),
|
|
19742
|
+
() => void 0
|
|
19743
|
+
);
|
|
19744
|
+
},
|
|
19745
|
+
async move(from, to) {
|
|
19746
|
+
requireKey(from);
|
|
19747
|
+
requireKey(to);
|
|
19748
|
+
if (from === to) return;
|
|
19749
|
+
detach(to);
|
|
19750
|
+
await moveRecords(from, to);
|
|
19751
|
+
},
|
|
19752
|
+
async list(prefix) {
|
|
19753
|
+
const keys = await connection.read(
|
|
19754
|
+
{ code: loadFailed2, action: `list "${prefix}"` },
|
|
19755
|
+
(store) => listKeys(store, prefix)
|
|
19756
|
+
);
|
|
19757
|
+
return keys.sort(compareCodePoints);
|
|
19758
|
+
},
|
|
19759
|
+
identity: () => connection.identity(),
|
|
19760
|
+
async lock() {
|
|
19761
|
+
const held = await lockIndexedDBStore(location);
|
|
19762
|
+
return {
|
|
19763
|
+
holds: () => held.holds(),
|
|
19764
|
+
async release() {
|
|
19765
|
+
connection.close();
|
|
19766
|
+
await held.release();
|
|
19767
|
+
}
|
|
19768
|
+
};
|
|
19769
|
+
}
|
|
19770
|
+
};
|
|
19771
|
+
}
|
|
19772
|
+
|
|
19773
|
+
// src/persistence/durability/snapshot-only/index-state.ts
|
|
19774
|
+
function createSnapshotIndexState() {
|
|
19775
|
+
return {
|
|
19776
|
+
mutationsSinceCheckpoint: 0,
|
|
19777
|
+
checkpointInFlight: null,
|
|
19778
|
+
applyChain: Promise.resolve(),
|
|
19779
|
+
unloading: false,
|
|
19780
|
+
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
19781
|
+
};
|
|
19782
|
+
}
|
|
19783
|
+
function metadataKey(indexName) {
|
|
19784
|
+
return `${indexName}/meta`;
|
|
19785
|
+
}
|
|
19786
|
+
function noteSnapshotCommitted(state, positions, partitionCount) {
|
|
19787
|
+
let recordsLeft = 0;
|
|
19788
|
+
for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
|
|
19789
|
+
if (partitionId >= partitionCount) {
|
|
19790
|
+
state.appliedSeqNoByPartition.delete(partitionId);
|
|
19791
|
+
continue;
|
|
19792
|
+
}
|
|
19793
|
+
recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
|
|
19794
|
+
}
|
|
19795
|
+
state.mutationsSinceCheckpoint = recordsLeft;
|
|
19796
|
+
}
|
|
19797
|
+
|
|
19798
|
+
// src/persistence/durability/snapshot-only/checkpoint.ts
|
|
19799
|
+
function capture2(input, prior, cutoffs) {
|
|
19800
|
+
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
19801
|
+
const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
|
|
19802
|
+
const targets = [];
|
|
19803
|
+
const contents = [];
|
|
19804
|
+
const positions = /* @__PURE__ */ new Map();
|
|
19805
|
+
for (let partitionId = 0; partitionId < input.manager.partitionCount; partitionId += 1) {
|
|
19806
|
+
const lastSeqNo = Math.min(
|
|
19807
|
+
input.appliedSeqNoByPartition.get(partitionId) ?? 0,
|
|
19808
|
+
cutoffs.get(partitionId) ?? Infinity
|
|
19809
|
+
);
|
|
19810
|
+
positions.set(partitionId, lastSeqNo);
|
|
19811
|
+
const unchanged = !input.rewritesEveryPartition && prior !== null && !partitionCountChanged && listedIds.has(partitionId) && checkpointLastSeqNo(prior.checkpoint, partitionId) === lastSeqNo;
|
|
19812
|
+
if (unchanged) continue;
|
|
19813
|
+
targets.push({ partitionId, lastSeqNo, primaryTerm: SINGLE_NODE_PRIMARY_TERM });
|
|
19814
|
+
contents.push(partitionContentsOf(input.manager, partitionId));
|
|
19815
|
+
}
|
|
19816
|
+
const plans = /* @__PURE__ */ new Map();
|
|
19817
|
+
try {
|
|
19818
|
+
for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
|
|
19819
|
+
const listed = prior?.vectors.find((ref) => ref.fieldPath === fieldPath);
|
|
19820
|
+
plans.set(
|
|
19821
|
+
fieldPath,
|
|
19822
|
+
vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key), false, true)
|
|
19823
|
+
);
|
|
19824
|
+
}
|
|
19825
|
+
return {
|
|
19826
|
+
targets,
|
|
19827
|
+
plans,
|
|
19828
|
+
documentCount: input.manager.countDocuments(),
|
|
19829
|
+
contents,
|
|
19830
|
+
positions,
|
|
19831
|
+
partitionCount: input.manager.partitionCount
|
|
19832
|
+
};
|
|
19833
|
+
} catch (error) {
|
|
19834
|
+
for (const plan of plans.values()) plan.release();
|
|
19835
|
+
throw error;
|
|
18709
19836
|
}
|
|
18710
19837
|
}
|
|
18711
19838
|
async function writeCaptured(input, store, prior, captured) {
|
|
@@ -18729,7 +19856,8 @@ async function writeCaptured(input, store, prior, captured) {
|
|
|
18729
19856
|
await input.onCommitted({
|
|
18730
19857
|
saved: committed.saved,
|
|
18731
19858
|
documentCount: captured.documentCount,
|
|
18732
|
-
partitionCount: captured.partitionCount
|
|
19859
|
+
partitionCount: captured.partitionCount,
|
|
19860
|
+
positions: captured.positions
|
|
18733
19861
|
});
|
|
18734
19862
|
const garbage = await checkpointGarbage(store, input.indexName, committed.manifest);
|
|
18735
19863
|
await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, input.vectorIndexes, garbage));
|
|
@@ -18751,7 +19879,6 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
18751
19879
|
});
|
|
18752
19880
|
try {
|
|
18753
19881
|
await writeCaptured(input, store, prior, captured);
|
|
18754
|
-
return { positions: captured.positions, partitionCount: captured.partitionCount };
|
|
18755
19882
|
} finally {
|
|
18756
19883
|
for (const plan of captured.plans.values()) plan.release();
|
|
18757
19884
|
}
|
|
@@ -18845,6 +19972,41 @@ function createSnapshotSavedSize(adapter, settleRestore) {
|
|
|
18845
19972
|
};
|
|
18846
19973
|
}
|
|
18847
19974
|
|
|
19975
|
+
// src/persistence/durability/snapshot-only/shutdown-save.ts
|
|
19976
|
+
function asError(error) {
|
|
19977
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
19978
|
+
}
|
|
19979
|
+
async function saveIndex(deps, indexName, indexState) {
|
|
19980
|
+
await indexState.applyChain;
|
|
19981
|
+
await checkpointInTurn(indexState, {
|
|
19982
|
+
joinsInFlight: false,
|
|
19983
|
+
mayStart: () => !indexState.unloading && indexState.mutationsSinceCheckpoint > 0,
|
|
19984
|
+
perform: () => indexState.applyChain.then(() => deps.checkpoint(indexName, indexState)),
|
|
19985
|
+
afterSettling: () => void 0
|
|
19986
|
+
});
|
|
19987
|
+
}
|
|
19988
|
+
async function saveUnsavedIndexes(deps) {
|
|
19989
|
+
let failure = null;
|
|
19990
|
+
for (const [indexName, indexState] of deps.indexes) {
|
|
19991
|
+
try {
|
|
19992
|
+
await saveIndex(deps, indexName, indexState);
|
|
19993
|
+
} catch (error) {
|
|
19994
|
+
const failed = asError(error);
|
|
19995
|
+
deps.onFailure(failed);
|
|
19996
|
+
failure ??= failed;
|
|
19997
|
+
}
|
|
19998
|
+
}
|
|
19999
|
+
if (failure !== null) throw failure;
|
|
20000
|
+
}
|
|
20001
|
+
async function saveAndReleaseIndexes(deps) {
|
|
20002
|
+
try {
|
|
20003
|
+
await saveUnsavedIndexes(deps);
|
|
20004
|
+
} finally {
|
|
20005
|
+
await deps.settleMetadataWrites();
|
|
20006
|
+
deps.release();
|
|
20007
|
+
}
|
|
20008
|
+
}
|
|
20009
|
+
|
|
18848
20010
|
// src/persistence/durability/snapshot-only/index.ts
|
|
18849
20011
|
function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
18850
20012
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
@@ -18894,7 +20056,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
18894
20056
|
const manager = hooks.getManager(indexName);
|
|
18895
20057
|
const metadata = hooks.buildTermsMetadata(indexName);
|
|
18896
20058
|
if (manager === void 0 || metadata === void 0) return;
|
|
18897
|
-
|
|
20059
|
+
await writeSnapshotOnlyCheckpoint({
|
|
18898
20060
|
adapter,
|
|
18899
20061
|
indexName,
|
|
18900
20062
|
metadata,
|
|
@@ -18903,12 +20065,12 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
18903
20065
|
appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
|
|
18904
20066
|
rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName),
|
|
18905
20067
|
onCommitted: async (committed) => {
|
|
20068
|
+
noteSnapshotCommitted(indexState, committed.positions, committed.partitionCount);
|
|
18906
20069
|
savedSize.record(indexName, committed.saved);
|
|
18907
20070
|
hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
|
|
18908
20071
|
await queueMetadataWrite(indexName, () => saveCheckpointMetadata(adapter, hooks, indexName, committed));
|
|
18909
20072
|
}
|
|
18910
20073
|
});
|
|
18911
|
-
noteSnapshotCommitted(indexState, positions, partitionCount);
|
|
18912
20074
|
if (publisher !== void 0) {
|
|
18913
20075
|
const partitionIds = [];
|
|
18914
20076
|
for (let i = 0; i < manager.partitionCount; i += 1) {
|
|
@@ -18999,11 +20161,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
18999
20161
|
}
|
|
19000
20162
|
}
|
|
19001
20163
|
async function recordMutation(record) {
|
|
20164
|
+
if (shuttingDown) throw new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
19002
20165
|
const indexState = getOrCreateIndexState(record.indexName);
|
|
19003
20166
|
const buffered = indexState.applyChain.then(async () => {
|
|
19004
20167
|
await record.apply();
|
|
19005
20168
|
const appliedSeqNo2 = (indexState.appliedSeqNoByPartition.get(record.partitionId) ?? 0) + 1;
|
|
19006
20169
|
indexState.appliedSeqNoByPartition.set(record.partitionId, appliedSeqNo2);
|
|
20170
|
+
indexState.mutationsSinceCheckpoint += 1;
|
|
19007
20171
|
return appliedSeqNo2;
|
|
19008
20172
|
});
|
|
19009
20173
|
indexState.applyChain = buffered.then(
|
|
@@ -19011,7 +20175,6 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
19011
20175
|
() => void 0
|
|
19012
20176
|
);
|
|
19013
20177
|
const appliedSeqNo = await buffered;
|
|
19014
|
-
indexState.mutationsSinceCheckpoint += 1;
|
|
19015
20178
|
startCheckpointTimer();
|
|
19016
20179
|
if (indexState.mutationsSinceCheckpoint >= checkpointMutationThreshold) {
|
|
19017
20180
|
void checkpointIndex(record.indexName).catch((err) => {
|
|
@@ -19146,12 +20309,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
19146
20309
|
checkpointTimer = null;
|
|
19147
20310
|
}
|
|
19148
20311
|
await restores.settleAll();
|
|
19149
|
-
|
|
19150
|
-
|
|
19151
|
-
|
|
19152
|
-
|
|
19153
|
-
|
|
19154
|
-
|
|
20312
|
+
await saveAndReleaseIndexes({
|
|
20313
|
+
indexes,
|
|
20314
|
+
checkpoint: (indexName, indexState) => performCheckpoint(indexName, indexState, false),
|
|
20315
|
+
onFailure: hooks.onFatalError,
|
|
20316
|
+
settleMetadataWrites: () => metadataWrites.settleAll(),
|
|
20317
|
+
release: () => indexes.clear()
|
|
20318
|
+
});
|
|
19155
20319
|
}
|
|
19156
20320
|
};
|
|
19157
20321
|
}
|
|
@@ -19198,7 +20362,11 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
19198
20362
|
function readerOf(indexName) {
|
|
19199
20363
|
return hooks.previousDocumentReader === void 0 ? null : hooks.previousDocumentReader(indexName);
|
|
19200
20364
|
}
|
|
19201
|
-
const manager = tier.kind === "wal" ? createDurabilityManager(
|
|
20365
|
+
const manager = tier.kind === "wal" ? createDurabilityManager(
|
|
20366
|
+
tier.config,
|
|
20367
|
+
managerHooks,
|
|
20368
|
+
tier.indexedDB === void 0 ? void 0 : createIndexedDBDirectory(tier.indexedDB, { relaxedAppends: tier.config.mode === "async" })
|
|
20369
|
+
) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
19202
20370
|
async function recordMutation(indexName, docId, operation, document, apply) {
|
|
19203
20371
|
return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
|
|
19204
20372
|
const partitionId = fnv1a(docId) % partitionCount;
|