@peerbit/native-backbone 0.2.8 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -20
- package/dist/src/index.d.ts +101 -8
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1120 -74
- package/dist/src/index.js.map +1 -1
- package/dist/wasm/README.md +55 -20
- package/dist/wasm/native_backbone.d.ts +118 -79
- package/dist/wasm/native_backbone.js +227 -0
- package/dist/wasm/native_backbone_bg.wasm +0 -0
- package/dist/wasm/native_backbone_bg.wasm.d.ts +87 -79
- package/package.json +1 -1
- package/src/append_tx/committed_latest.rs +10 -5
- package/src/append_tx/committed_no_next.rs +6 -3
- package/src/append_tx/facts.rs +427 -1
- package/src/append_tx/mod.rs +51 -18
- package/src/append_tx/storage.rs +12 -2
- package/src/index.ts +1703 -231
package/dist/src/index.js
CHANGED
|
@@ -168,6 +168,19 @@ const nativeBackboneAppendProfileKeys = [
|
|
|
168
168
|
"nativeBackboneRawReceiveGraphPutMs",
|
|
169
169
|
"nativeBackboneRawReceiveCoordinateCommitMs",
|
|
170
170
|
];
|
|
171
|
+
export class NativeBackboneCoordinatePersistenceReadLimitError extends Error {
|
|
172
|
+
file;
|
|
173
|
+
maxBytes;
|
|
174
|
+
observedBytes;
|
|
175
|
+
code = "ERR_NATIVE_BACKBONE_COORDINATE_READ_LIMIT";
|
|
176
|
+
constructor(file, maxBytes, observedBytes) {
|
|
177
|
+
super(`Native backbone coordinate persistence file ${file} exceeds the ${maxBytes} byte read limit (${observedBytes.toString()} bytes)`);
|
|
178
|
+
this.file = file;
|
|
179
|
+
this.maxBytes = maxBytes;
|
|
180
|
+
this.observedBytes = observedBytes;
|
|
181
|
+
this.name = "NativeBackboneCoordinatePersistenceReadLimitError";
|
|
182
|
+
}
|
|
183
|
+
}
|
|
171
184
|
const nativeBackboneCoordinatePersistenceFiles = {
|
|
172
185
|
snapshot: "coordinates.bin",
|
|
173
186
|
journal: "coordinates.wal",
|
|
@@ -178,9 +191,9 @@ const nativeBackboneCoordinatePersistenceFiles = {
|
|
|
178
191
|
};
|
|
179
192
|
export const nativeBackboneCoordinateDropTombstoneFile = "native-backbone-drop.tombstone";
|
|
180
193
|
export const defaultNativeBackboneCoordinateFlushMaxPendingBytes = 1024 * 1024;
|
|
181
|
-
/**
|
|
194
|
+
/** Recommended explicit byte threshold for crash-safe Node checkpointing. */
|
|
182
195
|
export const defaultNativeBackboneCoordinateCompactMaxJournalBytes = 64 * 1024 * 1024;
|
|
183
|
-
const nativeBackboneCoordinateCompactionDisabledMessage = "Native backbone coordinate persistence compaction is disabled
|
|
196
|
+
const nativeBackboneCoordinateCompactionDisabledMessage = "Native backbone coordinate persistence compaction is disabled for stores without crash-safe atomic replacement";
|
|
184
197
|
const nativeBackboneCoordinateJournalMagic = Uint8Array.from([
|
|
185
198
|
0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x57, 0x31,
|
|
186
199
|
]);
|
|
@@ -196,6 +209,171 @@ const coordinateJournalChecksum = (bytes) => {
|
|
|
196
209
|
}
|
|
197
210
|
return checksum;
|
|
198
211
|
};
|
|
212
|
+
const nativeBackboneCoordinateCheckpointMagic = Uint8Array.from([
|
|
213
|
+
0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x43, 0x31,
|
|
214
|
+
]);
|
|
215
|
+
const nativeBackboneCoordinateCheckpointStateMagic = Uint8Array.from([
|
|
216
|
+
0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x53, 0x31,
|
|
217
|
+
]);
|
|
218
|
+
const nativeBackboneCoordinateLegacyMigrationSentinel = (() => {
|
|
219
|
+
const payload = Uint8Array.of(0xff);
|
|
220
|
+
const bytes = new Uint8Array(nativeBackboneCoordinateJournalMagic.byteLength + 8 + payload.byteLength);
|
|
221
|
+
bytes.set(nativeBackboneCoordinateJournalMagic, 0);
|
|
222
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
223
|
+
view.setUint32(nativeBackboneCoordinateJournalMagic.byteLength, 1, true);
|
|
224
|
+
view.setUint32(nativeBackboneCoordinateJournalMagic.byteLength + 4, coordinateJournalChecksum(payload), true);
|
|
225
|
+
bytes.set(payload, nativeBackboneCoordinateJournalMagic.byteLength + 8);
|
|
226
|
+
return bytes;
|
|
227
|
+
})();
|
|
228
|
+
const nativeBackboneCoordinateCheckpointVersion = 1;
|
|
229
|
+
const nativeBackboneCoordinateCheckpointHeaderBytes = 56;
|
|
230
|
+
const nativeBackboneCoordinateCheckpointStateBytes = 60;
|
|
231
|
+
const nativeBackboneCoordinateMaxU64 = (1n << 64n) - 1n;
|
|
232
|
+
const hasBytesPrefix = (bytes, prefix) => bytes.byteLength >= prefix.byteLength &&
|
|
233
|
+
prefix.every((byte, index) => bytes[index] === byte);
|
|
234
|
+
const writeCoordinateCheckpointU64 = (view, offset, value) => {
|
|
235
|
+
if (value < 0n || value > nativeBackboneCoordinateMaxU64) {
|
|
236
|
+
throw new RangeError("Native backbone checkpoint generation exceeds u64");
|
|
237
|
+
}
|
|
238
|
+
view.setUint32(offset, Number(value & 0xffffffffn), true);
|
|
239
|
+
view.setUint32(offset + 4, Number(value >> 32n), true);
|
|
240
|
+
};
|
|
241
|
+
const readCoordinateCheckpointU64 = (view, offset) => BigInt(view.getUint32(offset, true)) |
|
|
242
|
+
(BigInt(view.getUint32(offset + 4, true)) << 32n);
|
|
243
|
+
const encodeNativeBackboneCoordinateCheckpoint = (checkpoint) => {
|
|
244
|
+
const coordinateLength = checkpoint.coordinateSnapshot.byteLength;
|
|
245
|
+
const documentLength = checkpoint.documentSnapshot.byteLength;
|
|
246
|
+
const signerLength = checkpoint.documentSignerSnapshot.byteLength;
|
|
247
|
+
const byteLength = nativeBackboneCoordinateCheckpointHeaderBytes +
|
|
248
|
+
coordinateLength +
|
|
249
|
+
documentLength +
|
|
250
|
+
signerLength;
|
|
251
|
+
if (!Number.isSafeInteger(byteLength) || byteLength > 0xffff_ffff) {
|
|
252
|
+
throw new RangeError("Native backbone coordinate checkpoint is too large");
|
|
253
|
+
}
|
|
254
|
+
const bytes = new Uint8Array(byteLength);
|
|
255
|
+
bytes.set(nativeBackboneCoordinateCheckpointMagic, 0);
|
|
256
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
257
|
+
view.setUint32(8, nativeBackboneCoordinateCheckpointVersion, true);
|
|
258
|
+
view.setUint32(12, nativeBackboneCoordinateCheckpointHeaderBytes, true);
|
|
259
|
+
writeCoordinateCheckpointU64(view, 16, checkpoint.generation);
|
|
260
|
+
view.setUint32(24, coordinateLength, true);
|
|
261
|
+
view.setUint32(28, documentLength, true);
|
|
262
|
+
view.setUint32(32, signerLength, true);
|
|
263
|
+
view.setUint32(36, coordinateJournalChecksum(checkpoint.coordinateSnapshot), true);
|
|
264
|
+
view.setUint32(40, coordinateJournalChecksum(checkpoint.documentSnapshot), true);
|
|
265
|
+
view.setUint32(44, coordinateJournalChecksum(checkpoint.documentSignerSnapshot), true);
|
|
266
|
+
view.setUint32(48, checkpoint.configurationChecksum, true);
|
|
267
|
+
view.setUint32(52, coordinateJournalChecksum(bytes.subarray(8, 52)), true);
|
|
268
|
+
let offset = nativeBackboneCoordinateCheckpointHeaderBytes;
|
|
269
|
+
bytes.set(checkpoint.coordinateSnapshot, offset);
|
|
270
|
+
offset += coordinateLength;
|
|
271
|
+
bytes.set(checkpoint.documentSnapshot, offset);
|
|
272
|
+
offset += documentLength;
|
|
273
|
+
bytes.set(checkpoint.documentSignerSnapshot, offset);
|
|
274
|
+
return bytes;
|
|
275
|
+
};
|
|
276
|
+
const parseNativeBackboneCoordinateCheckpoint = (bytes, name) => {
|
|
277
|
+
if (bytes.byteLength < nativeBackboneCoordinateCheckpointHeaderBytes ||
|
|
278
|
+
!hasBytesPrefix(bytes, nativeBackboneCoordinateCheckpointMagic)) {
|
|
279
|
+
throw new Error(`Native backbone ${name} has an invalid checkpoint header`);
|
|
280
|
+
}
|
|
281
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
282
|
+
if (view.getUint32(8, true) !== nativeBackboneCoordinateCheckpointVersion ||
|
|
283
|
+
view.getUint32(12, true) !==
|
|
284
|
+
nativeBackboneCoordinateCheckpointHeaderBytes ||
|
|
285
|
+
view.getUint32(52, true) !==
|
|
286
|
+
coordinateJournalChecksum(bytes.subarray(8, 52))) {
|
|
287
|
+
throw new Error(`Native backbone ${name} has invalid checkpoint metadata`);
|
|
288
|
+
}
|
|
289
|
+
const coordinateLength = view.getUint32(24, true);
|
|
290
|
+
const documentLength = view.getUint32(28, true);
|
|
291
|
+
const signerLength = view.getUint32(32, true);
|
|
292
|
+
const expectedLength = nativeBackboneCoordinateCheckpointHeaderBytes +
|
|
293
|
+
coordinateLength +
|
|
294
|
+
documentLength +
|
|
295
|
+
signerLength;
|
|
296
|
+
if (!Number.isSafeInteger(expectedLength) ||
|
|
297
|
+
expectedLength !== bytes.byteLength) {
|
|
298
|
+
throw new Error(`Native backbone ${name} has an invalid checkpoint length`);
|
|
299
|
+
}
|
|
300
|
+
let offset = nativeBackboneCoordinateCheckpointHeaderBytes;
|
|
301
|
+
const coordinateSnapshot = bytes.subarray(offset, offset + coordinateLength);
|
|
302
|
+
offset += coordinateLength;
|
|
303
|
+
const documentSnapshot = bytes.subarray(offset, offset + documentLength);
|
|
304
|
+
offset += documentLength;
|
|
305
|
+
const documentSignerSnapshot = bytes.subarray(offset, offset + signerLength);
|
|
306
|
+
if (coordinateJournalChecksum(coordinateSnapshot) !==
|
|
307
|
+
view.getUint32(36, true) ||
|
|
308
|
+
coordinateJournalChecksum(documentSnapshot) !== view.getUint32(40, true) ||
|
|
309
|
+
coordinateJournalChecksum(documentSignerSnapshot) !==
|
|
310
|
+
view.getUint32(44, true)) {
|
|
311
|
+
throw new Error(`Native backbone ${name} has a checkpoint checksum mismatch`);
|
|
312
|
+
}
|
|
313
|
+
return {
|
|
314
|
+
configurationChecksum: view.getUint32(48, true),
|
|
315
|
+
generation: readCoordinateCheckpointU64(view, 16),
|
|
316
|
+
coordinateSnapshot,
|
|
317
|
+
documentSnapshot,
|
|
318
|
+
documentSignerSnapshot,
|
|
319
|
+
};
|
|
320
|
+
};
|
|
321
|
+
const encodeNativeBackboneCoordinateCheckpointState = (state) => {
|
|
322
|
+
const bytes = new Uint8Array(nativeBackboneCoordinateCheckpointStateBytes);
|
|
323
|
+
bytes.set(nativeBackboneCoordinateCheckpointStateMagic, 0);
|
|
324
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
325
|
+
view.setUint32(8, nativeBackboneCoordinateCheckpointVersion, true);
|
|
326
|
+
view.setUint32(12, state.configurationChecksum, true);
|
|
327
|
+
writeCoordinateCheckpointU64(view, 16, state.highwater);
|
|
328
|
+
writeCoordinateCheckpointU64(view, 24, state.pending?.generation ?? 0n);
|
|
329
|
+
view.setUint32(32, state.pending?.byteLength ?? 0, true);
|
|
330
|
+
view.setUint32(36, state.pending?.checksum ?? 0, true);
|
|
331
|
+
writeCoordinateCheckpointU64(view, 40, state.completed?.generation ?? 0n);
|
|
332
|
+
view.setUint32(48, state.completed?.byteLength ?? 0, true);
|
|
333
|
+
view.setUint32(52, state.completed?.checksum ?? 0, true);
|
|
334
|
+
view.setUint32(56, coordinateJournalChecksum(bytes.subarray(8, 56)), true);
|
|
335
|
+
return bytes;
|
|
336
|
+
};
|
|
337
|
+
const parseNativeBackboneCoordinateCheckpointState = (bytes, name) => {
|
|
338
|
+
if (bytes.byteLength !== nativeBackboneCoordinateCheckpointStateBytes ||
|
|
339
|
+
!hasBytesPrefix(bytes, nativeBackboneCoordinateCheckpointStateMagic)) {
|
|
340
|
+
throw new Error(`Native backbone ${name} has an invalid checkpoint authority`);
|
|
341
|
+
}
|
|
342
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
343
|
+
if (view.getUint32(8, true) !== nativeBackboneCoordinateCheckpointVersion ||
|
|
344
|
+
view.getUint32(56, true) !==
|
|
345
|
+
coordinateJournalChecksum(bytes.subarray(8, 56))) {
|
|
346
|
+
throw new Error(`Native backbone ${name} has corrupt checkpoint authority`);
|
|
347
|
+
}
|
|
348
|
+
const configurationChecksum = view.getUint32(12, true);
|
|
349
|
+
const highwater = readCoordinateCheckpointU64(view, 16);
|
|
350
|
+
const pendingGeneration = readCoordinateCheckpointU64(view, 24);
|
|
351
|
+
const completedGeneration = readCoordinateCheckpointU64(view, 40);
|
|
352
|
+
const authority = (generation, lengthOffset, checksumOffset) => {
|
|
353
|
+
const byteLength = view.getUint32(lengthOffset, true);
|
|
354
|
+
const checksum = view.getUint32(checksumOffset, true);
|
|
355
|
+
if (generation === 0n) {
|
|
356
|
+
if (byteLength !== 0 || checksum !== 0) {
|
|
357
|
+
throw new Error(`Native backbone ${name} has empty authority metadata`);
|
|
358
|
+
}
|
|
359
|
+
return undefined;
|
|
360
|
+
}
|
|
361
|
+
if (byteLength < nativeBackboneCoordinateCheckpointHeaderBytes) {
|
|
362
|
+
throw new Error(`Native backbone ${name} has invalid authority length`);
|
|
363
|
+
}
|
|
364
|
+
return { generation, byteLength, checksum };
|
|
365
|
+
};
|
|
366
|
+
const pending = authority(pendingGeneration, 32, 36);
|
|
367
|
+
const completed = authority(completedGeneration, 48, 52);
|
|
368
|
+
if ((completed && completed.generation > highwater) ||
|
|
369
|
+
(pending && pending.generation !== highwater) ||
|
|
370
|
+
(pending && completed && pending.generation <= completed.generation) ||
|
|
371
|
+
(!pending && completed && completed.generation !== highwater) ||
|
|
372
|
+
(!pending && !completed && highwater !== 0n)) {
|
|
373
|
+
throw new Error(`Native backbone ${name} has invalid generation authority`);
|
|
374
|
+
}
|
|
375
|
+
return { configurationChecksum, highwater, pending, completed };
|
|
376
|
+
};
|
|
199
377
|
const hasCoordinateJournalMagic = (bytes) => bytes.byteLength >= nativeBackboneCoordinateJournalMagic.byteLength &&
|
|
200
378
|
nativeBackboneCoordinateJournalMagic.every((byte, index) => bytes[index] === byte);
|
|
201
379
|
/**
|
|
@@ -233,6 +411,12 @@ const resolveCoordinateFlushMaxPendingBytes = (options) => options.flushMaxPendi
|
|
|
233
411
|
: options.flushOnAppend === false
|
|
234
412
|
? defaultNativeBackboneCoordinateFlushMaxPendingBytes
|
|
235
413
|
: undefined;
|
|
414
|
+
const validateCoordinateCompactionThreshold = (value, name) => {
|
|
415
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
416
|
+
throw new RangeError(`Native backbone ${name} must be a positive safe integer`);
|
|
417
|
+
}
|
|
418
|
+
return value;
|
|
419
|
+
};
|
|
236
420
|
const isNotFoundError = (error) => {
|
|
237
421
|
const maybeError = error;
|
|
238
422
|
return maybeError?.code === "ENOENT" || maybeError?.name === "NotFoundError";
|
|
@@ -280,6 +464,17 @@ const validateCoordinatePersistenceName = (name) => {
|
|
|
280
464
|
}
|
|
281
465
|
return name;
|
|
282
466
|
};
|
|
467
|
+
const validateCoordinatePersistenceReadMaxBytes = (maxBytes) => {
|
|
468
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) {
|
|
469
|
+
throw new RangeError("Native backbone coordinate persistence read limit must be a non-negative safe integer");
|
|
470
|
+
}
|
|
471
|
+
return maxBytes;
|
|
472
|
+
};
|
|
473
|
+
const assertCoordinatePersistenceReadWithinLimit = (name, maxBytes, observedBytes) => {
|
|
474
|
+
if (BigInt(observedBytes) > BigInt(maxBytes)) {
|
|
475
|
+
throw new NativeBackboneCoordinatePersistenceReadLimitError(name, maxBytes, BigInt(observedBytes));
|
|
476
|
+
}
|
|
477
|
+
};
|
|
283
478
|
const nativeBackboneCoordinateDropTombstoneBodyBytes = (body) => new TextEncoder().encode(JSON.stringify(body));
|
|
284
479
|
const nativeBackboneCoordinateDropChecksum = (bytes) => {
|
|
285
480
|
let checksum = 0xffffffff;
|
|
@@ -632,7 +827,7 @@ const requestPruneEntryFromRow = (row) => {
|
|
|
632
827
|
return entry;
|
|
633
828
|
};
|
|
634
829
|
const storageAppendResultFromRow = (resolution, row) => {
|
|
635
|
-
const [entryRow, leaderRows, isLeader, assignedToRangeBoundary, coordinateRow, trimRows, trimHashRows, documentTrimmedHeadsProcessed, documentPreviousContextRow,] = row;
|
|
830
|
+
const [entryRow, leaderRows, isLeader, assignedToRangeBoundary, coordinateRow, trimRows, trimHashRows, documentTrimmedHeadsProcessed, documentPreviousContextRow, _trimGidExtension, trimGidRows,] = row;
|
|
636
831
|
return {
|
|
637
832
|
entry: storageFactsEntryFromRow(entryRow),
|
|
638
833
|
leaders: rowsToSamples(leaderRows),
|
|
@@ -640,12 +835,13 @@ const storageAppendResultFromRow = (resolution, row) => {
|
|
|
640
835
|
assignedToRangeBoundary,
|
|
641
836
|
coordinate: appendCoordinatePlanFromRow(resolution, coordinateRow),
|
|
642
837
|
...trimmedRowsAndHashesResult(trimRows, trimHashRows),
|
|
838
|
+
trimmedGids: trimGidRows ?? trimRows.map((trim) => trim[1]),
|
|
643
839
|
documentTrimmedHeadsProcessed,
|
|
644
840
|
documentPreviousContext: documentContextFactsFromRow(documentPreviousContextRow),
|
|
645
841
|
};
|
|
646
842
|
};
|
|
647
843
|
const committedStorageAppendResultFromRow = (resolution, row) => {
|
|
648
|
-
const [entryRow, leaderRows, isLeader, assignedToRangeBoundary, coordinateRow, trimRows, trimHashRows, documentTrimmedHeadsProcessed, documentPreviousContextRow,] = row;
|
|
844
|
+
const [entryRow, leaderRows, isLeader, assignedToRangeBoundary, coordinateRow, trimRows, trimHashRows, documentTrimmedHeadsProcessed, documentPreviousContextRow, _trimGidExtension, trimGidRows,] = row;
|
|
649
845
|
return {
|
|
650
846
|
entry: committedStorageFactsEntryFromRow(entryRow),
|
|
651
847
|
leaders: rowsToSamples(leaderRows),
|
|
@@ -653,15 +849,22 @@ const committedStorageAppendResultFromRow = (resolution, row) => {
|
|
|
653
849
|
assignedToRangeBoundary,
|
|
654
850
|
coordinate: appendCoordinatePlanFromRow(resolution, coordinateRow),
|
|
655
851
|
...trimmedRowsAndHashesResult(trimRows, trimHashRows),
|
|
852
|
+
trimmedGids: trimGidRows ?? trimRows.map((trim) => trim[1]),
|
|
656
853
|
documentTrimmedHeadsProcessed,
|
|
657
854
|
documentPreviousContext: documentContextFactsFromRow(documentPreviousContextRow),
|
|
658
855
|
};
|
|
659
856
|
};
|
|
660
857
|
const compactCommittedNoNextStorageAppendResultFromRow = (resolution, row) => {
|
|
661
|
-
const
|
|
858
|
+
const appendedTrimGids = row.length >= 12 &&
|
|
859
|
+
row[row.length - 2] === undefined &&
|
|
860
|
+
Array.isArray(row[row.length - 1])
|
|
861
|
+
? row[row.length - 1]
|
|
862
|
+
: undefined;
|
|
863
|
+
const baseRow = appendedTrimGids ? row.slice(0, -2) : row;
|
|
864
|
+
const [hash, byteLength, metaBytes, fourth] = baseRow;
|
|
662
865
|
const hasDigestRow = fourth instanceof Uint8Array;
|
|
663
866
|
const hashDigestBytes = hasDigestRow ? fourth : undefined;
|
|
664
|
-
const rest =
|
|
867
|
+
const rest = baseRow.slice(hasDigestRow ? 4 : 3);
|
|
665
868
|
const usesNestedCoordinateRow = Array.isArray(rest[0]);
|
|
666
869
|
let coordinate;
|
|
667
870
|
let leaderRows;
|
|
@@ -709,11 +912,18 @@ const compactCommittedNoNextStorageAppendResultFromRow = (resolution, row) => {
|
|
|
709
912
|
coordinate,
|
|
710
913
|
trimmed: [],
|
|
711
914
|
trimmedHashes: trimHashRows ?? [],
|
|
915
|
+
trimmedGids: appendedTrimGids,
|
|
712
916
|
documentTrimmedHeadsProcessed,
|
|
713
917
|
};
|
|
714
918
|
};
|
|
715
919
|
const compactCommittedLatestStorageAppendResultFromRow = (resolution, row) => {
|
|
716
|
-
const
|
|
920
|
+
const appendedTrimGids = row.length >= 13 &&
|
|
921
|
+
row[row.length - 2] === undefined &&
|
|
922
|
+
Array.isArray(row[row.length - 1])
|
|
923
|
+
? row[row.length - 1]
|
|
924
|
+
: undefined;
|
|
925
|
+
const baseRow = appendedTrimGids ? row.slice(0, -2) : row;
|
|
926
|
+
const [hash, byteLength, metaBytes, fourth, fifth, sixth, seventh, eighth, ninth, tenth, eleventh, twelfth,] = baseRow;
|
|
717
927
|
const hasDigestRow = fourth instanceof Uint8Array;
|
|
718
928
|
const hashDigestBytes = hasDigestRow ? fourth : undefined;
|
|
719
929
|
const next = (hasDigestRow ? fifth : fourth);
|
|
@@ -741,6 +951,7 @@ const compactCommittedLatestStorageAppendResultFromRow = (resolution, row) => {
|
|
|
741
951
|
coordinate,
|
|
742
952
|
trimmed: [],
|
|
743
953
|
trimmedHashes: trimHashRows ?? [],
|
|
954
|
+
trimmedGids: appendedTrimGids,
|
|
744
955
|
documentTrimmedHeadsProcessed,
|
|
745
956
|
documentPreviousContext: documentContextFactsFromRow(documentPreviousContextRow),
|
|
746
957
|
};
|
|
@@ -760,6 +971,14 @@ const preparedCommitFactsFromRow = (row) => {
|
|
|
760
971
|
}
|
|
761
972
|
return prepared;
|
|
762
973
|
};
|
|
974
|
+
const preparedCommitFactsWithTrimRefsFromRow = (row) => {
|
|
975
|
+
const prepared = committedStorageFactsEntryFromRow(row[0]);
|
|
976
|
+
return {
|
|
977
|
+
...prepared,
|
|
978
|
+
trimmedEntryHashes: row[1],
|
|
979
|
+
trimmedEntryGids: row[2],
|
|
980
|
+
};
|
|
981
|
+
};
|
|
763
982
|
const preparedCommitFactsWithLatestDocumentContextFromRow = (row) => {
|
|
764
983
|
const [entryRow, trimHashRows, documentTrimmedHeadsProcessed, contextRow] = row;
|
|
765
984
|
return {
|
|
@@ -769,24 +988,53 @@ const preparedCommitFactsWithLatestDocumentContextFromRow = (row) => {
|
|
|
769
988
|
documentPreviousContext: documentContextFactsFromRow(contextRow),
|
|
770
989
|
};
|
|
771
990
|
};
|
|
772
|
-
const
|
|
991
|
+
const preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow = (row) => {
|
|
992
|
+
const [entryRow, trimHashRows, trimGidRows, documentTrimmedHeadsProcessed, contextRow,] = row;
|
|
993
|
+
return {
|
|
994
|
+
...committedStorageFactsEntryFromRow(entryRow),
|
|
995
|
+
trimmedEntryHashes: trimHashRows,
|
|
996
|
+
trimmedEntryGids: trimGidRows,
|
|
997
|
+
documentTrimmedHeadsProcessed,
|
|
998
|
+
documentPreviousContext: documentContextFactsFromRow(contextRow),
|
|
999
|
+
};
|
|
1000
|
+
};
|
|
1001
|
+
const compactPreparedCommitFactsBaseFromRow = (row) => {
|
|
773
1002
|
const [hash, byteLength, metaBytes, fourth] = row;
|
|
774
1003
|
const hasDigestRow = fourth instanceof Uint8Array;
|
|
775
|
-
const hashDigestBytes = hasDigestRow ? fourth : undefined;
|
|
776
|
-
const trimHashOffset = hasDigestRow ? 4 : 3;
|
|
777
|
-
const trimHashRows = row[trimHashOffset];
|
|
778
|
-
const documentTrimmedHeadsProcessed = row[trimHashOffset + 1];
|
|
779
1004
|
return {
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
1005
|
+
entry: {
|
|
1006
|
+
cid: hash,
|
|
1007
|
+
hash,
|
|
1008
|
+
next: [],
|
|
1009
|
+
metaBytes,
|
|
1010
|
+
byteLength,
|
|
1011
|
+
hashDigestBytes: hasDigestRow ? fourth : undefined,
|
|
1012
|
+
},
|
|
1013
|
+
trimRowOffset: hasDigestRow ? 4 : 3,
|
|
1014
|
+
};
|
|
1015
|
+
};
|
|
1016
|
+
const compactPreparedCommitFactsWithTrimHashesFromRow = (row) => {
|
|
1017
|
+
const { entry, trimRowOffset } = compactPreparedCommitFactsBaseFromRow(row);
|
|
1018
|
+
const trimHashRows = row[trimRowOffset];
|
|
1019
|
+
const documentTrimmedHeadsProcessed = row[trimRowOffset + 1];
|
|
1020
|
+
return {
|
|
1021
|
+
...entry,
|
|
786
1022
|
trimmedEntryHashes: trimHashRows ?? [],
|
|
787
1023
|
documentTrimmedHeadsProcessed,
|
|
788
1024
|
};
|
|
789
1025
|
};
|
|
1026
|
+
const compactPreparedCommitFactsWithTrimRefsFromRow = (row) => {
|
|
1027
|
+
const { entry, trimRowOffset } = compactPreparedCommitFactsBaseFromRow(row);
|
|
1028
|
+
const trimHashRows = row[trimRowOffset];
|
|
1029
|
+
const trimGidRows = row[trimRowOffset + 1];
|
|
1030
|
+
const documentTrimmedHeadsProcessed = row[trimRowOffset + 2];
|
|
1031
|
+
return {
|
|
1032
|
+
...entry,
|
|
1033
|
+
trimmedEntryHashes: trimHashRows,
|
|
1034
|
+
trimmedEntryGids: trimGidRows,
|
|
1035
|
+
documentTrimmedHeadsProcessed,
|
|
1036
|
+
};
|
|
1037
|
+
};
|
|
790
1038
|
const nativeLogCommitEntryColumns = (entries) => {
|
|
791
1039
|
const hashes = new Array(entries.length);
|
|
792
1040
|
const blockBytes = new Array(entries.length);
|
|
@@ -1242,13 +1490,60 @@ class NativeBackboneLogGraph {
|
|
|
1242
1490
|
const documentIndex = input.documentIndex;
|
|
1243
1491
|
const documentIndexArgs = nativeDocumentIndexArgs(documentIndex);
|
|
1244
1492
|
const projection = documentIndex?.projection;
|
|
1493
|
+
if (!documentIndexArgs &&
|
|
1494
|
+
input.resolveTrimmedEntries === false &&
|
|
1495
|
+
input.trimLengthTo != null &&
|
|
1496
|
+
this.native.prepare_plain_entry_commit_facts_trim_refs) {
|
|
1497
|
+
return preparedCommitFactsWithTrimRefsFromRow(this.native.prepare_plain_entry_commit_facts_trim_refs(wallTime, logical, input.gid, input.next ?? [], entryType, input.metaData, input.payloadData, input.trimLengthTo));
|
|
1498
|
+
}
|
|
1245
1499
|
if (documentIndex?.useLatestContext &&
|
|
1246
1500
|
documentIndexArgs &&
|
|
1247
1501
|
input.resolveTrimmedEntries === false) {
|
|
1248
1502
|
if (projection && this.options?.documentProjectionPlanId) {
|
|
1249
|
-
|
|
1503
|
+
const args = [
|
|
1504
|
+
wallTime,
|
|
1505
|
+
logical,
|
|
1506
|
+
input.gid,
|
|
1507
|
+
entryType,
|
|
1508
|
+
input.metaData,
|
|
1509
|
+
input.payloadData,
|
|
1510
|
+
input.trimLengthTo,
|
|
1511
|
+
documentIndex.key,
|
|
1512
|
+
documentIndex.byteElementIndexLimit ?? 0,
|
|
1513
|
+
documentIndex.deleteTrimmedHeads === true,
|
|
1514
|
+
this.options.documentProjectionPlanId(projection.plan),
|
|
1515
|
+
projection.encodedDocument,
|
|
1516
|
+
projection.signer,
|
|
1517
|
+
];
|
|
1518
|
+
const prepareTrimRefs = this.native
|
|
1519
|
+
.prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_refs;
|
|
1520
|
+
if (prepareTrimRefs) {
|
|
1521
|
+
return preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow(prepareTrimRefs.call(this.native, ...args));
|
|
1522
|
+
}
|
|
1523
|
+
return preparedCommitFactsWithLatestDocumentContextFromRow(this.native.prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes(...args));
|
|
1524
|
+
}
|
|
1525
|
+
const args = [
|
|
1526
|
+
wallTime,
|
|
1527
|
+
logical,
|
|
1528
|
+
input.gid,
|
|
1529
|
+
entryType,
|
|
1530
|
+
input.metaData,
|
|
1531
|
+
input.payloadData,
|
|
1532
|
+
input.trimLengthTo,
|
|
1533
|
+
documentIndex.key,
|
|
1534
|
+
documentIndex.valuePrefixBytes ?? EMPTY_UINT8_ARRAY,
|
|
1535
|
+
documentIndex.byteElementIndexLimit ?? 0,
|
|
1536
|
+
documentIndex.deleteTrimmedHeads === true,
|
|
1537
|
+
projection?.plan,
|
|
1538
|
+
projection?.encodedDocument,
|
|
1539
|
+
projection?.signer,
|
|
1540
|
+
];
|
|
1541
|
+
const prepareTrimRefs = this.native
|
|
1542
|
+
.prepare_plain_entry_commit_latest_facts_document_index_trim_refs;
|
|
1543
|
+
if (prepareTrimRefs) {
|
|
1544
|
+
return preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow(prepareTrimRefs.call(this.native, ...args));
|
|
1250
1545
|
}
|
|
1251
|
-
return preparedCommitFactsWithLatestDocumentContextFromRow(this.native.prepare_plain_entry_commit_latest_facts_document_index_trim_hashes(
|
|
1546
|
+
return preparedCommitFactsWithLatestDocumentContextFromRow(this.native.prepare_plain_entry_commit_latest_facts_document_index_trim_hashes(...args));
|
|
1252
1547
|
}
|
|
1253
1548
|
if (documentIndexArgs &&
|
|
1254
1549
|
projection &&
|
|
@@ -1257,16 +1552,57 @@ class NativeBackboneLogGraph {
|
|
|
1257
1552
|
input.trimLengthTo != null &&
|
|
1258
1553
|
hasNoNext) {
|
|
1259
1554
|
const projectionPlanId = this.options.documentProjectionPlanId(projection.plan);
|
|
1555
|
+
const plainPutPayloadArgs = [
|
|
1556
|
+
wallTime,
|
|
1557
|
+
logical,
|
|
1558
|
+
input.gid,
|
|
1559
|
+
entryType,
|
|
1560
|
+
input.metaData,
|
|
1561
|
+
input.payloadData,
|
|
1562
|
+
input.trimLengthTo,
|
|
1563
|
+
documentIndex.key,
|
|
1564
|
+
documentIndex.existingCreated == null
|
|
1565
|
+
? ""
|
|
1566
|
+
: integerString(documentIndex.existingCreated),
|
|
1567
|
+
documentIndex.byteElementIndexLimit ?? 0,
|
|
1568
|
+
documentIndex.deleteTrimmedHeads === true,
|
|
1569
|
+
projectionPlanId,
|
|
1570
|
+
projection.signer,
|
|
1571
|
+
];
|
|
1572
|
+
const plainPutPayloadCommitWithTrimRefs = this.native
|
|
1573
|
+
.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs_plain_put_payload;
|
|
1574
|
+
if (plainPutPayloadCommitWithTrimRefs) {
|
|
1575
|
+
return compactPreparedCommitFactsWithTrimRefsFromRow(plainPutPayloadCommitWithTrimRefs.call(this.native, ...plainPutPayloadArgs));
|
|
1576
|
+
}
|
|
1260
1577
|
const plainPutPayloadCommit = this.native
|
|
1261
1578
|
.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes_plain_put_payload;
|
|
1262
1579
|
if (plainPutPayloadCommit) {
|
|
1263
|
-
return compactPreparedCommitFactsWithTrimHashesFromRow(plainPutPayloadCommit.call(this.native,
|
|
1580
|
+
return compactPreparedCommitFactsWithTrimHashesFromRow(plainPutPayloadCommit.call(this.native, ...plainPutPayloadArgs));
|
|
1581
|
+
}
|
|
1582
|
+
const args = [
|
|
1583
|
+
wallTime,
|
|
1584
|
+
logical,
|
|
1585
|
+
input.gid,
|
|
1586
|
+
entryType,
|
|
1587
|
+
input.metaData,
|
|
1588
|
+
input.payloadData,
|
|
1589
|
+
input.trimLengthTo,
|
|
1590
|
+
documentIndex.key,
|
|
1591
|
+
documentIndex.existingCreated == null
|
|
1264
1592
|
? ""
|
|
1265
|
-
: integerString(documentIndex.existingCreated),
|
|
1593
|
+
: integerString(documentIndex.existingCreated),
|
|
1594
|
+
documentIndex.byteElementIndexLimit ?? 0,
|
|
1595
|
+
documentIndex.deleteTrimmedHeads === true,
|
|
1596
|
+
projectionPlanId,
|
|
1597
|
+
projection.encodedDocument,
|
|
1598
|
+
projection.signer,
|
|
1599
|
+
];
|
|
1600
|
+
const prepareTrimRefs = this.native
|
|
1601
|
+
.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs;
|
|
1602
|
+
if (prepareTrimRefs) {
|
|
1603
|
+
return compactPreparedCommitFactsWithTrimRefsFromRow(prepareTrimRefs.call(this.native, ...args));
|
|
1266
1604
|
}
|
|
1267
|
-
return compactPreparedCommitFactsWithTrimHashesFromRow(this.native.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes(
|
|
1268
|
-
? ""
|
|
1269
|
-
: integerString(documentIndex.existingCreated), documentIndex.byteElementIndexLimit ?? 0, documentIndex.deleteTrimmedHeads === true, projectionPlanId, projection.encodedDocument, projection.signer));
|
|
1605
|
+
return compactPreparedCommitFactsWithTrimHashesFromRow(this.native.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes(...args));
|
|
1270
1606
|
}
|
|
1271
1607
|
if (documentIndexArgs &&
|
|
1272
1608
|
projection &&
|
|
@@ -1309,6 +1645,11 @@ class NativeBackboneLogGraph {
|
|
|
1309
1645
|
input.resolveTrimmedEntries === false &&
|
|
1310
1646
|
input.trimLengthTo != null &&
|
|
1311
1647
|
hasNoNext) {
|
|
1648
|
+
const prepareTrimRefs = this.native
|
|
1649
|
+
.prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_refs;
|
|
1650
|
+
if (prepareTrimRefs) {
|
|
1651
|
+
return compactPreparedCommitFactsWithTrimRefsFromRow(prepareTrimRefs.call(this.native, wallTime, logical, input.gid, entryType, input.metaData, input.payloadData, input.trimLengthTo, ...documentIndexArgs));
|
|
1652
|
+
}
|
|
1312
1653
|
return compactPreparedCommitFactsWithTrimHashesFromRow(this.native.prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_hashes(wallTime, logical, input.gid, entryType, input.metaData, input.payloadData, input.trimLengthTo, ...documentIndexArgs));
|
|
1313
1654
|
}
|
|
1314
1655
|
const baseArgs = [
|
|
@@ -2961,6 +3302,15 @@ export class NativeBackboneMemoryCoordinatePersistenceStore {
|
|
|
2961
3302
|
const file = this.files.get(validateCoordinatePersistenceName(name));
|
|
2962
3303
|
return file ? copyBytes(file) : undefined;
|
|
2963
3304
|
}
|
|
3305
|
+
async readLimited(name, maxBytes) {
|
|
3306
|
+
const validName = validateCoordinatePersistenceName(name);
|
|
3307
|
+
const limit = validateCoordinatePersistenceReadMaxBytes(maxBytes);
|
|
3308
|
+
const file = this.files.get(validName);
|
|
3309
|
+
if (file) {
|
|
3310
|
+
assertCoordinatePersistenceReadWithinLimit(validName, limit, file.byteLength);
|
|
3311
|
+
}
|
|
3312
|
+
return file ? copyBytes(file) : undefined;
|
|
3313
|
+
}
|
|
2964
3314
|
async write(name, bytes) {
|
|
2965
3315
|
this.files.set(validateCoordinatePersistenceName(name), copyBytes(bytes));
|
|
2966
3316
|
}
|
|
@@ -2981,7 +3331,9 @@ export class NativeBackboneNodeCoordinatePersistenceStore {
|
|
|
2981
3331
|
filePaths = new Map();
|
|
2982
3332
|
directoryEnsured = false;
|
|
2983
3333
|
appendFailure;
|
|
3334
|
+
readLimited;
|
|
2984
3335
|
durableBarrier;
|
|
3336
|
+
atomicReplace;
|
|
2985
3337
|
constructor(directory, fs) {
|
|
2986
3338
|
this.directory = directory;
|
|
2987
3339
|
this.fs = fs;
|
|
@@ -2991,6 +3343,13 @@ export class NativeBackboneNodeCoordinatePersistenceStore {
|
|
|
2991
3343
|
if (!fs || typeof fs.open === "function") {
|
|
2992
3344
|
this.durableBarrier = (name) => this.syncDurably(name);
|
|
2993
3345
|
}
|
|
3346
|
+
if (!fs || typeof fs.openBoundedRead === "function") {
|
|
3347
|
+
this.readLimited = (name, maxBytes) => this.readWithinLimit(name, maxBytes);
|
|
3348
|
+
}
|
|
3349
|
+
if (!fs ||
|
|
3350
|
+
(typeof fs.open === "function" && typeof fs.rename === "function")) {
|
|
3351
|
+
this.atomicReplace = (name, bytes) => this.replaceAtomically(name, bytes);
|
|
3352
|
+
}
|
|
2994
3353
|
}
|
|
2995
3354
|
async nodeFs() {
|
|
2996
3355
|
return this.fs ?? (await importNodeFs());
|
|
@@ -3043,12 +3402,122 @@ export class NativeBackboneNodeCoordinatePersistenceStore {
|
|
|
3043
3402
|
throw error;
|
|
3044
3403
|
}
|
|
3045
3404
|
}
|
|
3405
|
+
async readWithinLimit(name, maxBytes) {
|
|
3406
|
+
const validName = validateCoordinatePersistenceName(name);
|
|
3407
|
+
const limit = validateCoordinatePersistenceReadMaxBytes(maxBytes);
|
|
3408
|
+
const fs = await this.nodeFs();
|
|
3409
|
+
const path = await this.filePath(validName);
|
|
3410
|
+
let handle;
|
|
3411
|
+
try {
|
|
3412
|
+
if (this.fs) {
|
|
3413
|
+
handle = await this.fs.openBoundedRead(path);
|
|
3414
|
+
}
|
|
3415
|
+
else {
|
|
3416
|
+
if (!fs.open) {
|
|
3417
|
+
throw new Error("Default Node coordinate persistence does not expose FileHandle.open");
|
|
3418
|
+
}
|
|
3419
|
+
const opened = await fs.open(path, "r");
|
|
3420
|
+
if (!opened.stat || !opened.read) {
|
|
3421
|
+
await opened.close();
|
|
3422
|
+
throw new Error("Default Node coordinate persistence bounded reads require FileHandle.stat and FileHandle.read");
|
|
3423
|
+
}
|
|
3424
|
+
handle = opened;
|
|
3425
|
+
}
|
|
3426
|
+
}
|
|
3427
|
+
catch (error) {
|
|
3428
|
+
if (isNotFoundError(error)) {
|
|
3429
|
+
return undefined;
|
|
3430
|
+
}
|
|
3431
|
+
throw error;
|
|
3432
|
+
}
|
|
3433
|
+
try {
|
|
3434
|
+
const initial = await handle.stat({ bigint: true });
|
|
3435
|
+
if (typeof initial.size !== "bigint" || initial.size < 0n) {
|
|
3436
|
+
throw new Error("Node coordinate persistence returned an invalid bigint file size");
|
|
3437
|
+
}
|
|
3438
|
+
assertCoordinatePersistenceReadWithinLimit(validName, limit, initial.size);
|
|
3439
|
+
if (initial.size > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
3440
|
+
throw new RangeError("Node coordinate persistence file is too large to materialize safely");
|
|
3441
|
+
}
|
|
3442
|
+
const byteLength = Number(initial.size);
|
|
3443
|
+
const bytes = new Uint8Array(byteLength);
|
|
3444
|
+
let offset = 0;
|
|
3445
|
+
while (offset < byteLength) {
|
|
3446
|
+
const { bytesRead } = await handle.read(bytes, offset, byteLength - offset, offset);
|
|
3447
|
+
if (!Number.isSafeInteger(bytesRead) ||
|
|
3448
|
+
bytesRead <= 0 ||
|
|
3449
|
+
bytesRead > byteLength - offset) {
|
|
3450
|
+
throw new Error("Node coordinate persistence file changed during a bounded read");
|
|
3451
|
+
}
|
|
3452
|
+
offset += bytesRead;
|
|
3453
|
+
}
|
|
3454
|
+
const growthProbe = new Uint8Array(1);
|
|
3455
|
+
const { bytesRead: growthBytes } = await handle.read(growthProbe, 0, 1, byteLength);
|
|
3456
|
+
if (!Number.isSafeInteger(growthBytes) ||
|
|
3457
|
+
growthBytes < 0 ||
|
|
3458
|
+
growthBytes > 1) {
|
|
3459
|
+
throw new Error("Node coordinate persistence returned invalid bounded read progress");
|
|
3460
|
+
}
|
|
3461
|
+
const final = await handle.stat({ bigint: true });
|
|
3462
|
+
if (typeof final.size !== "bigint" || final.size < 0n) {
|
|
3463
|
+
throw new Error("Node coordinate persistence returned an invalid bigint file size");
|
|
3464
|
+
}
|
|
3465
|
+
if (growthBytes !== 0 || final.size > initial.size) {
|
|
3466
|
+
assertCoordinatePersistenceReadWithinLimit(validName, limit, final.size > initial.size ? final.size : initial.size + 1n);
|
|
3467
|
+
throw new Error("Node coordinate persistence file changed during a bounded read");
|
|
3468
|
+
}
|
|
3469
|
+
if (final.size !== initial.size) {
|
|
3470
|
+
throw new Error("Node coordinate persistence file changed during a bounded read");
|
|
3471
|
+
}
|
|
3472
|
+
return bytes;
|
|
3473
|
+
}
|
|
3474
|
+
finally {
|
|
3475
|
+
await handle.close();
|
|
3476
|
+
}
|
|
3477
|
+
}
|
|
3046
3478
|
async write(name, bytes) {
|
|
3047
3479
|
const fs = await this.ensureDirectory();
|
|
3048
3480
|
const path = await this.filePath(name);
|
|
3049
3481
|
await this.closeAppendHandle(path);
|
|
3050
3482
|
await fs.writeFile(path, bytes);
|
|
3051
3483
|
}
|
|
3484
|
+
async replaceAtomically(name, bytes) {
|
|
3485
|
+
const fs = await this.ensureDirectory();
|
|
3486
|
+
if (!fs.open || !fs.rename) {
|
|
3487
|
+
throw new Error("Node coordinate persistence does not expose atomic file replacement");
|
|
3488
|
+
}
|
|
3489
|
+
const validName = validateCoordinatePersistenceName(name);
|
|
3490
|
+
const temporaryName = validateCoordinatePersistenceName(`${validName}.tmp`);
|
|
3491
|
+
const path = await this.filePath(validName);
|
|
3492
|
+
const temporaryPath = await this.filePath(temporaryName);
|
|
3493
|
+
await this.closeAppendHandle(path);
|
|
3494
|
+
await this.closeAppendHandle(temporaryPath);
|
|
3495
|
+
await fs.rm(temporaryPath, { force: true });
|
|
3496
|
+
await fs.writeFile(temporaryPath, bytes);
|
|
3497
|
+
let temporaryHandle;
|
|
3498
|
+
try {
|
|
3499
|
+
temporaryHandle = await fs.open(temporaryPath, "r");
|
|
3500
|
+
if (typeof temporaryHandle.sync !== "function") {
|
|
3501
|
+
throw new Error("Node coordinate persistence atomic replacement does not expose FileHandle.sync");
|
|
3502
|
+
}
|
|
3503
|
+
await temporaryHandle.sync();
|
|
3504
|
+
}
|
|
3505
|
+
finally {
|
|
3506
|
+
await temporaryHandle?.close();
|
|
3507
|
+
}
|
|
3508
|
+
await fs.rename(temporaryPath, path);
|
|
3509
|
+
let directoryHandle;
|
|
3510
|
+
try {
|
|
3511
|
+
directoryHandle = await fs.open(this.directory, "r");
|
|
3512
|
+
if (typeof directoryHandle.sync !== "function") {
|
|
3513
|
+
throw new Error("Node coordinate persistence atomic replacement does not expose directory sync");
|
|
3514
|
+
}
|
|
3515
|
+
await directoryHandle.sync();
|
|
3516
|
+
}
|
|
3517
|
+
finally {
|
|
3518
|
+
await directoryHandle?.close();
|
|
3519
|
+
}
|
|
3520
|
+
}
|
|
3052
3521
|
async append(name, bytes) {
|
|
3053
3522
|
if (this.appendFailure !== undefined) {
|
|
3054
3523
|
throw this.appendFailure;
|
|
@@ -3214,6 +3683,31 @@ export class NativeBackboneOPFSCoordinatePersistenceStore {
|
|
|
3214
3683
|
throw error;
|
|
3215
3684
|
}
|
|
3216
3685
|
}
|
|
3686
|
+
async readLimited(name, maxBytes) {
|
|
3687
|
+
const validName = validateCoordinatePersistenceName(name);
|
|
3688
|
+
const limit = validateCoordinatePersistenceReadMaxBytes(maxBytes);
|
|
3689
|
+
try {
|
|
3690
|
+
const handle = await this.directory.getFileHandle(validName, {
|
|
3691
|
+
create: false,
|
|
3692
|
+
});
|
|
3693
|
+
const file = await handle.getFile();
|
|
3694
|
+
if (!Number.isSafeInteger(file.size) || file.size < 0) {
|
|
3695
|
+
throw new Error("OPFS coordinate persistence returned an invalid file size");
|
|
3696
|
+
}
|
|
3697
|
+
assertCoordinatePersistenceReadWithinLimit(validName, limit, file.size);
|
|
3698
|
+
const buffer = await file.arrayBuffer();
|
|
3699
|
+
if (buffer.byteLength !== file.size) {
|
|
3700
|
+
throw new Error("OPFS coordinate persistence file changed during a bounded read");
|
|
3701
|
+
}
|
|
3702
|
+
return new Uint8Array(buffer);
|
|
3703
|
+
}
|
|
3704
|
+
catch (error) {
|
|
3705
|
+
if (isNotFoundError(error)) {
|
|
3706
|
+
return undefined;
|
|
3707
|
+
}
|
|
3708
|
+
throw error;
|
|
3709
|
+
}
|
|
3710
|
+
}
|
|
3217
3711
|
async write(name, bytes) {
|
|
3218
3712
|
const handle = await this.directory.getFileHandle(validateCoordinatePersistenceName(name), { create: true });
|
|
3219
3713
|
const writable = await handle.createWritable();
|
|
@@ -3320,19 +3814,31 @@ export class NativeBackboneBufferedCoordinatePersistenceStore {
|
|
|
3320
3814
|
options;
|
|
3321
3815
|
buffers = new Map();
|
|
3322
3816
|
bufferedBytes = 0;
|
|
3817
|
+
readLimited;
|
|
3323
3818
|
supportsRemoval;
|
|
3324
3819
|
durableBarrier;
|
|
3820
|
+
atomicReplace;
|
|
3325
3821
|
constructor(inner, options = {}) {
|
|
3326
3822
|
this.inner = inner;
|
|
3327
3823
|
this.options = options;
|
|
3328
3824
|
this.supportsRemoval =
|
|
3329
3825
|
inner.supportsRemoval ?? typeof inner.remove === "function";
|
|
3826
|
+
if (typeof inner.readLimited === "function") {
|
|
3827
|
+
const innerReadLimited = inner.readLimited.bind(inner);
|
|
3828
|
+
this.readLimited = (name, maxBytes) => this.readWithinLimit(name, maxBytes, innerReadLimited);
|
|
3829
|
+
}
|
|
3330
3830
|
if (typeof inner.durableBarrier === "function") {
|
|
3331
3831
|
this.durableBarrier = async (name) => {
|
|
3332
3832
|
await this.flush(name);
|
|
3333
3833
|
await inner.durableBarrier(name);
|
|
3334
3834
|
};
|
|
3335
3835
|
}
|
|
3836
|
+
if (typeof inner.atomicReplace === "function") {
|
|
3837
|
+
this.atomicReplace = async (name, bytes) => {
|
|
3838
|
+
await this.flush(name);
|
|
3839
|
+
await inner.atomicReplace(name, bytes);
|
|
3840
|
+
};
|
|
3841
|
+
}
|
|
3336
3842
|
}
|
|
3337
3843
|
buffer(name) {
|
|
3338
3844
|
const validName = validateCoordinatePersistenceName(name);
|
|
@@ -3347,6 +3853,44 @@ export class NativeBackboneBufferedCoordinatePersistenceStore {
|
|
|
3347
3853
|
await this.flush(name);
|
|
3348
3854
|
return this.inner.read(name);
|
|
3349
3855
|
}
|
|
3856
|
+
async readWithinLimit(name, maxBytes, innerReadLimited) {
|
|
3857
|
+
const validName = validateCoordinatePersistenceName(name);
|
|
3858
|
+
const limit = validateCoordinatePersistenceReadMaxBytes(maxBytes);
|
|
3859
|
+
const pending = this.buffers.get(validName);
|
|
3860
|
+
if (!pending || pending.length === 0) {
|
|
3861
|
+
return innerReadLimited(validName, limit);
|
|
3862
|
+
}
|
|
3863
|
+
const pendingLength = pending.length;
|
|
3864
|
+
let pendingBytes = 0n;
|
|
3865
|
+
for (const chunk of pending) {
|
|
3866
|
+
pendingBytes += BigInt(chunk.byteLength);
|
|
3867
|
+
}
|
|
3868
|
+
assertCoordinatePersistenceReadWithinLimit(validName, limit, pendingBytes);
|
|
3869
|
+
const remaining = limit - Number(pendingBytes);
|
|
3870
|
+
let existingBytes;
|
|
3871
|
+
try {
|
|
3872
|
+
existingBytes = (await innerReadLimited(validName, remaining))
|
|
3873
|
+
?.byteLength;
|
|
3874
|
+
}
|
|
3875
|
+
catch (error) {
|
|
3876
|
+
if (error instanceof NativeBackboneCoordinatePersistenceReadLimitError) {
|
|
3877
|
+
throw new NativeBackboneCoordinatePersistenceReadLimitError(validName, limit, pendingBytes + error.observedBytes);
|
|
3878
|
+
}
|
|
3879
|
+
throw error;
|
|
3880
|
+
}
|
|
3881
|
+
let confirmedPendingBytes = 0n;
|
|
3882
|
+
for (const chunk of pending) {
|
|
3883
|
+
confirmedPendingBytes += BigInt(chunk.byteLength);
|
|
3884
|
+
}
|
|
3885
|
+
if (this.buffers.get(validName) !== pending ||
|
|
3886
|
+
pending.length !== pendingLength ||
|
|
3887
|
+
confirmedPendingBytes !== pendingBytes) {
|
|
3888
|
+
throw new Error("Native backbone coordinate persistence pending bytes changed during a bounded read");
|
|
3889
|
+
}
|
|
3890
|
+
assertCoordinatePersistenceReadWithinLimit(validName, limit, pendingBytes + BigInt(existingBytes ?? 0));
|
|
3891
|
+
await this.flush(validName);
|
|
3892
|
+
return innerReadLimited(validName, limit);
|
|
3893
|
+
}
|
|
3350
3894
|
async write(name, bytes) {
|
|
3351
3895
|
await this.flush(name);
|
|
3352
3896
|
await this.inner.write(name, bytes);
|
|
@@ -3411,7 +3955,7 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3411
3955
|
flushIntervalMs;
|
|
3412
3956
|
compactMaxJournalBytes;
|
|
3413
3957
|
compactMaxJournalRecords;
|
|
3414
|
-
crashSafeCompaction
|
|
3958
|
+
crashSafeCompaction;
|
|
3415
3959
|
durableBarrier;
|
|
3416
3960
|
supportsDrop;
|
|
3417
3961
|
dropIsTerminal = true;
|
|
@@ -3421,9 +3965,22 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3421
3965
|
documentJournalFile;
|
|
3422
3966
|
documentSignerSnapshotFile;
|
|
3423
3967
|
documentSignerJournalFile;
|
|
3968
|
+
checkpointConfigurationChecksum;
|
|
3969
|
+
checkpointStateFile;
|
|
3970
|
+
checkpointFiles;
|
|
3971
|
+
checkpointJournalFiles;
|
|
3424
3972
|
journalInitialized;
|
|
3973
|
+
journalByteLength = 0;
|
|
3974
|
+
journalRecordCount = 0;
|
|
3425
3975
|
documentJournalInitialized;
|
|
3976
|
+
documentJournalByteLength = 0;
|
|
3977
|
+
documentJournalRecordCount = 0;
|
|
3426
3978
|
documentSignerJournalInitialized;
|
|
3979
|
+
documentSignerJournalByteLength = 0;
|
|
3980
|
+
documentSignerJournalRecordCount = 0;
|
|
3981
|
+
checkpointHighwater = 0n;
|
|
3982
|
+
checkpointCompleted;
|
|
3983
|
+
checkpointPending;
|
|
3427
3984
|
lastFlushMs = Date.now();
|
|
3428
3985
|
persistenceQueue;
|
|
3429
3986
|
persistenceLifecycle = "active";
|
|
@@ -3449,24 +4006,61 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3449
4006
|
nativeBackboneCoordinatePersistenceFiles.documentSignerSnapshot);
|
|
3450
4007
|
this.documentSignerJournalFile = validateCoordinatePersistenceName(options.documentSignerJournal ??
|
|
3451
4008
|
nativeBackboneCoordinatePersistenceFiles.documentSignerJournal);
|
|
3452
|
-
|
|
3453
|
-
|
|
3454
|
-
|
|
4009
|
+
this.checkpointConfigurationChecksum = coordinateJournalChecksum(new TextEncoder().encode(JSON.stringify([
|
|
4010
|
+
this.snapshotFile,
|
|
4011
|
+
this.journalFile,
|
|
4012
|
+
this.documentSnapshotFile,
|
|
4013
|
+
this.documentJournalFile,
|
|
4014
|
+
this.documentSignerSnapshotFile,
|
|
4015
|
+
this.documentSignerJournalFile,
|
|
4016
|
+
])));
|
|
4017
|
+
this.checkpointStateFile = validateCoordinatePersistenceName(`${this.snapshotFile}.checkpoint-state`);
|
|
4018
|
+
this.checkpointFiles = {
|
|
4019
|
+
a: validateCoordinatePersistenceName(`${this.snapshotFile}.checkpoint-a`),
|
|
4020
|
+
b: validateCoordinatePersistenceName(`${this.snapshotFile}.checkpoint-b`),
|
|
4021
|
+
};
|
|
4022
|
+
this.checkpointJournalFiles = {
|
|
4023
|
+
a: {
|
|
4024
|
+
coordinate: validateCoordinatePersistenceName(`${this.journalFile}.checkpoint-a`),
|
|
4025
|
+
document: validateCoordinatePersistenceName(`${this.documentJournalFile}.checkpoint-a`),
|
|
4026
|
+
signer: validateCoordinatePersistenceName(`${this.documentSignerJournalFile}.checkpoint-a`),
|
|
4027
|
+
},
|
|
4028
|
+
b: {
|
|
4029
|
+
coordinate: validateCoordinatePersistenceName(`${this.journalFile}.checkpoint-b`),
|
|
4030
|
+
document: validateCoordinatePersistenceName(`${this.documentJournalFile}.checkpoint-b`),
|
|
4031
|
+
signer: validateCoordinatePersistenceName(`${this.documentSignerJournalFile}.checkpoint-b`),
|
|
4032
|
+
},
|
|
4033
|
+
};
|
|
4034
|
+
if (this.configuredFiles().includes(nativeBackboneCoordinateDropTombstoneFile) ||
|
|
4035
|
+
new Set(this.configuredFiles()).size !== this.configuredFiles().length) {
|
|
4036
|
+
throw new Error("Native backbone coordinate persistence files conflict with one another or with its drop tombstone");
|
|
4037
|
+
}
|
|
4038
|
+
this.crashSafeCompaction =
|
|
4039
|
+
typeof store.atomicReplace === "function" &&
|
|
4040
|
+
this.supportsDrop &&
|
|
4041
|
+
this.durableBarrier;
|
|
3455
4042
|
this.flushOnAppend = options.flushOnAppend ?? true;
|
|
3456
4043
|
this.flushMaxPendingBytes = resolveCoordinateFlushMaxPendingBytes(options);
|
|
3457
4044
|
if (options.flushIntervalMs != null) {
|
|
3458
4045
|
this.flushIntervalMs = Math.max(0, options.flushIntervalMs);
|
|
3459
4046
|
}
|
|
3460
|
-
if (options.compactMaxJournalBytes != null ||
|
|
3461
|
-
options.compactMaxJournalRecords != null)
|
|
4047
|
+
if ((options.compactMaxJournalBytes != null ||
|
|
4048
|
+
options.compactMaxJournalRecords != null) &&
|
|
4049
|
+
!this.crashSafeCompaction) {
|
|
3462
4050
|
throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
|
|
3463
4051
|
}
|
|
4052
|
+
if (options.compactMaxJournalBytes != null) {
|
|
4053
|
+
this.compactMaxJournalBytes = validateCoordinateCompactionThreshold(options.compactMaxJournalBytes, "compactMaxJournalBytes");
|
|
4054
|
+
}
|
|
4055
|
+
if (options.compactMaxJournalRecords != null) {
|
|
4056
|
+
this.compactMaxJournalRecords = validateCoordinateCompactionThreshold(options.compactMaxJournalRecords, "compactMaxJournalRecords");
|
|
4057
|
+
}
|
|
3464
4058
|
}
|
|
3465
4059
|
/** Durable operation-intent capability for strict shared-log transactions. */
|
|
3466
4060
|
get intentStore() {
|
|
3467
4061
|
return this.store;
|
|
3468
4062
|
}
|
|
3469
|
-
|
|
4063
|
+
legacyConfiguredFiles() {
|
|
3470
4064
|
return [
|
|
3471
4065
|
this.snapshotFile,
|
|
3472
4066
|
this.journalFile,
|
|
@@ -3476,26 +4070,79 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3476
4070
|
this.documentSignerJournalFile,
|
|
3477
4071
|
];
|
|
3478
4072
|
}
|
|
4073
|
+
configuredFiles() {
|
|
4074
|
+
return [
|
|
4075
|
+
...this.legacyConfiguredFiles(),
|
|
4076
|
+
this.checkpointStateFile,
|
|
4077
|
+
this.checkpointFiles.a,
|
|
4078
|
+
this.checkpointFiles.b,
|
|
4079
|
+
this.checkpointJournalFiles.a.coordinate,
|
|
4080
|
+
this.checkpointJournalFiles.a.document,
|
|
4081
|
+
this.checkpointJournalFiles.a.signer,
|
|
4082
|
+
this.checkpointJournalFiles.b.coordinate,
|
|
4083
|
+
this.checkpointJournalFiles.b.document,
|
|
4084
|
+
this.checkpointJournalFiles.b.signer,
|
|
4085
|
+
`${this.checkpointStateFile}.tmp`,
|
|
4086
|
+
`${this.checkpointFiles.a}.tmp`,
|
|
4087
|
+
`${this.checkpointFiles.b}.tmp`,
|
|
4088
|
+
`${this.journalFile}.tmp`,
|
|
4089
|
+
`${nativeBackboneCoordinateDropTombstoneFile}.tmp`,
|
|
4090
|
+
];
|
|
4091
|
+
}
|
|
4092
|
+
checkpointSlot(generation) {
|
|
4093
|
+
return generation % 2n === 1n ? "a" : "b";
|
|
4094
|
+
}
|
|
4095
|
+
activeJournalFiles() {
|
|
4096
|
+
const completed = this.checkpointCompleted;
|
|
4097
|
+
return completed
|
|
4098
|
+
? this.checkpointJournalFiles[this.checkpointSlot(completed.generation)]
|
|
4099
|
+
: {
|
|
4100
|
+
coordinate: this.journalFile,
|
|
4101
|
+
document: this.documentJournalFile,
|
|
4102
|
+
signer: this.documentSignerJournalFile,
|
|
4103
|
+
};
|
|
4104
|
+
}
|
|
3479
4105
|
assertLifecycleActive(operation) {
|
|
3480
4106
|
if (this.persistenceLifecycle !== "active") {
|
|
3481
4107
|
throw new Error(`Native backbone coordinate persistence can not ${operation} while ${this.persistenceLifecycle}`);
|
|
3482
4108
|
}
|
|
3483
4109
|
}
|
|
3484
4110
|
assertActive(operation) {
|
|
3485
|
-
|
|
3486
|
-
throw this.persistenceFailure;
|
|
3487
|
-
}
|
|
4111
|
+
this.assertPersistenceHealthy();
|
|
3488
4112
|
this.assertLifecycleActive(operation);
|
|
3489
4113
|
if (this.dropInitiatedOnGeneration) {
|
|
3490
4114
|
throw new Error(`Native backbone coordinate persistence can not ${operation} after drop was initiated; retry drop or resume drop first`);
|
|
3491
4115
|
}
|
|
3492
4116
|
}
|
|
4117
|
+
assertPersistenceHealthy() {
|
|
4118
|
+
if (this.persistenceFailure !== undefined) {
|
|
4119
|
+
throw this.persistenceFailure;
|
|
4120
|
+
}
|
|
4121
|
+
}
|
|
3493
4122
|
resetJournalTracking() {
|
|
3494
4123
|
this.journalInitialized = undefined;
|
|
4124
|
+
this.journalByteLength = 0;
|
|
4125
|
+
this.journalRecordCount = 0;
|
|
3495
4126
|
this.documentJournalInitialized = undefined;
|
|
4127
|
+
this.documentJournalByteLength = 0;
|
|
4128
|
+
this.documentJournalRecordCount = 0;
|
|
3496
4129
|
this.documentSignerJournalInitialized = undefined;
|
|
4130
|
+
this.documentSignerJournalByteLength = 0;
|
|
4131
|
+
this.documentSignerJournalRecordCount = 0;
|
|
4132
|
+
this.checkpointHighwater = 0n;
|
|
4133
|
+
this.checkpointCompleted = undefined;
|
|
4134
|
+
this.checkpointPending = undefined;
|
|
3497
4135
|
this.lastFlushMs = Date.now();
|
|
3498
4136
|
}
|
|
4137
|
+
async persistDropTombstone(files, preferAtomicReplace = false) {
|
|
4138
|
+
const bytes = nativeBackboneCoordinateDropTombstoneBytes(files);
|
|
4139
|
+
if (preferAtomicReplace && this.store.atomicReplace) {
|
|
4140
|
+
await this.store.atomicReplace(nativeBackboneCoordinateDropTombstoneFile, bytes);
|
|
4141
|
+
return;
|
|
4142
|
+
}
|
|
4143
|
+
await this.store.write(nativeBackboneCoordinateDropTombstoneFile, bytes);
|
|
4144
|
+
await this.barrierFile(nativeBackboneCoordinateDropTombstoneFile);
|
|
4145
|
+
}
|
|
3499
4146
|
async eraseDropFiles(files) {
|
|
3500
4147
|
if (!this.supportsDrop || !this.store.remove) {
|
|
3501
4148
|
throw new Error("Native backbone coordinate persistence store does not support removal");
|
|
@@ -3531,13 +4178,7 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3531
4178
|
this.dropInitiatedOnGeneration = true;
|
|
3532
4179
|
this.persistenceLifecycle = "dropping";
|
|
3533
4180
|
await this.enqueuePersistence(async () => {
|
|
3534
|
-
await this.
|
|
3535
|
-
if (this.store.durableBarrier) {
|
|
3536
|
-
await this.store.durableBarrier(nativeBackboneCoordinateDropTombstoneFile);
|
|
3537
|
-
}
|
|
3538
|
-
else {
|
|
3539
|
-
await this.store.flush?.(nativeBackboneCoordinateDropTombstoneFile);
|
|
3540
|
-
}
|
|
4181
|
+
await this.persistDropTombstone(files);
|
|
3541
4182
|
await this.eraseDropFiles(files);
|
|
3542
4183
|
this.resetJournalTracking();
|
|
3543
4184
|
this.persistenceLifecycle = "dropped";
|
|
@@ -3570,6 +4211,43 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3570
4211
|
return this.resumeDropInternal(completesInitiatedDrop ? "dropped" : "active");
|
|
3571
4212
|
});
|
|
3572
4213
|
}
|
|
4214
|
+
async barrierFile(name) {
|
|
4215
|
+
if (this.store.durableBarrier) {
|
|
4216
|
+
await this.store.durableBarrier(name);
|
|
4217
|
+
}
|
|
4218
|
+
else {
|
|
4219
|
+
await this.store.flush?.(name);
|
|
4220
|
+
}
|
|
4221
|
+
}
|
|
4222
|
+
async hasDurableLegacyMigrationSentinel() {
|
|
4223
|
+
const existing = await this.store.read(this.journalFile);
|
|
4224
|
+
if (existing &&
|
|
4225
|
+
existing.byteLength ===
|
|
4226
|
+
nativeBackboneCoordinateLegacyMigrationSentinel.byteLength &&
|
|
4227
|
+
nativeBackboneCoordinateLegacyMigrationSentinel.every((byte, index) => existing[index] === byte)) {
|
|
4228
|
+
// A previous process may have observed the bytes from page cache before
|
|
4229
|
+
// its durability barrier rejected. Cross a fresh barrier on every reopen
|
|
4230
|
+
// before any post-checkpoint WAL append can be admitted.
|
|
4231
|
+
await this.barrierFile(this.journalFile);
|
|
4232
|
+
return true;
|
|
4233
|
+
}
|
|
4234
|
+
return false;
|
|
4235
|
+
}
|
|
4236
|
+
async installLegacyMigrationSentinel() {
|
|
4237
|
+
if (await this.hasDurableLegacyMigrationSentinel()) {
|
|
4238
|
+
return;
|
|
4239
|
+
}
|
|
4240
|
+
if (!this.store.atomicReplace) {
|
|
4241
|
+
throw new Error("Native backbone completed checkpoint can not install its downgrade sentinel atomically");
|
|
4242
|
+
}
|
|
4243
|
+
await this.store.atomicReplace(this.journalFile, nativeBackboneCoordinateLegacyMigrationSentinel);
|
|
4244
|
+
}
|
|
4245
|
+
async requireLegacyMigrationSentinel() {
|
|
4246
|
+
if (await this.hasDurableLegacyMigrationSentinel()) {
|
|
4247
|
+
return;
|
|
4248
|
+
}
|
|
4249
|
+
throw new Error("Native backbone completed checkpoint downgrade sentinel is missing or has been replaced; refusing to overwrite possible legacy writes");
|
|
4250
|
+
}
|
|
3573
4251
|
async hydrate(backbone) {
|
|
3574
4252
|
this.assertActive("hydrate");
|
|
3575
4253
|
// Claim the lifecycle synchronously. close() queues after this complete
|
|
@@ -3580,24 +4258,147 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3580
4258
|
this.assertHydrating();
|
|
3581
4259
|
await this.resumeDropInternal("hydrating");
|
|
3582
4260
|
this.assertHydrating();
|
|
3583
|
-
const
|
|
3584
|
-
this.store.read(this.snapshotFile),
|
|
3585
|
-
this.store.read(this.journalFile),
|
|
3586
|
-
this.store.read(this.documentSnapshotFile),
|
|
3587
|
-
this.store.read(this.documentJournalFile),
|
|
3588
|
-
this.store.read(this.documentSignerSnapshotFile),
|
|
3589
|
-
this.store.read(this.documentSignerJournalFile),
|
|
3590
|
-
]);
|
|
4261
|
+
const checkpointStateBytes = await this.store.read(this.checkpointStateFile);
|
|
3591
4262
|
this.assertHydrating();
|
|
4263
|
+
let checkpointState;
|
|
4264
|
+
if (checkpointStateBytes) {
|
|
4265
|
+
try {
|
|
4266
|
+
checkpointState = parseNativeBackboneCoordinateCheckpointState(checkpointStateBytes, this.checkpointStateFile);
|
|
4267
|
+
if (checkpointState.configurationChecksum !==
|
|
4268
|
+
this.checkpointConfigurationChecksum) {
|
|
4269
|
+
throw new Error("Native backbone checkpoint authority does not match the configured persistence files");
|
|
4270
|
+
}
|
|
4271
|
+
}
|
|
4272
|
+
catch (error) {
|
|
4273
|
+
this.persistenceFailure ??= error;
|
|
4274
|
+
throw this.persistenceFailure;
|
|
4275
|
+
}
|
|
4276
|
+
}
|
|
4277
|
+
this.checkpointHighwater = checkpointState?.highwater ?? 0n;
|
|
4278
|
+
this.checkpointPending = checkpointState?.pending;
|
|
4279
|
+
this.checkpointCompleted = checkpointState?.completed;
|
|
4280
|
+
let checkpointAuthority = this.checkpointCompleted;
|
|
4281
|
+
let promotePendingCheckpoint = false;
|
|
4282
|
+
if (!checkpointAuthority && this.checkpointPending) {
|
|
4283
|
+
try {
|
|
4284
|
+
// On the first generation only, the sentinel is the durable stage
|
|
4285
|
+
// boundary: publication writes it after the pending checkpoint and its
|
|
4286
|
+
// WAL headers, but before completed authority. A pre-checkpoint reader
|
|
4287
|
+
// can no longer consume legacy state, so recovery may validate and
|
|
4288
|
+
// promote the staged generation. Once a completed generation exists the
|
|
4289
|
+
// sentinel predates later pending work and is not a promotion signal.
|
|
4290
|
+
if (await this.hasDurableLegacyMigrationSentinel()) {
|
|
4291
|
+
checkpointAuthority = this.checkpointPending;
|
|
4292
|
+
promotePendingCheckpoint = true;
|
|
4293
|
+
}
|
|
4294
|
+
}
|
|
4295
|
+
catch (error) {
|
|
4296
|
+
this.persistenceFailure ??= error;
|
|
4297
|
+
throw this.persistenceFailure;
|
|
4298
|
+
}
|
|
4299
|
+
}
|
|
4300
|
+
let snapshot;
|
|
4301
|
+
let journal;
|
|
4302
|
+
let documentSnapshot;
|
|
4303
|
+
let documentJournal;
|
|
4304
|
+
let documentSignerSnapshot;
|
|
4305
|
+
let documentSignerJournal;
|
|
4306
|
+
let journalNames = {
|
|
4307
|
+
coordinate: this.journalFile,
|
|
4308
|
+
document: this.documentJournalFile,
|
|
4309
|
+
signer: this.documentSignerJournalFile,
|
|
4310
|
+
};
|
|
4311
|
+
if (checkpointAuthority) {
|
|
4312
|
+
const authority = checkpointAuthority;
|
|
4313
|
+
const authorityKind = promotePendingCheckpoint
|
|
4314
|
+
? "promotable pending"
|
|
4315
|
+
: "completed";
|
|
4316
|
+
const slot = this.checkpointSlot(authority.generation);
|
|
4317
|
+
const checkpointFile = this.checkpointFiles[slot];
|
|
4318
|
+
journalNames = this.checkpointJournalFiles[slot];
|
|
4319
|
+
const [checkpointBytes, coordinateJournal, valueJournal, signerJournal,] = await Promise.all([
|
|
4320
|
+
this.store.read(checkpointFile),
|
|
4321
|
+
this.store.read(journalNames.coordinate),
|
|
4322
|
+
this.store.read(journalNames.document),
|
|
4323
|
+
this.store.read(journalNames.signer),
|
|
4324
|
+
]);
|
|
4325
|
+
this.assertHydrating();
|
|
4326
|
+
try {
|
|
4327
|
+
if (!checkpointBytes ||
|
|
4328
|
+
checkpointBytes.byteLength !== authority.byteLength ||
|
|
4329
|
+
coordinateJournalChecksum(checkpointBytes) !== authority.checksum) {
|
|
4330
|
+
throw new Error(`Native backbone ${authorityKind} checkpoint authority is missing or corrupt`);
|
|
4331
|
+
}
|
|
4332
|
+
if (!coordinateJournal || !valueJournal || !signerJournal) {
|
|
4333
|
+
throw new Error(`Native backbone ${authorityKind} checkpoint WAL generation is incomplete`);
|
|
4334
|
+
}
|
|
4335
|
+
const checkpoint = parseNativeBackboneCoordinateCheckpoint(checkpointBytes, checkpointFile);
|
|
4336
|
+
if (checkpoint.generation !== authority.generation ||
|
|
4337
|
+
checkpoint.configurationChecksum !==
|
|
4338
|
+
this.checkpointConfigurationChecksum) {
|
|
4339
|
+
throw new Error(`Native backbone checkpoint generation or configuration does not match its ${authorityKind} authority`);
|
|
4340
|
+
}
|
|
4341
|
+
snapshot = checkpoint.coordinateSnapshot;
|
|
4342
|
+
documentSnapshot = checkpoint.documentSnapshot;
|
|
4343
|
+
documentSignerSnapshot = checkpoint.documentSignerSnapshot;
|
|
4344
|
+
journal = coordinateJournal;
|
|
4345
|
+
documentJournal = valueJournal;
|
|
4346
|
+
documentSignerJournal = signerJournal;
|
|
4347
|
+
}
|
|
4348
|
+
catch (error) {
|
|
4349
|
+
this.persistenceFailure ??= error;
|
|
4350
|
+
throw this.persistenceFailure;
|
|
4351
|
+
}
|
|
4352
|
+
}
|
|
4353
|
+
else {
|
|
4354
|
+
[
|
|
4355
|
+
snapshot,
|
|
4356
|
+
journal,
|
|
4357
|
+
documentSnapshot,
|
|
4358
|
+
documentJournal,
|
|
4359
|
+
documentSignerSnapshot,
|
|
4360
|
+
documentSignerJournal,
|
|
4361
|
+
] = await Promise.all([
|
|
4362
|
+
this.store.read(this.snapshotFile),
|
|
4363
|
+
this.store.read(this.journalFile),
|
|
4364
|
+
this.store.read(this.documentSnapshotFile),
|
|
4365
|
+
this.store.read(this.documentJournalFile),
|
|
4366
|
+
this.store.read(this.documentSignerSnapshotFile),
|
|
4367
|
+
this.store.read(this.documentSignerJournalFile),
|
|
4368
|
+
]);
|
|
4369
|
+
this.assertHydrating();
|
|
4370
|
+
}
|
|
3592
4371
|
try {
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
4372
|
+
if (checkpointAuthority &&
|
|
4373
|
+
(!journal ||
|
|
4374
|
+
!hasCoordinateJournalMagic(journal) ||
|
|
4375
|
+
!documentJournal ||
|
|
4376
|
+
!hasCoordinateJournalMagic(documentJournal) ||
|
|
4377
|
+
!documentSignerJournal ||
|
|
4378
|
+
!hasCoordinateJournalMagic(documentSignerJournal))) {
|
|
4379
|
+
throw new Error(`Native backbone ${promotePendingCheckpoint ? "promotable pending" : "completed"} checkpoint WAL generation has a missing or invalid header`);
|
|
4380
|
+
}
|
|
4381
|
+
validateCoordinateJournal(journal, journalNames.coordinate);
|
|
4382
|
+
validateCoordinateJournal(documentJournal, journalNames.document);
|
|
4383
|
+
validateCoordinateJournal(documentSignerJournal, journalNames.signer);
|
|
3596
4384
|
}
|
|
3597
4385
|
catch (error) {
|
|
3598
4386
|
this.persistenceFailure ??= error;
|
|
3599
4387
|
throw this.persistenceFailure;
|
|
3600
4388
|
}
|
|
4389
|
+
if (checkpointAuthority && !promotePendingCheckpoint) {
|
|
4390
|
+
try {
|
|
4391
|
+
// Completed authority is never permission to overwrite an ordinary
|
|
4392
|
+
// legacy WAL: it may contain writes made by a downgraded process in the
|
|
4393
|
+
// publication cut. Require the exact durable sentinel before loading.
|
|
4394
|
+
await this.requireLegacyMigrationSentinel();
|
|
4395
|
+
this.assertHydrating();
|
|
4396
|
+
}
|
|
4397
|
+
catch (error) {
|
|
4398
|
+
this.persistenceFailure ??= error;
|
|
4399
|
+
throw this.persistenceFailure;
|
|
4400
|
+
}
|
|
4401
|
+
}
|
|
3601
4402
|
let operations;
|
|
3602
4403
|
let documentOperations;
|
|
3603
4404
|
let documentSignerOperations;
|
|
@@ -3619,12 +4420,49 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3619
4420
|
}
|
|
3620
4421
|
throw error;
|
|
3621
4422
|
}
|
|
4423
|
+
if (promotePendingCheckpoint) {
|
|
4424
|
+
try {
|
|
4425
|
+
const authority = checkpointAuthority;
|
|
4426
|
+
if (!this.store.atomicReplace) {
|
|
4427
|
+
throw new Error("Native backbone pending checkpoint can not publish completed authority atomically");
|
|
4428
|
+
}
|
|
4429
|
+
await this.store.atomicReplace(this.checkpointStateFile, encodeNativeBackboneCoordinateCheckpointState({
|
|
4430
|
+
configurationChecksum: this.checkpointConfigurationChecksum,
|
|
4431
|
+
highwater: authority.generation,
|
|
4432
|
+
completed: authority,
|
|
4433
|
+
}));
|
|
4434
|
+
this.checkpointCompleted = authority;
|
|
4435
|
+
this.checkpointPending = undefined;
|
|
4436
|
+
this.assertHydrating();
|
|
4437
|
+
}
|
|
4438
|
+
catch (error) {
|
|
4439
|
+
this.persistenceFailure ??= error;
|
|
4440
|
+
throw this.persistenceFailure;
|
|
4441
|
+
}
|
|
4442
|
+
}
|
|
4443
|
+
if (this.checkpointCompleted || this.checkpointPending) {
|
|
4444
|
+
try {
|
|
4445
|
+
await this.cleanupRetiredCheckpointFiles();
|
|
4446
|
+
this.assertHydrating();
|
|
4447
|
+
}
|
|
4448
|
+
catch (error) {
|
|
4449
|
+
this.persistenceFailure ??= error;
|
|
4450
|
+
throw this.persistenceFailure;
|
|
4451
|
+
}
|
|
4452
|
+
}
|
|
3622
4453
|
this.assertHydrating();
|
|
3623
4454
|
this.journalInitialized = !!journal && journal.byteLength > 0;
|
|
4455
|
+
this.journalByteLength = journal?.byteLength ?? 0;
|
|
4456
|
+
this.journalRecordCount = operations;
|
|
3624
4457
|
this.documentJournalInitialized =
|
|
3625
4458
|
!!documentJournal && documentJournal.byteLength > 0;
|
|
4459
|
+
this.documentJournalByteLength = documentJournal?.byteLength ?? 0;
|
|
4460
|
+
this.documentJournalRecordCount = documentOperations;
|
|
3626
4461
|
this.documentSignerJournalInitialized =
|
|
3627
4462
|
!!documentSignerJournal && documentSignerJournal.byteLength > 0;
|
|
4463
|
+
this.documentSignerJournalByteLength =
|
|
4464
|
+
documentSignerJournal?.byteLength ?? 0;
|
|
4465
|
+
this.documentSignerJournalRecordCount = documentSignerOperations;
|
|
3628
4466
|
backbone.setCoordinateJournalEnabled(true);
|
|
3629
4467
|
backbone.setDocumentJournalEnabled(true);
|
|
3630
4468
|
backbone.setDocumentSignerJournalEnabled(true);
|
|
@@ -3660,13 +4498,17 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3660
4498
|
return false;
|
|
3661
4499
|
}
|
|
3662
4500
|
let tombstone;
|
|
4501
|
+
let expandedFiles;
|
|
3663
4502
|
try {
|
|
3664
4503
|
tombstone = parseNativeBackboneCoordinateDropTombstone(bytes);
|
|
3665
|
-
for (const file of this.
|
|
4504
|
+
for (const file of this.legacyConfiguredFiles()) {
|
|
3666
4505
|
if (!tombstone.files.includes(file)) {
|
|
3667
4506
|
throw new Error("Native backbone drop tombstone does not cover the configured namespace");
|
|
3668
4507
|
}
|
|
3669
4508
|
}
|
|
4509
|
+
expandedFiles = [
|
|
4510
|
+
...new Set([...tombstone.files, ...this.configuredFiles()]),
|
|
4511
|
+
];
|
|
3670
4512
|
}
|
|
3671
4513
|
catch (error) {
|
|
3672
4514
|
// Corruption must never hydrate stale files, but an explicit drop must
|
|
@@ -3675,6 +4517,17 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3675
4517
|
this.persistenceLifecycle = this.closePromise ? "closing" : "active";
|
|
3676
4518
|
throw this.persistenceFailure;
|
|
3677
4519
|
}
|
|
4520
|
+
if (expandedFiles.length !== tombstone.files.length) {
|
|
4521
|
+
// Older valid markers only knew the six snapshot/WAL files. Widen the
|
|
4522
|
+
// destructive intent durably before erasing any checkpoint generation,
|
|
4523
|
+
// while preserving additional namespace files recorded by that version.
|
|
4524
|
+
await this.persistDropTombstone(expandedFiles, true);
|
|
4525
|
+
tombstone = { ...tombstone, files: expandedFiles };
|
|
4526
|
+
}
|
|
4527
|
+
// A prior replacement attempt may have made the expanded bytes visible before
|
|
4528
|
+
// its durability barrier rejected. Cross a fresh barrier on every recovery
|
|
4529
|
+
// before allowing any target removal.
|
|
4530
|
+
await this.barrierFile(nativeBackboneCoordinateDropTombstoneFile);
|
|
3678
4531
|
await this.eraseDropFiles(tombstone.files);
|
|
3679
4532
|
this.resetJournalTracking();
|
|
3680
4533
|
this.persistenceLifecycle =
|
|
@@ -3718,7 +4571,13 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3718
4571
|
this.assertActive("flush");
|
|
3719
4572
|
// Serialized with compact() so a flush never clears records appended to
|
|
3720
4573
|
// the wasm journal while a previous flush was awaiting its disk write.
|
|
3721
|
-
return this.enqueuePersistence(() =>
|
|
4574
|
+
return this.enqueuePersistence(() => {
|
|
4575
|
+
// A call can be admitted while an earlier queued write is still in flight.
|
|
4576
|
+
// Re-check the sticky failure at execution time so no suffix can ACK after
|
|
4577
|
+
// that earlier write or authority switch failed.
|
|
4578
|
+
this.assertPersistenceHealthy();
|
|
4579
|
+
return this.flushJournalInternal(backbone);
|
|
4580
|
+
});
|
|
3722
4581
|
}
|
|
3723
4582
|
enqueuePersistence(fn) {
|
|
3724
4583
|
// Runs `fn` immediately when no other persistence operation is in
|
|
@@ -3734,7 +4593,151 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3734
4593
|
});
|
|
3735
4594
|
return next;
|
|
3736
4595
|
}
|
|
3737
|
-
|
|
4596
|
+
shouldCompactJournal(additionalBytes, additionalRecords) {
|
|
4597
|
+
return ((this.compactMaxJournalBytes != null &&
|
|
4598
|
+
this.journalByteLength +
|
|
4599
|
+
this.documentJournalByteLength +
|
|
4600
|
+
this.documentSignerJournalByteLength +
|
|
4601
|
+
additionalBytes >=
|
|
4602
|
+
this.compactMaxJournalBytes) ||
|
|
4603
|
+
(this.compactMaxJournalRecords != null &&
|
|
4604
|
+
this.journalRecordCount +
|
|
4605
|
+
this.documentJournalRecordCount +
|
|
4606
|
+
this.documentSignerJournalRecordCount +
|
|
4607
|
+
additionalRecords >=
|
|
4608
|
+
this.compactMaxJournalRecords));
|
|
4609
|
+
}
|
|
4610
|
+
nextCheckpointGeneration() {
|
|
4611
|
+
let generation = this.checkpointHighwater + 1n;
|
|
4612
|
+
const active = this.checkpointCompleted;
|
|
4613
|
+
if (active &&
|
|
4614
|
+
this.checkpointSlot(generation) === this.checkpointSlot(active.generation)) {
|
|
4615
|
+
generation += 1n;
|
|
4616
|
+
}
|
|
4617
|
+
if (generation > nativeBackboneCoordinateMaxU64) {
|
|
4618
|
+
throw new RangeError("Native backbone checkpoint generation highwater is exhausted");
|
|
4619
|
+
}
|
|
4620
|
+
return generation;
|
|
4621
|
+
}
|
|
4622
|
+
async publishCheckpoint(checkpoint) {
|
|
4623
|
+
if (!this.crashSafeCompaction ||
|
|
4624
|
+
!this.store.atomicReplace ||
|
|
4625
|
+
!this.store.remove) {
|
|
4626
|
+
throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
|
|
4627
|
+
}
|
|
4628
|
+
const generation = this.nextCheckpointGeneration();
|
|
4629
|
+
const slot = this.checkpointSlot(generation);
|
|
4630
|
+
const checkpointBytes = encodeNativeBackboneCoordinateCheckpoint({
|
|
4631
|
+
configurationChecksum: this.checkpointConfigurationChecksum,
|
|
4632
|
+
generation,
|
|
4633
|
+
...checkpoint,
|
|
4634
|
+
});
|
|
4635
|
+
const authority = {
|
|
4636
|
+
generation,
|
|
4637
|
+
byteLength: checkpointBytes.byteLength,
|
|
4638
|
+
checksum: coordinateJournalChecksum(checkpointBytes),
|
|
4639
|
+
};
|
|
4640
|
+
const pendingState = {
|
|
4641
|
+
configurationChecksum: this.checkpointConfigurationChecksum,
|
|
4642
|
+
highwater: generation,
|
|
4643
|
+
pending: authority,
|
|
4644
|
+
completed: this.checkpointCompleted,
|
|
4645
|
+
};
|
|
4646
|
+
await this.store.atomicReplace(this.checkpointStateFile, encodeNativeBackboneCoordinateCheckpointState(pendingState));
|
|
4647
|
+
this.checkpointHighwater = generation;
|
|
4648
|
+
this.checkpointPending = authority;
|
|
4649
|
+
const targetJournals = this.checkpointJournalFiles[slot];
|
|
4650
|
+
for (const file of [
|
|
4651
|
+
targetJournals.coordinate,
|
|
4652
|
+
targetJournals.document,
|
|
4653
|
+
targetJournals.signer,
|
|
4654
|
+
]) {
|
|
4655
|
+
await this.store.remove(file);
|
|
4656
|
+
}
|
|
4657
|
+
const journalHeaders = [
|
|
4658
|
+
[targetJournals.coordinate, nativeBackboneCoordinateJournalMagic],
|
|
4659
|
+
[targetJournals.document, nativeBackboneCoordinateJournalMagic],
|
|
4660
|
+
[targetJournals.signer, nativeBackboneCoordinateJournalMagic],
|
|
4661
|
+
];
|
|
4662
|
+
for (const [file, header] of journalHeaders) {
|
|
4663
|
+
await this.store.write(file, header);
|
|
4664
|
+
await this.barrierFile(file);
|
|
4665
|
+
}
|
|
4666
|
+
await this.store.atomicReplace(this.checkpointFiles[slot], checkpointBytes);
|
|
4667
|
+
// A prior release does not understand checkpoint authority. The sentinel is
|
|
4668
|
+
// the first-generation publication boundary: install and durably revalidate
|
|
4669
|
+
// it before completed authority becomes visible. On later generations it
|
|
4670
|
+
// must already be exact; never overwrite possible writes from a downgrade.
|
|
4671
|
+
if (this.checkpointCompleted) {
|
|
4672
|
+
await this.requireLegacyMigrationSentinel();
|
|
4673
|
+
}
|
|
4674
|
+
else {
|
|
4675
|
+
await this.installLegacyMigrationSentinel();
|
|
4676
|
+
}
|
|
4677
|
+
const completedState = {
|
|
4678
|
+
configurationChecksum: this.checkpointConfigurationChecksum,
|
|
4679
|
+
highwater: generation,
|
|
4680
|
+
completed: authority,
|
|
4681
|
+
};
|
|
4682
|
+
await this.store.atomicReplace(this.checkpointStateFile, encodeNativeBackboneCoordinateCheckpointState(completedState));
|
|
4683
|
+
// The completed authority is now the only writable generation. Update all
|
|
4684
|
+
// in-memory routing synchronously before another queued flush can start.
|
|
4685
|
+
this.checkpointCompleted = authority;
|
|
4686
|
+
this.checkpointPending = undefined;
|
|
4687
|
+
this.journalInitialized = true;
|
|
4688
|
+
this.documentJournalInitialized = true;
|
|
4689
|
+
this.documentSignerJournalInitialized = true;
|
|
4690
|
+
this.journalByteLength = nativeBackboneCoordinateJournalMagic.byteLength;
|
|
4691
|
+
this.documentJournalByteLength =
|
|
4692
|
+
nativeBackboneCoordinateJournalMagic.byteLength;
|
|
4693
|
+
this.documentSignerJournalByteLength =
|
|
4694
|
+
nativeBackboneCoordinateJournalMagic.byteLength;
|
|
4695
|
+
this.journalRecordCount = 0;
|
|
4696
|
+
this.documentJournalRecordCount = 0;
|
|
4697
|
+
this.documentSignerJournalRecordCount = 0;
|
|
4698
|
+
}
|
|
4699
|
+
async cleanupRetiredCheckpointFiles() {
|
|
4700
|
+
if (!this.store.remove ||
|
|
4701
|
+
(!this.checkpointCompleted && !this.checkpointPending)) {
|
|
4702
|
+
return;
|
|
4703
|
+
}
|
|
4704
|
+
const retiredSlot = this
|
|
4705
|
+
.checkpointCompleted
|
|
4706
|
+
? this.checkpointSlot(this.checkpointCompleted.generation) === "a"
|
|
4707
|
+
? "b"
|
|
4708
|
+
: "a"
|
|
4709
|
+
: this.checkpointSlot(this.checkpointPending.generation);
|
|
4710
|
+
const retiredJournals = this.checkpointJournalFiles[retiredSlot];
|
|
4711
|
+
const files = [
|
|
4712
|
+
...(this.checkpointCompleted
|
|
4713
|
+
? [
|
|
4714
|
+
this.snapshotFile,
|
|
4715
|
+
this.documentSnapshotFile,
|
|
4716
|
+
this.documentJournalFile,
|
|
4717
|
+
this.documentSignerSnapshotFile,
|
|
4718
|
+
this.documentSignerJournalFile,
|
|
4719
|
+
]
|
|
4720
|
+
: []),
|
|
4721
|
+
this.checkpointFiles[retiredSlot],
|
|
4722
|
+
`${this.checkpointFiles[retiredSlot]}.tmp`,
|
|
4723
|
+
retiredJournals.coordinate,
|
|
4724
|
+
retiredJournals.document,
|
|
4725
|
+
retiredJournals.signer,
|
|
4726
|
+
];
|
|
4727
|
+
const removals = await Promise.allSettled(files.map((file) => this.store.remove(file)));
|
|
4728
|
+
if (removals.every((result) => result.status === "fulfilled")) {
|
|
4729
|
+
try {
|
|
4730
|
+
await this.store.durableBarrier?.();
|
|
4731
|
+
}
|
|
4732
|
+
catch {
|
|
4733
|
+
// Cleanup is retryable debt. Publication and the caller's ACK already
|
|
4734
|
+
// belong to the new generation, so never turn this into an ambiguous
|
|
4735
|
+
// append failure after the authority switch.
|
|
4736
|
+
}
|
|
4737
|
+
}
|
|
4738
|
+
}
|
|
4739
|
+
async flushJournalInternal(backbone, forceCheckpoint = false) {
|
|
4740
|
+
this.assertPersistenceHealthy();
|
|
3738
4741
|
let written = 0;
|
|
3739
4742
|
let persistenceMutationStarted = false;
|
|
3740
4743
|
let coordinateBytes;
|
|
@@ -3746,10 +4749,39 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3746
4749
|
const documentRecordCount = backbone.documentPendingJournalLength;
|
|
3747
4750
|
const signerRecords = backbone.documentSignerJournal();
|
|
3748
4751
|
const signerRecordCount = backbone.documentSignerPendingJournalLength;
|
|
4752
|
+
const additionalBytes = coordinateRecords.byteLength +
|
|
4753
|
+
documentRecords.byteLength +
|
|
4754
|
+
signerRecords.byteLength +
|
|
4755
|
+
(coordinateRecords.byteLength > 0 && this.journalInitialized !== true
|
|
4756
|
+
? backbone.coordinateJournalHeader().byteLength
|
|
4757
|
+
: 0) +
|
|
4758
|
+
(documentRecords.byteLength > 0 &&
|
|
4759
|
+
this.documentJournalInitialized !== true
|
|
4760
|
+
? backbone.documentJournalHeader().byteLength
|
|
4761
|
+
: 0) +
|
|
4762
|
+
(signerRecords.byteLength > 0 &&
|
|
4763
|
+
this.documentSignerJournalInitialized !== true
|
|
4764
|
+
? backbone.documentSignerJournalHeader().byteLength
|
|
4765
|
+
: 0);
|
|
4766
|
+
const additionalRecords = coordinateRecordCount + documentRecordCount + signerRecordCount;
|
|
4767
|
+
const checkpointRequested = forceCheckpoint ||
|
|
4768
|
+
(this.crashSafeCompaction &&
|
|
4769
|
+
this.shouldCompactJournal(additionalBytes, additionalRecords));
|
|
4770
|
+
// These three synchronous copies and the journal prefixes above are one
|
|
4771
|
+
// logical cut. No mutation can slip into the checkpoint without also being
|
|
4772
|
+
// represented by one of the captured prefixes.
|
|
4773
|
+
const checkpoint = checkpointRequested
|
|
4774
|
+
? {
|
|
4775
|
+
coordinateSnapshot: backbone.coordinateSnapshot(),
|
|
4776
|
+
documentSnapshot: backbone.documentSnapshot(),
|
|
4777
|
+
documentSignerSnapshot: backbone.documentSignerSnapshot(),
|
|
4778
|
+
}
|
|
4779
|
+
: undefined;
|
|
3749
4780
|
try {
|
|
4781
|
+
const activeJournals = this.activeJournalFiles();
|
|
3750
4782
|
if (coordinateRecords.byteLength > 0) {
|
|
3751
4783
|
if (this.journalInitialized === undefined) {
|
|
3752
|
-
const existing = await this.store.read(
|
|
4784
|
+
const existing = await this.store.read(activeJournals.coordinate);
|
|
3753
4785
|
this.journalInitialized = !!existing && existing.byteLength > 0;
|
|
3754
4786
|
}
|
|
3755
4787
|
coordinateBytes = this.journalInitialized
|
|
@@ -3758,26 +4790,26 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3758
4790
|
backbone.coordinateJournalHeader(),
|
|
3759
4791
|
coordinateRecords,
|
|
3760
4792
|
]);
|
|
3761
|
-
await this.store.append(
|
|
4793
|
+
await this.store.append(activeJournals.coordinate, coordinateBytes);
|
|
3762
4794
|
persistenceMutationStarted = true;
|
|
3763
4795
|
written += coordinateRecords.byteLength;
|
|
3764
4796
|
}
|
|
3765
4797
|
if (documentRecords.byteLength > 0) {
|
|
3766
4798
|
if (this.documentJournalInitialized === undefined) {
|
|
3767
|
-
const existing = await this.store.read(
|
|
4799
|
+
const existing = await this.store.read(activeJournals.document);
|
|
3768
4800
|
this.documentJournalInitialized =
|
|
3769
4801
|
!!existing && existing.byteLength > 0;
|
|
3770
4802
|
}
|
|
3771
4803
|
documentBytes = this.documentJournalInitialized
|
|
3772
4804
|
? documentRecords
|
|
3773
4805
|
: concatBytes([backbone.documentJournalHeader(), documentRecords]);
|
|
3774
|
-
await this.store.append(
|
|
4806
|
+
await this.store.append(activeJournals.document, documentBytes);
|
|
3775
4807
|
persistenceMutationStarted = true;
|
|
3776
4808
|
written += documentRecords.byteLength;
|
|
3777
4809
|
}
|
|
3778
4810
|
if (signerRecords.byteLength > 0) {
|
|
3779
4811
|
if (this.documentSignerJournalInitialized === undefined) {
|
|
3780
|
-
const existing = await this.store.read(
|
|
4812
|
+
const existing = await this.store.read(activeJournals.signer);
|
|
3781
4813
|
this.documentSignerJournalInitialized =
|
|
3782
4814
|
!!existing && existing.byteLength > 0;
|
|
3783
4815
|
}
|
|
@@ -3787,42 +4819,50 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3787
4819
|
backbone.documentSignerJournalHeader(),
|
|
3788
4820
|
signerRecords,
|
|
3789
4821
|
]);
|
|
3790
|
-
await this.store.append(
|
|
4822
|
+
await this.store.append(activeJournals.signer, signerBytes);
|
|
3791
4823
|
persistenceMutationStarted = true;
|
|
3792
4824
|
written += signerRecords.byteLength;
|
|
3793
4825
|
}
|
|
3794
|
-
if (written === 0) {
|
|
4826
|
+
if (written === 0 && !checkpoint) {
|
|
3795
4827
|
this.lastFlushMs = Date.now();
|
|
3796
4828
|
return 0;
|
|
3797
4829
|
}
|
|
3798
4830
|
// `append` may only enqueue bytes in a buffered store. Drain and fsync
|
|
3799
4831
|
// every affected WAL before clearing its wasm prefix or returning an ACK.
|
|
3800
4832
|
for (const file of [
|
|
3801
|
-
coordinateBytes ?
|
|
3802
|
-
documentBytes ?
|
|
3803
|
-
signerBytes ?
|
|
4833
|
+
coordinateBytes ? activeJournals.coordinate : undefined,
|
|
4834
|
+
documentBytes ? activeJournals.document : undefined,
|
|
4835
|
+
signerBytes ? activeJournals.signer : undefined,
|
|
3804
4836
|
]) {
|
|
3805
4837
|
if (file) {
|
|
3806
|
-
|
|
3807
|
-
await this.store.durableBarrier(file);
|
|
3808
|
-
}
|
|
3809
|
-
else {
|
|
3810
|
-
await this.store.flush?.(file);
|
|
3811
|
-
}
|
|
4838
|
+
await this.barrierFile(file);
|
|
3812
4839
|
}
|
|
3813
4840
|
}
|
|
3814
4841
|
if (coordinateBytes) {
|
|
3815
4842
|
this.journalInitialized = true;
|
|
4843
|
+
this.journalByteLength += coordinateBytes.byteLength;
|
|
4844
|
+
this.journalRecordCount += coordinateRecordCount;
|
|
3816
4845
|
}
|
|
3817
4846
|
if (documentBytes) {
|
|
3818
4847
|
this.documentJournalInitialized = true;
|
|
4848
|
+
this.documentJournalByteLength += documentBytes.byteLength;
|
|
4849
|
+
this.documentJournalRecordCount += documentRecordCount;
|
|
3819
4850
|
}
|
|
3820
4851
|
if (signerBytes) {
|
|
3821
4852
|
this.documentSignerJournalInitialized = true;
|
|
4853
|
+
this.documentSignerJournalByteLength += signerBytes.byteLength;
|
|
4854
|
+
this.documentSignerJournalRecordCount += signerRecordCount;
|
|
4855
|
+
}
|
|
4856
|
+
if (checkpoint) {
|
|
4857
|
+
persistenceMutationStarted = true;
|
|
4858
|
+
await this.publishCheckpoint(checkpoint);
|
|
3822
4859
|
}
|
|
3823
4860
|
backbone.clearCoordinateJournalPrefix(coordinateRecords.byteLength, coordinateRecordCount);
|
|
3824
4861
|
backbone.clearDocumentJournalPrefix(documentRecords.byteLength, documentRecordCount);
|
|
3825
4862
|
backbone.clearDocumentSignerJournalPrefix(signerRecords.byteLength, signerRecordCount);
|
|
4863
|
+
if (checkpoint) {
|
|
4864
|
+
await this.cleanupRetiredCheckpointFiles();
|
|
4865
|
+
}
|
|
3826
4866
|
this.lastFlushMs = Date.now();
|
|
3827
4867
|
return written;
|
|
3828
4868
|
}
|
|
@@ -3834,9 +4874,15 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
3834
4874
|
throw error;
|
|
3835
4875
|
}
|
|
3836
4876
|
}
|
|
3837
|
-
async compact(
|
|
4877
|
+
async compact(backbone) {
|
|
3838
4878
|
this.assertActive("compact");
|
|
3839
|
-
|
|
4879
|
+
if (!this.crashSafeCompaction) {
|
|
4880
|
+
throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
|
|
4881
|
+
}
|
|
4882
|
+
await this.enqueuePersistence(async () => {
|
|
4883
|
+
this.assertPersistenceHealthy();
|
|
4884
|
+
await this.flushJournalInternal(backbone, true);
|
|
4885
|
+
});
|
|
3840
4886
|
}
|
|
3841
4887
|
close() {
|
|
3842
4888
|
if (this.closePromise) {
|