@peerbit/native-backbone 0.2.9 → 0.2.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -9,6 +9,94 @@ export * from "./durability/storage.js";
9
9
 
10
10
  export type RangeResolution = "u32" | "u64";
11
11
 
12
+ type NativePrepareNoNextDocumentIndexCompactTrimFacts = (
13
+ wallTime: bigint,
14
+ logical: number,
15
+ gid: string,
16
+ type: number,
17
+ metaData: Uint8Array | undefined,
18
+ payloadData: Uint8Array,
19
+ trimLengthTo: number,
20
+ documentKey: string,
21
+ documentValuePrefixBytes: Uint8Array,
22
+ documentExistingCreated: string,
23
+ documentByteElementIndexLimit: number,
24
+ documentDeleteTrimmedHeads: boolean,
25
+ documentProjectionPlan:
26
+ | NativeBackboneSimpleDocumentProjectionPlan
27
+ | undefined,
28
+ documentProjectionEncodedDocument: Uint8Array | undefined,
29
+ documentProjectionSigner: Uint8Array | undefined,
30
+ ) => unknown[];
31
+
32
+ type NativePrepareNoNextCachedDocumentIndexCompactTrimFacts = (
33
+ wallTime: bigint,
34
+ logical: number,
35
+ gid: string,
36
+ type: number,
37
+ metaData: Uint8Array | undefined,
38
+ payloadData: Uint8Array,
39
+ trimLengthTo: number,
40
+ documentKey: string,
41
+ documentExistingCreated: string,
42
+ documentByteElementIndexLimit: number,
43
+ documentDeleteTrimmedHeads: boolean,
44
+ documentProjectionPlanId: number,
45
+ documentProjectionEncodedDocument: Uint8Array,
46
+ documentProjectionSigner: Uint8Array | undefined,
47
+ ) => unknown[];
48
+
49
+ type NativePrepareNoNextCachedPlainPutPayloadDocumentIndexCompactTrimFacts = (
50
+ wallTime: bigint,
51
+ logical: number,
52
+ gid: string,
53
+ type: number,
54
+ metaData: Uint8Array | undefined,
55
+ payloadData: Uint8Array,
56
+ trimLengthTo: number,
57
+ documentKey: string,
58
+ documentExistingCreated: string,
59
+ documentByteElementIndexLimit: number,
60
+ documentDeleteTrimmedHeads: boolean,
61
+ documentProjectionPlanId: number,
62
+ documentProjectionSigner: Uint8Array | undefined,
63
+ ) => unknown[];
64
+
65
+ type NativePrepareLatestDocumentIndexTrimFacts = (
66
+ wallTime: bigint,
67
+ logical: number,
68
+ gid: string,
69
+ type: number,
70
+ metaData: Uint8Array | undefined,
71
+ payloadData: Uint8Array,
72
+ trimLengthTo: number | undefined,
73
+ documentKey: string,
74
+ documentValuePrefixBytes: Uint8Array,
75
+ documentByteElementIndexLimit: number,
76
+ documentDeleteTrimmedHeads: boolean,
77
+ documentProjectionPlan:
78
+ | NativeBackboneSimpleDocumentProjectionPlan
79
+ | undefined,
80
+ documentProjectionEncodedDocument: Uint8Array | undefined,
81
+ documentProjectionSigner: Uint8Array | undefined,
82
+ ) => unknown[];
83
+
84
+ type NativePrepareLatestCachedDocumentIndexTrimFacts = (
85
+ wallTime: bigint,
86
+ logical: number,
87
+ gid: string,
88
+ type: number,
89
+ metaData: Uint8Array | undefined,
90
+ payloadData: Uint8Array,
91
+ trimLengthTo: number | undefined,
92
+ documentKey: string,
93
+ documentByteElementIndexLimit: number,
94
+ documentDeleteTrimmedHeads: boolean,
95
+ documentProjectionPlanId: number,
96
+ documentProjectionEncodedDocument: Uint8Array,
97
+ documentProjectionSigner: Uint8Array | undefined,
98
+ ) => unknown[];
99
+
12
100
  type NativePeerbitBackboneHandle = {
13
101
  log_len: () => number;
14
102
  block_len: () => number;
@@ -618,6 +706,17 @@ type NativePeerbitBackboneHandle = {
618
706
  minReplicas: number,
619
707
  selfHash: string,
620
708
  ) => string[];
709
+ get_routing_full_replica_leaders?: (
710
+ replicas: number,
711
+ roleAgeMs: number,
712
+ now: string,
713
+ peerFilter: string[] | undefined,
714
+ expandPeerFilter: boolean,
715
+ selfHash: string,
716
+ includeSelf: boolean,
717
+ fullReplicaFallback: boolean,
718
+ includeStrictFullReplica: boolean,
719
+ ) => unknown[] | undefined;
621
720
  put_range: (
622
721
  id: string,
623
722
  hash: string,
@@ -942,6 +1041,16 @@ type NativePeerbitBackboneHandle = {
942
1041
  payloadData: Uint8Array,
943
1042
  trimLengthTo: number | undefined,
944
1043
  ) => unknown[];
1044
+ prepare_plain_entry_commit_facts_trim_refs?: (
1045
+ wallTime: bigint,
1046
+ logical: number,
1047
+ gid: string,
1048
+ next: string[],
1049
+ type: number,
1050
+ metaData: Uint8Array | undefined,
1051
+ payloadData: Uint8Array,
1052
+ trimLengthTo: number,
1053
+ ) => unknown[];
945
1054
  prepare_plain_entry_commit_facts_document_index: (
946
1055
  wallTime: bigint,
947
1056
  logical: number,
@@ -1045,25 +1154,8 @@ type NativePeerbitBackboneHandle = {
1045
1154
  documentProjectionEncodedDocument: Uint8Array | undefined,
1046
1155
  documentProjectionSigner: Uint8Array | undefined,
1047
1156
  ) => unknown[];
1048
- prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_hashes: (
1049
- wallTime: bigint,
1050
- logical: number,
1051
- gid: string,
1052
- type: number,
1053
- metaData: Uint8Array | undefined,
1054
- payloadData: Uint8Array,
1055
- trimLengthTo: number,
1056
- documentKey: string,
1057
- documentValuePrefixBytes: Uint8Array,
1058
- documentExistingCreated: string,
1059
- documentByteElementIndexLimit: number,
1060
- documentDeleteTrimmedHeads: boolean,
1061
- documentProjectionPlan:
1062
- | NativeBackboneSimpleDocumentProjectionPlan
1063
- | undefined,
1064
- documentProjectionEncodedDocument: Uint8Array | undefined,
1065
- documentProjectionSigner: Uint8Array | undefined,
1066
- ) => unknown[];
1157
+ prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_hashes: NativePrepareNoNextDocumentIndexCompactTrimFacts;
1158
+ prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_refs?: NativePrepareNoNextDocumentIndexCompactTrimFacts;
1067
1159
  prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_trim_hashes: (
1068
1160
  wallTime: bigint,
1069
1161
  logical: number,
@@ -1080,70 +1172,14 @@ type NativePeerbitBackboneHandle = {
1080
1172
  documentProjectionEncodedDocument: Uint8Array,
1081
1173
  documentProjectionSigner: Uint8Array | undefined,
1082
1174
  ) => unknown[];
1083
- prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes: (
1084
- wallTime: bigint,
1085
- logical: number,
1086
- gid: string,
1087
- type: number,
1088
- metaData: Uint8Array | undefined,
1089
- payloadData: Uint8Array,
1090
- trimLengthTo: number,
1091
- documentKey: string,
1092
- documentExistingCreated: string,
1093
- documentByteElementIndexLimit: number,
1094
- documentDeleteTrimmedHeads: boolean,
1095
- documentProjectionPlanId: number,
1096
- documentProjectionEncodedDocument: Uint8Array,
1097
- documentProjectionSigner: Uint8Array | undefined,
1098
- ) => unknown[];
1099
- prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes_plain_put_payload?: (
1100
- wallTime: bigint,
1101
- logical: number,
1102
- gid: string,
1103
- type: number,
1104
- metaData: Uint8Array | undefined,
1105
- payloadData: Uint8Array,
1106
- trimLengthTo: number,
1107
- documentKey: string,
1108
- documentExistingCreated: string,
1109
- documentByteElementIndexLimit: number,
1110
- documentDeleteTrimmedHeads: boolean,
1111
- documentProjectionPlanId: number,
1112
- documentProjectionSigner: Uint8Array | undefined,
1113
- ) => unknown[];
1114
- prepare_plain_entry_commit_latest_facts_document_index_trim_hashes: (
1115
- wallTime: bigint,
1116
- logical: number,
1117
- gid: string,
1118
- type: number,
1119
- metaData: Uint8Array | undefined,
1120
- payloadData: Uint8Array,
1121
- trimLengthTo: number | undefined,
1122
- documentKey: string,
1123
- documentValuePrefixBytes: Uint8Array,
1124
- documentByteElementIndexLimit: number,
1125
- documentDeleteTrimmedHeads: boolean,
1126
- documentProjectionPlan:
1127
- | NativeBackboneSimpleDocumentProjectionPlan
1128
- | undefined,
1129
- documentProjectionEncodedDocument: Uint8Array | undefined,
1130
- documentProjectionSigner: Uint8Array | undefined,
1131
- ) => unknown[];
1132
- prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes: (
1133
- wallTime: bigint,
1134
- logical: number,
1135
- gid: string,
1136
- type: number,
1137
- metaData: Uint8Array | undefined,
1138
- payloadData: Uint8Array,
1139
- trimLengthTo: number | undefined,
1140
- documentKey: string,
1141
- documentByteElementIndexLimit: number,
1142
- documentDeleteTrimmedHeads: boolean,
1143
- documentProjectionPlanId: number,
1144
- documentProjectionEncodedDocument: Uint8Array,
1145
- documentProjectionSigner: Uint8Array | undefined,
1146
- ) => unknown[];
1175
+ prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes: NativePrepareNoNextCachedDocumentIndexCompactTrimFacts;
1176
+ prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs?: NativePrepareNoNextCachedDocumentIndexCompactTrimFacts;
1177
+ prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes_plain_put_payload?: NativePrepareNoNextCachedPlainPutPayloadDocumentIndexCompactTrimFacts;
1178
+ prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs_plain_put_payload?: NativePrepareNoNextCachedPlainPutPayloadDocumentIndexCompactTrimFacts;
1179
+ prepare_plain_entry_commit_latest_facts_document_index_trim_hashes: NativePrepareLatestDocumentIndexTrimFacts;
1180
+ prepare_plain_entry_commit_latest_facts_document_index_trim_refs?: NativePrepareLatestDocumentIndexTrimFacts;
1181
+ prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes: NativePrepareLatestCachedDocumentIndexTrimFacts;
1182
+ prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_refs?: NativePrepareLatestCachedDocumentIndexTrimFacts;
1147
1183
  prepare_plain_entry_storage_facts_and_put: (
1148
1184
  wallTime: bigint,
1149
1185
  logical: number,
@@ -2727,6 +2763,7 @@ export type NativeBackboneAppendResult = {
2727
2763
  assignedToRangeBoundary: boolean;
2728
2764
  trimmed: NativeBackboneTrimmedEntry[];
2729
2765
  trimmedHashes?: string[];
2766
+ trimmedGids?: string[];
2730
2767
  documentTrimmedHeadsProcessed?: boolean;
2731
2768
  documentPreviousContext?: NativeBackboneDocumentContextFacts;
2732
2769
  };
@@ -2739,6 +2776,7 @@ type NativeBackboneStorageAppendResult = {
2739
2776
  assignedToRangeBoundary: boolean;
2740
2777
  trimmed: NativeBackboneTrimmedEntry[];
2741
2778
  trimmedHashes?: string[];
2779
+ trimmedGids?: string[];
2742
2780
  documentTrimmedHeadsProcessed?: boolean;
2743
2781
  documentPreviousContext?: NativeBackboneDocumentContextFacts;
2744
2782
  };
@@ -2982,6 +3020,12 @@ export type NativeBackboneCoordinatePersistenceStore = {
2982
3020
  */
2983
3021
  readLimited?(name: string, maxBytes: number): Promise<Uint8Array | undefined>;
2984
3022
  write(name: string, bytes: Uint8Array): Promise<void>;
3023
+ /**
3024
+ * Durably replace one named file without exposing a torn destination. The
3025
+ * replacement is not visible until its bytes and containing directory have
3026
+ * crossed their physical durability barriers.
3027
+ */
3028
+ atomicReplace?(name: string, bytes: Uint8Array): Promise<void>;
2985
3029
  /**
2986
3030
  * Appends `bytes` to the named file. Callers hand over ownership of
2987
3031
  * `bytes` and must not mutate it afterwards: buffering stores keep the
@@ -3097,12 +3141,12 @@ export const nativeBackboneCoordinateDropTombstoneFile =
3097
3141
  "native-backbone-drop.tombstone";
3098
3142
 
3099
3143
  export const defaultNativeBackboneCoordinateFlushMaxPendingBytes = 1024 * 1024;
3100
- /** @deprecated Built-in coordinate persistence compaction is currently disabled. */
3144
+ /** Recommended explicit byte threshold for crash-safe Node checkpointing. */
3101
3145
  export const defaultNativeBackboneCoordinateCompactMaxJournalBytes =
3102
3146
  64 * 1024 * 1024;
3103
3147
 
3104
3148
  const nativeBackboneCoordinateCompactionDisabledMessage =
3105
- "Native backbone coordinate persistence compaction is disabled until snapshots use a crash-safe generation protocol";
3149
+ "Native backbone coordinate persistence compaction is disabled for stores without crash-safe atomic replacement";
3106
3150
 
3107
3151
  const nativeBackboneCoordinateJournalMagic = Uint8Array.from([
3108
3152
  0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x57, 0x31,
@@ -3123,6 +3167,260 @@ const coordinateJournalChecksum = (bytes: Uint8Array): number => {
3123
3167
  return checksum;
3124
3168
  };
3125
3169
 
3170
+ type NativeBackboneCoordinateCheckpointSlot = "a" | "b";
3171
+
3172
+ type NativeBackboneCoordinateCheckpointAuthority = {
3173
+ generation: bigint;
3174
+ byteLength: number;
3175
+ checksum: number;
3176
+ };
3177
+
3178
+ type NativeBackboneCoordinateCheckpointState = {
3179
+ configurationChecksum: number;
3180
+ highwater: bigint;
3181
+ pending?: NativeBackboneCoordinateCheckpointAuthority;
3182
+ completed?: NativeBackboneCoordinateCheckpointAuthority;
3183
+ };
3184
+
3185
+ type NativeBackboneCoordinateCheckpoint = {
3186
+ configurationChecksum: number;
3187
+ generation: bigint;
3188
+ coordinateSnapshot: Uint8Array;
3189
+ documentSnapshot: Uint8Array;
3190
+ documentSignerSnapshot: Uint8Array;
3191
+ };
3192
+
3193
+ const nativeBackboneCoordinateCheckpointMagic = Uint8Array.from([
3194
+ 0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x43, 0x31,
3195
+ ]);
3196
+ const nativeBackboneCoordinateCheckpointStateMagic = Uint8Array.from([
3197
+ 0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x53, 0x31,
3198
+ ]);
3199
+ const nativeBackboneCoordinateLegacyMigrationSentinel = (() => {
3200
+ const payload = Uint8Array.of(0xff);
3201
+ const bytes = new Uint8Array(
3202
+ nativeBackboneCoordinateJournalMagic.byteLength + 8 + payload.byteLength,
3203
+ );
3204
+ bytes.set(nativeBackboneCoordinateJournalMagic, 0);
3205
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
3206
+ view.setUint32(nativeBackboneCoordinateJournalMagic.byteLength, 1, true);
3207
+ view.setUint32(
3208
+ nativeBackboneCoordinateJournalMagic.byteLength + 4,
3209
+ coordinateJournalChecksum(payload),
3210
+ true,
3211
+ );
3212
+ bytes.set(payload, nativeBackboneCoordinateJournalMagic.byteLength + 8);
3213
+ return bytes;
3214
+ })();
3215
+ const nativeBackboneCoordinateCheckpointVersion = 1;
3216
+ const nativeBackboneCoordinateCheckpointHeaderBytes = 56;
3217
+ const nativeBackboneCoordinateCheckpointStateBytes = 60;
3218
+ const nativeBackboneCoordinateMaxU64 = (1n << 64n) - 1n;
3219
+
3220
+ const hasBytesPrefix = (bytes: Uint8Array, prefix: Uint8Array): boolean =>
3221
+ bytes.byteLength >= prefix.byteLength &&
3222
+ prefix.every((byte, index) => bytes[index] === byte);
3223
+
3224
+ const writeCoordinateCheckpointU64 = (
3225
+ view: DataView,
3226
+ offset: number,
3227
+ value: bigint,
3228
+ ): void => {
3229
+ if (value < 0n || value > nativeBackboneCoordinateMaxU64) {
3230
+ throw new RangeError("Native backbone checkpoint generation exceeds u64");
3231
+ }
3232
+ view.setUint32(offset, Number(value & 0xffff_ffffn), true);
3233
+ view.setUint32(offset + 4, Number(value >> 32n), true);
3234
+ };
3235
+
3236
+ const readCoordinateCheckpointU64 = (view: DataView, offset: number): bigint =>
3237
+ BigInt(view.getUint32(offset, true)) |
3238
+ (BigInt(view.getUint32(offset + 4, true)) << 32n);
3239
+
3240
+ const encodeNativeBackboneCoordinateCheckpoint = (
3241
+ checkpoint: NativeBackboneCoordinateCheckpoint,
3242
+ ): Uint8Array => {
3243
+ const coordinateLength = checkpoint.coordinateSnapshot.byteLength;
3244
+ const documentLength = checkpoint.documentSnapshot.byteLength;
3245
+ const signerLength = checkpoint.documentSignerSnapshot.byteLength;
3246
+ const byteLength =
3247
+ nativeBackboneCoordinateCheckpointHeaderBytes +
3248
+ coordinateLength +
3249
+ documentLength +
3250
+ signerLength;
3251
+ if (!Number.isSafeInteger(byteLength) || byteLength > 0xffff_ffff) {
3252
+ throw new RangeError("Native backbone coordinate checkpoint is too large");
3253
+ }
3254
+ const bytes = new Uint8Array(byteLength);
3255
+ bytes.set(nativeBackboneCoordinateCheckpointMagic, 0);
3256
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
3257
+ view.setUint32(8, nativeBackboneCoordinateCheckpointVersion, true);
3258
+ view.setUint32(12, nativeBackboneCoordinateCheckpointHeaderBytes, true);
3259
+ writeCoordinateCheckpointU64(view, 16, checkpoint.generation);
3260
+ view.setUint32(24, coordinateLength, true);
3261
+ view.setUint32(28, documentLength, true);
3262
+ view.setUint32(32, signerLength, true);
3263
+ view.setUint32(
3264
+ 36,
3265
+ coordinateJournalChecksum(checkpoint.coordinateSnapshot),
3266
+ true,
3267
+ );
3268
+ view.setUint32(
3269
+ 40,
3270
+ coordinateJournalChecksum(checkpoint.documentSnapshot),
3271
+ true,
3272
+ );
3273
+ view.setUint32(
3274
+ 44,
3275
+ coordinateJournalChecksum(checkpoint.documentSignerSnapshot),
3276
+ true,
3277
+ );
3278
+ view.setUint32(48, checkpoint.configurationChecksum, true);
3279
+ view.setUint32(52, coordinateJournalChecksum(bytes.subarray(8, 52)), true);
3280
+ let offset = nativeBackboneCoordinateCheckpointHeaderBytes;
3281
+ bytes.set(checkpoint.coordinateSnapshot, offset);
3282
+ offset += coordinateLength;
3283
+ bytes.set(checkpoint.documentSnapshot, offset);
3284
+ offset += documentLength;
3285
+ bytes.set(checkpoint.documentSignerSnapshot, offset);
3286
+ return bytes;
3287
+ };
3288
+
3289
+ const parseNativeBackboneCoordinateCheckpoint = (
3290
+ bytes: Uint8Array,
3291
+ name: string,
3292
+ ): NativeBackboneCoordinateCheckpoint => {
3293
+ if (
3294
+ bytes.byteLength < nativeBackboneCoordinateCheckpointHeaderBytes ||
3295
+ !hasBytesPrefix(bytes, nativeBackboneCoordinateCheckpointMagic)
3296
+ ) {
3297
+ throw new Error(`Native backbone ${name} has an invalid checkpoint header`);
3298
+ }
3299
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
3300
+ if (
3301
+ view.getUint32(8, true) !== nativeBackboneCoordinateCheckpointVersion ||
3302
+ view.getUint32(12, true) !==
3303
+ nativeBackboneCoordinateCheckpointHeaderBytes ||
3304
+ view.getUint32(52, true) !==
3305
+ coordinateJournalChecksum(bytes.subarray(8, 52))
3306
+ ) {
3307
+ throw new Error(`Native backbone ${name} has invalid checkpoint metadata`);
3308
+ }
3309
+ const coordinateLength = view.getUint32(24, true);
3310
+ const documentLength = view.getUint32(28, true);
3311
+ const signerLength = view.getUint32(32, true);
3312
+ const expectedLength =
3313
+ nativeBackboneCoordinateCheckpointHeaderBytes +
3314
+ coordinateLength +
3315
+ documentLength +
3316
+ signerLength;
3317
+ if (
3318
+ !Number.isSafeInteger(expectedLength) ||
3319
+ expectedLength !== bytes.byteLength
3320
+ ) {
3321
+ throw new Error(`Native backbone ${name} has an invalid checkpoint length`);
3322
+ }
3323
+ let offset = nativeBackboneCoordinateCheckpointHeaderBytes;
3324
+ const coordinateSnapshot = bytes.subarray(offset, offset + coordinateLength);
3325
+ offset += coordinateLength;
3326
+ const documentSnapshot = bytes.subarray(offset, offset + documentLength);
3327
+ offset += documentLength;
3328
+ const documentSignerSnapshot = bytes.subarray(offset, offset + signerLength);
3329
+ if (
3330
+ coordinateJournalChecksum(coordinateSnapshot) !==
3331
+ view.getUint32(36, true) ||
3332
+ coordinateJournalChecksum(documentSnapshot) !== view.getUint32(40, true) ||
3333
+ coordinateJournalChecksum(documentSignerSnapshot) !==
3334
+ view.getUint32(44, true)
3335
+ ) {
3336
+ throw new Error(
3337
+ `Native backbone ${name} has a checkpoint checksum mismatch`,
3338
+ );
3339
+ }
3340
+ return {
3341
+ configurationChecksum: view.getUint32(48, true),
3342
+ generation: readCoordinateCheckpointU64(view, 16),
3343
+ coordinateSnapshot,
3344
+ documentSnapshot,
3345
+ documentSignerSnapshot,
3346
+ };
3347
+ };
3348
+
3349
+ const encodeNativeBackboneCoordinateCheckpointState = (
3350
+ state: NativeBackboneCoordinateCheckpointState,
3351
+ ): Uint8Array => {
3352
+ const bytes = new Uint8Array(nativeBackboneCoordinateCheckpointStateBytes);
3353
+ bytes.set(nativeBackboneCoordinateCheckpointStateMagic, 0);
3354
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
3355
+ view.setUint32(8, nativeBackboneCoordinateCheckpointVersion, true);
3356
+ view.setUint32(12, state.configurationChecksum, true);
3357
+ writeCoordinateCheckpointU64(view, 16, state.highwater);
3358
+ writeCoordinateCheckpointU64(view, 24, state.pending?.generation ?? 0n);
3359
+ view.setUint32(32, state.pending?.byteLength ?? 0, true);
3360
+ view.setUint32(36, state.pending?.checksum ?? 0, true);
3361
+ writeCoordinateCheckpointU64(view, 40, state.completed?.generation ?? 0n);
3362
+ view.setUint32(48, state.completed?.byteLength ?? 0, true);
3363
+ view.setUint32(52, state.completed?.checksum ?? 0, true);
3364
+ view.setUint32(56, coordinateJournalChecksum(bytes.subarray(8, 56)), true);
3365
+ return bytes;
3366
+ };
3367
+
3368
+ const parseNativeBackboneCoordinateCheckpointState = (
3369
+ bytes: Uint8Array,
3370
+ name: string,
3371
+ ): NativeBackboneCoordinateCheckpointState => {
3372
+ if (
3373
+ bytes.byteLength !== nativeBackboneCoordinateCheckpointStateBytes ||
3374
+ !hasBytesPrefix(bytes, nativeBackboneCoordinateCheckpointStateMagic)
3375
+ ) {
3376
+ throw new Error(
3377
+ `Native backbone ${name} has an invalid checkpoint authority`,
3378
+ );
3379
+ }
3380
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
3381
+ if (
3382
+ view.getUint32(8, true) !== nativeBackboneCoordinateCheckpointVersion ||
3383
+ view.getUint32(56, true) !==
3384
+ coordinateJournalChecksum(bytes.subarray(8, 56))
3385
+ ) {
3386
+ throw new Error(`Native backbone ${name} has corrupt checkpoint authority`);
3387
+ }
3388
+ const configurationChecksum = view.getUint32(12, true);
3389
+ const highwater = readCoordinateCheckpointU64(view, 16);
3390
+ const pendingGeneration = readCoordinateCheckpointU64(view, 24);
3391
+ const completedGeneration = readCoordinateCheckpointU64(view, 40);
3392
+ const authority = (
3393
+ generation: bigint,
3394
+ lengthOffset: number,
3395
+ checksumOffset: number,
3396
+ ): NativeBackboneCoordinateCheckpointAuthority | undefined => {
3397
+ const byteLength = view.getUint32(lengthOffset, true);
3398
+ const checksum = view.getUint32(checksumOffset, true);
3399
+ if (generation === 0n) {
3400
+ if (byteLength !== 0 || checksum !== 0) {
3401
+ throw new Error(`Native backbone ${name} has empty authority metadata`);
3402
+ }
3403
+ return undefined;
3404
+ }
3405
+ if (byteLength < nativeBackboneCoordinateCheckpointHeaderBytes) {
3406
+ throw new Error(`Native backbone ${name} has invalid authority length`);
3407
+ }
3408
+ return { generation, byteLength, checksum };
3409
+ };
3410
+ const pending = authority(pendingGeneration, 32, 36);
3411
+ const completed = authority(completedGeneration, 48, 52);
3412
+ if (
3413
+ (completed && completed.generation > highwater) ||
3414
+ (pending && pending.generation !== highwater) ||
3415
+ (pending && completed && pending.generation <= completed.generation) ||
3416
+ (!pending && completed && completed.generation !== highwater) ||
3417
+ (!pending && !completed && highwater !== 0n)
3418
+ ) {
3419
+ throw new Error(`Native backbone ${name} has invalid generation authority`);
3420
+ }
3421
+ return { configurationChecksum, highwater, pending, completed };
3422
+ };
3423
+
3126
3424
  const hasCoordinateJournalMagic = (bytes: Uint8Array): boolean =>
3127
3425
  bytes.byteLength >= nativeBackboneCoordinateJournalMagic.byteLength &&
3128
3426
  nativeBackboneCoordinateJournalMagic.every(
@@ -3172,11 +3470,24 @@ const resolveCoordinateFlushMaxPendingBytes = (
3172
3470
  ? defaultNativeBackboneCoordinateFlushMaxPendingBytes
3173
3471
  : undefined;
3174
3472
 
3473
+ const validateCoordinateCompactionThreshold = (
3474
+ value: number,
3475
+ name: string,
3476
+ ): number => {
3477
+ if (!Number.isSafeInteger(value) || value <= 0) {
3478
+ throw new RangeError(
3479
+ `Native backbone ${name} must be a positive safe integer`,
3480
+ );
3481
+ }
3482
+ return value;
3483
+ };
3484
+
3175
3485
  type NativeBackboneNodeFs = {
3176
3486
  mkdir(path: string, options?: { recursive?: boolean }): Promise<unknown>;
3177
3487
  readFile(path: string): Promise<Uint8Array>;
3178
3488
  writeFile(path: string, data: Uint8Array): Promise<unknown>;
3179
3489
  appendFile(path: string, data: Uint8Array): Promise<unknown>;
3490
+ rename?(from: string, to: string): Promise<unknown>;
3180
3491
  /** Explicit capability for pre-allocation-bounded positional reads. */
3181
3492
  openBoundedRead?(
3182
3493
  path: string,
@@ -3947,6 +4258,8 @@ const storageAppendResultFromRow = (
3947
4258
  trimHashRows,
3948
4259
  documentTrimmedHeadsProcessed,
3949
4260
  documentPreviousContextRow,
4261
+ _trimGidExtension,
4262
+ trimGidRows,
3950
4263
  ] = row as [
3951
4264
  unknown[],
3952
4265
  unknown[] | undefined,
@@ -3957,6 +4270,8 @@ const storageAppendResultFromRow = (
3957
4270
  unknown[] | undefined,
3958
4271
  boolean | undefined,
3959
4272
  unknown[] | undefined,
4273
+ undefined,
4274
+ string[] | undefined,
3960
4275
  ];
3961
4276
  return {
3962
4277
  entry: storageFactsEntryFromRow(entryRow),
@@ -3965,6 +4280,8 @@ const storageAppendResultFromRow = (
3965
4280
  assignedToRangeBoundary,
3966
4281
  coordinate: appendCoordinatePlanFromRow(resolution, coordinateRow),
3967
4282
  ...trimmedRowsAndHashesResult(trimRows, trimHashRows),
4283
+ trimmedGids:
4284
+ trimGidRows ?? (trimRows as unknown[][]).map((trim) => trim[1] as string),
3968
4285
  documentTrimmedHeadsProcessed,
3969
4286
  documentPreviousContext: documentContextFactsFromRow(
3970
4287
  documentPreviousContextRow,
@@ -3986,6 +4303,8 @@ const committedStorageAppendResultFromRow = (
3986
4303
  trimHashRows,
3987
4304
  documentTrimmedHeadsProcessed,
3988
4305
  documentPreviousContextRow,
4306
+ _trimGidExtension,
4307
+ trimGidRows,
3989
4308
  ] = row as [
3990
4309
  unknown[],
3991
4310
  unknown[] | undefined,
@@ -3996,6 +4315,8 @@ const committedStorageAppendResultFromRow = (
3996
4315
  unknown[] | undefined,
3997
4316
  boolean | undefined,
3998
4317
  unknown[] | undefined,
4318
+ undefined,
4319
+ string[] | undefined,
3999
4320
  ];
4000
4321
  return {
4001
4322
  entry: committedStorageFactsEntryFromRow(entryRow),
@@ -4004,6 +4325,8 @@ const committedStorageAppendResultFromRow = (
4004
4325
  assignedToRangeBoundary,
4005
4326
  coordinate: appendCoordinatePlanFromRow(resolution, coordinateRow),
4006
4327
  ...trimmedRowsAndHashesResult(trimRows, trimHashRows),
4328
+ trimmedGids:
4329
+ trimGidRows ?? (trimRows as unknown[][]).map((trim) => trim[1] as string),
4007
4330
  documentTrimmedHeadsProcessed,
4008
4331
  documentPreviousContext: documentContextFactsFromRow(
4009
4332
  documentPreviousContextRow,
@@ -4015,7 +4338,14 @@ const compactCommittedNoNextStorageAppendResultFromRow = (
4015
4338
  resolution: RangeResolution,
4016
4339
  row: unknown[],
4017
4340
  ): NativeBackboneAppendResult => {
4018
- const [hash, byteLength, metaBytes, fourth] = row as [
4341
+ const appendedTrimGids =
4342
+ row.length >= 12 &&
4343
+ row[row.length - 2] === undefined &&
4344
+ Array.isArray(row[row.length - 1])
4345
+ ? (row[row.length - 1] as string[])
4346
+ : undefined;
4347
+ const baseRow = appendedTrimGids ? row.slice(0, -2) : row;
4348
+ const [hash, byteLength, metaBytes, fourth] = baseRow as [
4019
4349
  string,
4020
4350
  number,
4021
4351
  Uint8Array | undefined,
@@ -4023,7 +4353,7 @@ const compactCommittedNoNextStorageAppendResultFromRow = (
4023
4353
  ];
4024
4354
  const hasDigestRow = fourth instanceof Uint8Array;
4025
4355
  const hashDigestBytes = hasDigestRow ? (fourth as Uint8Array) : undefined;
4026
- const rest = row.slice(hasDigestRow ? 4 : 3);
4356
+ const rest = baseRow.slice(hasDigestRow ? 4 : 3);
4027
4357
  const usesNestedCoordinateRow = Array.isArray(rest[0]);
4028
4358
  let coordinate: NativeBackboneCoordinatePlan;
4029
4359
  let leaderRows: unknown[] | undefined;
@@ -4106,6 +4436,7 @@ const compactCommittedNoNextStorageAppendResultFromRow = (
4106
4436
  coordinate,
4107
4437
  trimmed: [],
4108
4438
  trimmedHashes: trimHashRows ?? [],
4439
+ trimmedGids: appendedTrimGids,
4109
4440
  documentTrimmedHeadsProcessed,
4110
4441
  };
4111
4442
  };
@@ -4114,6 +4445,13 @@ const compactCommittedLatestStorageAppendResultFromRow = (
4114
4445
  resolution: RangeResolution,
4115
4446
  row: unknown[],
4116
4447
  ): NativeBackboneAppendResult => {
4448
+ const appendedTrimGids =
4449
+ row.length >= 13 &&
4450
+ row[row.length - 2] === undefined &&
4451
+ Array.isArray(row[row.length - 1])
4452
+ ? (row[row.length - 1] as string[])
4453
+ : undefined;
4454
+ const baseRow = appendedTrimGids ? row.slice(0, -2) : row;
4117
4455
  const [
4118
4456
  hash,
4119
4457
  byteLength,
@@ -4127,7 +4465,7 @@ const compactCommittedLatestStorageAppendResultFromRow = (
4127
4465
  tenth,
4128
4466
  eleventh,
4129
4467
  twelfth,
4130
- ] = row as [
4468
+ ] = baseRow as [
4131
4469
  string,
4132
4470
  number,
4133
4471
  Uint8Array | undefined,
@@ -4172,6 +4510,7 @@ const compactCommittedLatestStorageAppendResultFromRow = (
4172
4510
  coordinate,
4173
4511
  trimmed: [],
4174
4512
  trimmedHashes: trimHashRows ?? [],
4513
+ trimmedGids: appendedTrimGids,
4175
4514
  documentTrimmedHeadsProcessed,
4176
4515
  documentPreviousContext: documentContextFactsFromRow(
4177
4516
  documentPreviousContextRow,
@@ -4200,6 +4539,20 @@ const preparedCommitFactsFromRow = (
4200
4539
  return prepared;
4201
4540
  };
4202
4541
 
4542
+ const preparedCommitFactsWithTrimRefsFromRow = (
4543
+ row: unknown[],
4544
+ ): NativeBackboneCommittedEntry & {
4545
+ trimmedEntryHashes: string[];
4546
+ trimmedEntryGids: string[];
4547
+ } => {
4548
+ const prepared = committedStorageFactsEntryFromRow(row[0] as unknown[]);
4549
+ return {
4550
+ ...prepared,
4551
+ trimmedEntryHashes: row[1] as string[],
4552
+ trimmedEntryGids: row[2] as string[],
4553
+ };
4554
+ };
4555
+
4203
4556
  const preparedCommitFactsWithLatestDocumentContextFromRow = (
4204
4557
  row: unknown[],
4205
4558
  ): NativeBackboneCommittedEntry & {
@@ -4222,12 +4575,39 @@ const preparedCommitFactsWithLatestDocumentContextFromRow = (
4222
4575
  };
4223
4576
  };
4224
4577
 
4225
- const compactPreparedCommitFactsWithTrimHashesFromRow = (
4578
+ const preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow = (
4226
4579
  row: unknown[],
4227
4580
  ): NativeBackboneCommittedEntry & {
4228
- trimmedEntryHashes?: string[];
4581
+ trimmedEntryHashes: string[];
4582
+ trimmedEntryGids: string[];
4229
4583
  documentTrimmedHeadsProcessed?: boolean;
4584
+ documentPreviousContext?: NativeBackboneDocumentContextFacts;
4230
4585
  } => {
4586
+ const [
4587
+ entryRow,
4588
+ trimHashRows,
4589
+ trimGidRows,
4590
+ documentTrimmedHeadsProcessed,
4591
+ contextRow,
4592
+ ] = row as [
4593
+ unknown[],
4594
+ string[],
4595
+ string[],
4596
+ boolean | undefined,
4597
+ unknown[] | undefined,
4598
+ ];
4599
+ return {
4600
+ ...committedStorageFactsEntryFromRow(entryRow),
4601
+ trimmedEntryHashes: trimHashRows,
4602
+ trimmedEntryGids: trimGidRows,
4603
+ documentTrimmedHeadsProcessed,
4604
+ documentPreviousContext: documentContextFactsFromRow(contextRow),
4605
+ };
4606
+ };
4607
+
4608
+ const compactPreparedCommitFactsBaseFromRow = (
4609
+ row: unknown[],
4610
+ ): { entry: NativeBackboneCommittedEntry; trimRowOffset: number } => {
4231
4611
  const [hash, byteLength, metaBytes, fourth] = row as [
4232
4612
  string,
4233
4613
  number,
@@ -4235,24 +4615,58 @@ const compactPreparedCommitFactsWithTrimHashesFromRow = (
4235
4615
  Uint8Array | string[] | undefined,
4236
4616
  ];
4237
4617
  const hasDigestRow = fourth instanceof Uint8Array;
4238
- const hashDigestBytes = hasDigestRow ? fourth : undefined;
4239
- const trimHashOffset = hasDigestRow ? 4 : 3;
4240
- const trimHashRows = row[trimHashOffset] as string[] | undefined;
4241
- const documentTrimmedHeadsProcessed = row[trimHashOffset + 1] as
4618
+ return {
4619
+ entry: {
4620
+ cid: hash,
4621
+ hash,
4622
+ next: [],
4623
+ metaBytes,
4624
+ byteLength,
4625
+ hashDigestBytes: hasDigestRow ? fourth : undefined,
4626
+ },
4627
+ trimRowOffset: hasDigestRow ? 4 : 3,
4628
+ };
4629
+ };
4630
+
4631
+ const compactPreparedCommitFactsWithTrimHashesFromRow = (
4632
+ row: unknown[],
4633
+ ): NativeBackboneCommittedEntry & {
4634
+ trimmedEntryHashes?: string[];
4635
+ documentTrimmedHeadsProcessed?: boolean;
4636
+ } => {
4637
+ const { entry, trimRowOffset } = compactPreparedCommitFactsBaseFromRow(row);
4638
+ const trimHashRows = row[trimRowOffset] as string[] | undefined;
4639
+ const documentTrimmedHeadsProcessed = row[trimRowOffset + 1] as
4242
4640
  | boolean
4243
4641
  | undefined;
4244
4642
  return {
4245
- cid: hash,
4246
- hash,
4247
- next: [],
4248
- metaBytes,
4249
- byteLength,
4250
- hashDigestBytes,
4643
+ ...entry,
4251
4644
  trimmedEntryHashes: trimHashRows ?? [],
4252
4645
  documentTrimmedHeadsProcessed,
4253
4646
  };
4254
4647
  };
4255
4648
 
4649
+ const compactPreparedCommitFactsWithTrimRefsFromRow = (
4650
+ row: unknown[],
4651
+ ): NativeBackboneCommittedEntry & {
4652
+ trimmedEntryHashes: string[];
4653
+ trimmedEntryGids: string[];
4654
+ documentTrimmedHeadsProcessed?: boolean;
4655
+ } => {
4656
+ const { entry, trimRowOffset } = compactPreparedCommitFactsBaseFromRow(row);
4657
+ const trimHashRows = row[trimRowOffset] as string[];
4658
+ const trimGidRows = row[trimRowOffset + 1] as string[];
4659
+ const documentTrimmedHeadsProcessed = row[trimRowOffset + 2] as
4660
+ | boolean
4661
+ | undefined;
4662
+ return {
4663
+ ...entry,
4664
+ trimmedEntryHashes: trimHashRows,
4665
+ trimmedEntryGids: trimGidRows,
4666
+ documentTrimmedHeadsProcessed,
4667
+ };
4668
+ };
4669
+
4256
4670
  const nativeLogCommitEntryColumns = (
4257
4671
  entries: NativeBackboneLogCommitEntry[],
4258
4672
  ) => {
@@ -5080,7 +5494,9 @@ class NativeBackboneLogGraph {
5080
5494
  | (NativeBackboneCommittedEntry & {
5081
5495
  trimmedEntries?: NativeBackboneLogEntry[];
5082
5496
  trimmedEntryHashes?: string[];
5497
+ trimmedEntryGids?: string[];
5083
5498
  documentTrimmedHeadsProcessed?: boolean;
5499
+ documentPreviousContext?: NativeBackboneDocumentContextFacts;
5084
5500
  })
5085
5501
  | undefined {
5086
5502
  if (this.options?.commitBlocks === false) {
@@ -5100,31 +5516,31 @@ class NativeBackboneLogGraph {
5100
5516
  const documentIndexArgs = nativeDocumentIndexArgs(documentIndex);
5101
5517
  const projection = documentIndex?.projection;
5102
5518
  if (
5103
- documentIndex?.useLatestContext &&
5104
- documentIndexArgs &&
5105
- input.resolveTrimmedEntries === false
5519
+ !documentIndexArgs &&
5520
+ input.resolveTrimmedEntries === false &&
5521
+ input.trimLengthTo != null &&
5522
+ this.native.prepare_plain_entry_commit_facts_trim_refs
5106
5523
  ) {
5107
- if (projection && this.options?.documentProjectionPlanId) {
5108
- return preparedCommitFactsWithLatestDocumentContextFromRow(
5109
- this.native.prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes(
5110
- wallTime,
5111
- logical,
5112
- input.gid,
5113
- entryType,
5114
- input.metaData,
5115
- input.payloadData,
5116
- input.trimLengthTo,
5117
- documentIndex.key,
5118
- documentIndex.byteElementIndexLimit ?? 0,
5119
- documentIndex.deleteTrimmedHeads === true,
5120
- this.options.documentProjectionPlanId(projection.plan),
5121
- projection.encodedDocument,
5122
- projection.signer,
5123
- ),
5124
- );
5125
- }
5126
- return preparedCommitFactsWithLatestDocumentContextFromRow(
5127
- this.native.prepare_plain_entry_commit_latest_facts_document_index_trim_hashes(
5524
+ return preparedCommitFactsWithTrimRefsFromRow(
5525
+ this.native.prepare_plain_entry_commit_facts_trim_refs(
5526
+ wallTime,
5527
+ logical,
5528
+ input.gid,
5529
+ input.next ?? [],
5530
+ entryType,
5531
+ input.metaData,
5532
+ input.payloadData,
5533
+ input.trimLengthTo,
5534
+ ),
5535
+ );
5536
+ }
5537
+ if (
5538
+ documentIndex?.useLatestContext &&
5539
+ documentIndexArgs &&
5540
+ input.resolveTrimmedEntries === false
5541
+ ) {
5542
+ if (projection && this.options?.documentProjectionPlanId) {
5543
+ const args = [
5128
5544
  wallTime,
5129
5545
  logical,
5130
5546
  input.gid,
@@ -5133,12 +5549,53 @@ class NativeBackboneLogGraph {
5133
5549
  input.payloadData,
5134
5550
  input.trimLengthTo,
5135
5551
  documentIndex.key,
5136
- documentIndex.valuePrefixBytes ?? EMPTY_UINT8_ARRAY,
5137
5552
  documentIndex.byteElementIndexLimit ?? 0,
5138
5553
  documentIndex.deleteTrimmedHeads === true,
5139
- projection?.plan,
5140
- projection?.encodedDocument,
5141
- projection?.signer,
5554
+ this.options.documentProjectionPlanId(projection.plan),
5555
+ projection.encodedDocument,
5556
+ projection.signer,
5557
+ ] as const;
5558
+ const prepareTrimRefs =
5559
+ this.native
5560
+ .prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_refs;
5561
+ if (prepareTrimRefs) {
5562
+ return preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow(
5563
+ prepareTrimRefs.call(this.native, ...args),
5564
+ );
5565
+ }
5566
+ return preparedCommitFactsWithLatestDocumentContextFromRow(
5567
+ this.native.prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes(
5568
+ ...args,
5569
+ ),
5570
+ );
5571
+ }
5572
+ const args = [
5573
+ wallTime,
5574
+ logical,
5575
+ input.gid,
5576
+ entryType,
5577
+ input.metaData,
5578
+ input.payloadData,
5579
+ input.trimLengthTo,
5580
+ documentIndex.key,
5581
+ documentIndex.valuePrefixBytes ?? EMPTY_UINT8_ARRAY,
5582
+ documentIndex.byteElementIndexLimit ?? 0,
5583
+ documentIndex.deleteTrimmedHeads === true,
5584
+ projection?.plan,
5585
+ projection?.encodedDocument,
5586
+ projection?.signer,
5587
+ ] as const;
5588
+ const prepareTrimRefs =
5589
+ this.native
5590
+ .prepare_plain_entry_commit_latest_facts_document_index_trim_refs;
5591
+ if (prepareTrimRefs) {
5592
+ return preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow(
5593
+ prepareTrimRefs.call(this.native, ...args),
5594
+ );
5595
+ }
5596
+ return preparedCommitFactsWithLatestDocumentContextFromRow(
5597
+ this.native.prepare_plain_entry_commit_latest_facts_document_index_trim_hashes(
5598
+ ...args,
5142
5599
  ),
5143
5600
  );
5144
5601
  }
@@ -5153,49 +5610,71 @@ class NativeBackboneLogGraph {
5153
5610
  const projectionPlanId = this.options.documentProjectionPlanId(
5154
5611
  projection.plan,
5155
5612
  );
5613
+ const plainPutPayloadArgs = [
5614
+ wallTime,
5615
+ logical,
5616
+ input.gid,
5617
+ entryType,
5618
+ input.metaData,
5619
+ input.payloadData,
5620
+ input.trimLengthTo,
5621
+ documentIndex.key,
5622
+ documentIndex.existingCreated == null
5623
+ ? ""
5624
+ : integerString(documentIndex.existingCreated),
5625
+ documentIndex.byteElementIndexLimit ?? 0,
5626
+ documentIndex.deleteTrimmedHeads === true,
5627
+ projectionPlanId,
5628
+ projection.signer,
5629
+ ] as const;
5630
+ const plainPutPayloadCommitWithTrimRefs =
5631
+ this.native
5632
+ .prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs_plain_put_payload;
5633
+ if (plainPutPayloadCommitWithTrimRefs) {
5634
+ return compactPreparedCommitFactsWithTrimRefsFromRow(
5635
+ plainPutPayloadCommitWithTrimRefs.call(
5636
+ this.native,
5637
+ ...plainPutPayloadArgs,
5638
+ ),
5639
+ );
5640
+ }
5156
5641
  const plainPutPayloadCommit =
5157
5642
  this.native
5158
5643
  .prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes_plain_put_payload;
5159
5644
  if (plainPutPayloadCommit) {
5160
5645
  return compactPreparedCommitFactsWithTrimHashesFromRow(
5161
- plainPutPayloadCommit.call(
5162
- this.native,
5163
- wallTime,
5164
- logical,
5165
- input.gid,
5166
- entryType,
5167
- input.metaData,
5168
- input.payloadData,
5169
- input.trimLengthTo,
5170
- documentIndex.key,
5171
- documentIndex.existingCreated == null
5172
- ? ""
5173
- : integerString(documentIndex.existingCreated),
5174
- documentIndex.byteElementIndexLimit ?? 0,
5175
- documentIndex.deleteTrimmedHeads === true,
5176
- projectionPlanId,
5177
- projection.signer,
5178
- ),
5646
+ plainPutPayloadCommit.call(this.native, ...plainPutPayloadArgs),
5647
+ );
5648
+ }
5649
+ const args = [
5650
+ wallTime,
5651
+ logical,
5652
+ input.gid,
5653
+ entryType,
5654
+ input.metaData,
5655
+ input.payloadData,
5656
+ input.trimLengthTo,
5657
+ documentIndex.key,
5658
+ documentIndex.existingCreated == null
5659
+ ? ""
5660
+ : integerString(documentIndex.existingCreated),
5661
+ documentIndex.byteElementIndexLimit ?? 0,
5662
+ documentIndex.deleteTrimmedHeads === true,
5663
+ projectionPlanId,
5664
+ projection.encodedDocument,
5665
+ projection.signer,
5666
+ ] as const;
5667
+ const prepareTrimRefs =
5668
+ this.native
5669
+ .prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs;
5670
+ if (prepareTrimRefs) {
5671
+ return compactPreparedCommitFactsWithTrimRefsFromRow(
5672
+ prepareTrimRefs.call(this.native, ...args),
5179
5673
  );
5180
5674
  }
5181
5675
  return compactPreparedCommitFactsWithTrimHashesFromRow(
5182
5676
  this.native.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes(
5183
- wallTime,
5184
- logical,
5185
- input.gid,
5186
- entryType,
5187
- input.metaData,
5188
- input.payloadData,
5189
- input.trimLengthTo,
5190
- documentIndex.key,
5191
- documentIndex.existingCreated == null
5192
- ? ""
5193
- : integerString(documentIndex.existingCreated),
5194
- documentIndex.byteElementIndexLimit ?? 0,
5195
- documentIndex.deleteTrimmedHeads === true,
5196
- projectionPlanId,
5197
- projection.encodedDocument,
5198
- projection.signer,
5677
+ ...args,
5199
5678
  ),
5200
5679
  );
5201
5680
  }
@@ -5302,6 +5781,24 @@ class NativeBackboneLogGraph {
5302
5781
  input.trimLengthTo != null &&
5303
5782
  hasNoNext
5304
5783
  ) {
5784
+ const prepareTrimRefs =
5785
+ this.native
5786
+ .prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_refs;
5787
+ if (prepareTrimRefs) {
5788
+ return compactPreparedCommitFactsWithTrimRefsFromRow(
5789
+ prepareTrimRefs.call(
5790
+ this.native,
5791
+ wallTime,
5792
+ logical,
5793
+ input.gid,
5794
+ entryType,
5795
+ input.metaData,
5796
+ input.payloadData,
5797
+ input.trimLengthTo,
5798
+ ...documentIndexArgs,
5799
+ ),
5800
+ );
5801
+ }
5305
5802
  return compactPreparedCommitFactsWithTrimHashesFromRow(
5306
5803
  this.native.prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_hashes(
5307
5804
  wallTime,
@@ -6455,9 +6952,7 @@ export class NativePeerbitBackbone {
6455
6952
  end2: bigint | number | string;
6456
6953
  limit: number;
6457
6954
  }): bigint[] | undefined {
6458
- if (
6459
- typeof this.native.entry_hash_numbers_in_range_limited !== "function"
6460
- ) {
6955
+ if (typeof this.native.entry_hash_numbers_in_range_limited !== "function") {
6461
6956
  return undefined;
6462
6957
  }
6463
6958
  return rowsToNumbers(
@@ -6967,6 +7462,21 @@ export class NativePeerbitBackbone {
6967
7462
  return this.native.full_replica_candidates_for(minReplicas, selfHash);
6968
7463
  }
6969
7464
 
7465
+ getRoutingFullReplicaLeaders(
7466
+ replicas: number,
7467
+ options?: NativeBackboneFindLeaderOptions,
7468
+ ): Map<string, NativeBackboneLeaderSample> | undefined {
7469
+ if (!this.native.get_routing_full_replica_leaders) {
7470
+ return undefined;
7471
+ }
7472
+ return rowsToSamples(
7473
+ this.native.get_routing_full_replica_leaders(
7474
+ replicas,
7475
+ ...findLeaderArguments(options),
7476
+ ),
7477
+ );
7478
+ }
7479
+
6970
7480
  clearEntryCoordinates(): void {
6971
7481
  this.native.clear_entry_coordinates();
6972
7482
  }
@@ -8768,6 +9278,7 @@ export class NativeBackboneNodeCoordinatePersistenceStore
8768
9278
  maxBytes: number,
8769
9279
  ) => Promise<Uint8Array | undefined>;
8770
9280
  readonly durableBarrier?: (name?: string) => Promise<void>;
9281
+ readonly atomicReplace?: (name: string, bytes: Uint8Array) => Promise<void>;
8771
9282
 
8772
9283
  constructor(
8773
9284
  private readonly directory: string,
@@ -8783,6 +9294,12 @@ export class NativeBackboneNodeCoordinatePersistenceStore
8783
9294
  this.readLimited = (name, maxBytes) =>
8784
9295
  this.readWithinLimit(name, maxBytes);
8785
9296
  }
9297
+ if (
9298
+ !fs ||
9299
+ (typeof fs.open === "function" && typeof fs.rename === "function")
9300
+ ) {
9301
+ this.atomicReplace = (name, bytes) => this.replaceAtomically(name, bytes);
9302
+ }
8786
9303
  }
8787
9304
 
8788
9305
  private async nodeFs(): Promise<NativeBackboneNodeFs> {
@@ -8965,6 +9482,51 @@ export class NativeBackboneNodeCoordinatePersistenceStore
8965
9482
  await fs.writeFile(path, bytes);
8966
9483
  }
8967
9484
 
9485
+ private async replaceAtomically(
9486
+ name: string,
9487
+ bytes: Uint8Array,
9488
+ ): Promise<void> {
9489
+ const fs = await this.ensureDirectory();
9490
+ if (!fs.open || !fs.rename) {
9491
+ throw new Error(
9492
+ "Node coordinate persistence does not expose atomic file replacement",
9493
+ );
9494
+ }
9495
+ const validName = validateCoordinatePersistenceName(name);
9496
+ const temporaryName = validateCoordinatePersistenceName(`${validName}.tmp`);
9497
+ const path = await this.filePath(validName);
9498
+ const temporaryPath = await this.filePath(temporaryName);
9499
+ await this.closeAppendHandle(path);
9500
+ await this.closeAppendHandle(temporaryPath);
9501
+ await fs.rm(temporaryPath, { force: true });
9502
+ await fs.writeFile(temporaryPath, bytes);
9503
+ let temporaryHandle: NativeBackboneNodeAppendFileHandle | undefined;
9504
+ try {
9505
+ temporaryHandle = await fs.open(temporaryPath, "r");
9506
+ if (typeof temporaryHandle.sync !== "function") {
9507
+ throw new Error(
9508
+ "Node coordinate persistence atomic replacement does not expose FileHandle.sync",
9509
+ );
9510
+ }
9511
+ await temporaryHandle.sync();
9512
+ } finally {
9513
+ await temporaryHandle?.close();
9514
+ }
9515
+ await fs.rename(temporaryPath, path);
9516
+ let directoryHandle: NativeBackboneNodeAppendFileHandle | undefined;
9517
+ try {
9518
+ directoryHandle = await fs.open(this.directory, "r");
9519
+ if (typeof directoryHandle.sync !== "function") {
9520
+ throw new Error(
9521
+ "Node coordinate persistence atomic replacement does not expose directory sync",
9522
+ );
9523
+ }
9524
+ await directoryHandle.sync();
9525
+ } finally {
9526
+ await directoryHandle?.close();
9527
+ }
9528
+ }
9529
+
8968
9530
  async append(name: string, bytes: Uint8Array): Promise<void> {
8969
9531
  if (this.appendFailure !== undefined) {
8970
9532
  throw this.appendFailure;
@@ -9317,6 +9879,7 @@ export class NativeBackboneBufferedCoordinatePersistenceStore
9317
9879
  ) => Promise<Uint8Array | undefined>;
9318
9880
  readonly supportsRemoval: boolean;
9319
9881
  readonly durableBarrier?: (name?: string) => Promise<void>;
9882
+ readonly atomicReplace?: (name: string, bytes: Uint8Array) => Promise<void>;
9320
9883
 
9321
9884
  constructor(
9322
9885
  private readonly inner: NativeBackboneCoordinatePersistenceStore,
@@ -9335,6 +9898,12 @@ export class NativeBackboneBufferedCoordinatePersistenceStore
9335
9898
  await inner.durableBarrier!(name);
9336
9899
  };
9337
9900
  }
9901
+ if (typeof inner.atomicReplace === "function") {
9902
+ this.atomicReplace = async (name, bytes) => {
9903
+ await this.flush(name);
9904
+ await inner.atomicReplace!(name, bytes);
9905
+ };
9906
+ }
9338
9907
  }
9339
9908
 
9340
9909
  private buffer(name: string): Uint8Array[] {
@@ -9480,7 +10049,7 @@ export class NativeBackboneCoordinatePersistence {
9480
10049
  readonly flushIntervalMs?: number;
9481
10050
  readonly compactMaxJournalBytes?: number;
9482
10051
  readonly compactMaxJournalRecords?: number;
9483
- readonly crashSafeCompaction = false;
10052
+ readonly crashSafeCompaction: boolean;
9484
10053
  readonly durableBarrier: boolean;
9485
10054
  readonly supportsDrop: boolean;
9486
10055
  readonly dropIsTerminal = true;
@@ -9490,9 +10059,32 @@ export class NativeBackboneCoordinatePersistence {
9490
10059
  private readonly documentJournalFile: string;
9491
10060
  private readonly documentSignerSnapshotFile: string;
9492
10061
  private readonly documentSignerJournalFile: string;
10062
+ private readonly checkpointConfigurationChecksum: number;
10063
+ private readonly checkpointStateFile: string;
10064
+ private readonly checkpointFiles: Record<
10065
+ NativeBackboneCoordinateCheckpointSlot,
10066
+ string
10067
+ >;
10068
+ private readonly checkpointJournalFiles: Record<
10069
+ NativeBackboneCoordinateCheckpointSlot,
10070
+ {
10071
+ coordinate: string;
10072
+ document: string;
10073
+ signer: string;
10074
+ }
10075
+ >;
9493
10076
  private journalInitialized: boolean | undefined;
10077
+ private journalByteLength = 0;
10078
+ private journalRecordCount = 0;
9494
10079
  private documentJournalInitialized: boolean | undefined;
10080
+ private documentJournalByteLength = 0;
10081
+ private documentJournalRecordCount = 0;
9495
10082
  private documentSignerJournalInitialized: boolean | undefined;
10083
+ private documentSignerJournalByteLength = 0;
10084
+ private documentSignerJournalRecordCount = 0;
10085
+ private checkpointHighwater = 0n;
10086
+ private checkpointCompleted?: NativeBackboneCoordinateCheckpointAuthority;
10087
+ private checkpointPending?: NativeBackboneCoordinateCheckpointAuthority;
9496
10088
  private lastFlushMs = Date.now();
9497
10089
  private persistenceQueue: Promise<unknown> | undefined;
9498
10090
  private persistenceLifecycle:
@@ -9539,24 +10131,87 @@ export class NativeBackboneCoordinatePersistence {
9539
10131
  options.documentSignerJournal ??
9540
10132
  nativeBackboneCoordinatePersistenceFiles.documentSignerJournal,
9541
10133
  );
10134
+ this.checkpointConfigurationChecksum = coordinateJournalChecksum(
10135
+ new TextEncoder().encode(
10136
+ JSON.stringify([
10137
+ this.snapshotFile,
10138
+ this.journalFile,
10139
+ this.documentSnapshotFile,
10140
+ this.documentJournalFile,
10141
+ this.documentSignerSnapshotFile,
10142
+ this.documentSignerJournalFile,
10143
+ ]),
10144
+ ),
10145
+ );
10146
+ this.checkpointStateFile = validateCoordinatePersistenceName(
10147
+ `${this.snapshotFile}.checkpoint-state`,
10148
+ );
10149
+ this.checkpointFiles = {
10150
+ a: validateCoordinatePersistenceName(`${this.snapshotFile}.checkpoint-a`),
10151
+ b: validateCoordinatePersistenceName(`${this.snapshotFile}.checkpoint-b`),
10152
+ };
10153
+ this.checkpointJournalFiles = {
10154
+ a: {
10155
+ coordinate: validateCoordinatePersistenceName(
10156
+ `${this.journalFile}.checkpoint-a`,
10157
+ ),
10158
+ document: validateCoordinatePersistenceName(
10159
+ `${this.documentJournalFile}.checkpoint-a`,
10160
+ ),
10161
+ signer: validateCoordinatePersistenceName(
10162
+ `${this.documentSignerJournalFile}.checkpoint-a`,
10163
+ ),
10164
+ },
10165
+ b: {
10166
+ coordinate: validateCoordinatePersistenceName(
10167
+ `${this.journalFile}.checkpoint-b`,
10168
+ ),
10169
+ document: validateCoordinatePersistenceName(
10170
+ `${this.documentJournalFile}.checkpoint-b`,
10171
+ ),
10172
+ signer: validateCoordinatePersistenceName(
10173
+ `${this.documentSignerJournalFile}.checkpoint-b`,
10174
+ ),
10175
+ },
10176
+ };
9542
10177
  if (
9543
- this.configuredFiles().includes(nativeBackboneCoordinateDropTombstoneFile)
10178
+ this.configuredFiles().includes(
10179
+ nativeBackboneCoordinateDropTombstoneFile,
10180
+ ) ||
10181
+ new Set(this.configuredFiles()).size !== this.configuredFiles().length
9544
10182
  ) {
9545
10183
  throw new Error(
9546
- "Native backbone coordinate persistence file conflicts with its drop tombstone",
10184
+ "Native backbone coordinate persistence files conflict with one another or with its drop tombstone",
9547
10185
  );
9548
10186
  }
10187
+ this.crashSafeCompaction =
10188
+ typeof store.atomicReplace === "function" &&
10189
+ this.supportsDrop &&
10190
+ this.durableBarrier;
9549
10191
  this.flushOnAppend = options.flushOnAppend ?? true;
9550
10192
  this.flushMaxPendingBytes = resolveCoordinateFlushMaxPendingBytes(options);
9551
10193
  if (options.flushIntervalMs != null) {
9552
10194
  this.flushIntervalMs = Math.max(0, options.flushIntervalMs);
9553
10195
  }
9554
10196
  if (
9555
- options.compactMaxJournalBytes != null ||
9556
- options.compactMaxJournalRecords != null
10197
+ (options.compactMaxJournalBytes != null ||
10198
+ options.compactMaxJournalRecords != null) &&
10199
+ !this.crashSafeCompaction
9557
10200
  ) {
9558
10201
  throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
9559
10202
  }
10203
+ if (options.compactMaxJournalBytes != null) {
10204
+ this.compactMaxJournalBytes = validateCoordinateCompactionThreshold(
10205
+ options.compactMaxJournalBytes,
10206
+ "compactMaxJournalBytes",
10207
+ );
10208
+ }
10209
+ if (options.compactMaxJournalRecords != null) {
10210
+ this.compactMaxJournalRecords = validateCoordinateCompactionThreshold(
10211
+ options.compactMaxJournalRecords,
10212
+ "compactMaxJournalRecords",
10213
+ );
10214
+ }
9560
10215
  }
9561
10216
 
9562
10217
  /** Durable operation-intent capability for strict shared-log transactions. */
@@ -9564,7 +10219,7 @@ export class NativeBackboneCoordinatePersistence {
9564
10219
  return this.store;
9565
10220
  }
9566
10221
 
9567
- private configuredFiles(): string[] {
10222
+ private legacyConfiguredFiles(): string[] {
9568
10223
  return [
9569
10224
  this.snapshotFile,
9570
10225
  this.journalFile,
@@ -9575,6 +10230,47 @@ export class NativeBackboneCoordinatePersistence {
9575
10230
  ];
9576
10231
  }
9577
10232
 
10233
+ private configuredFiles(): string[] {
10234
+ return [
10235
+ ...this.legacyConfiguredFiles(),
10236
+ this.checkpointStateFile,
10237
+ this.checkpointFiles.a,
10238
+ this.checkpointFiles.b,
10239
+ this.checkpointJournalFiles.a.coordinate,
10240
+ this.checkpointJournalFiles.a.document,
10241
+ this.checkpointJournalFiles.a.signer,
10242
+ this.checkpointJournalFiles.b.coordinate,
10243
+ this.checkpointJournalFiles.b.document,
10244
+ this.checkpointJournalFiles.b.signer,
10245
+ `${this.checkpointStateFile}.tmp`,
10246
+ `${this.checkpointFiles.a}.tmp`,
10247
+ `${this.checkpointFiles.b}.tmp`,
10248
+ `${this.journalFile}.tmp`,
10249
+ `${nativeBackboneCoordinateDropTombstoneFile}.tmp`,
10250
+ ];
10251
+ }
10252
+
10253
+ private checkpointSlot(
10254
+ generation: bigint,
10255
+ ): NativeBackboneCoordinateCheckpointSlot {
10256
+ return generation % 2n === 1n ? "a" : "b";
10257
+ }
10258
+
10259
+ private activeJournalFiles(): {
10260
+ coordinate: string;
10261
+ document: string;
10262
+ signer: string;
10263
+ } {
10264
+ const completed = this.checkpointCompleted;
10265
+ return completed
10266
+ ? this.checkpointJournalFiles[this.checkpointSlot(completed.generation)]
10267
+ : {
10268
+ coordinate: this.journalFile,
10269
+ document: this.documentJournalFile,
10270
+ signer: this.documentSignerJournalFile,
10271
+ };
10272
+ }
10273
+
9578
10274
  private assertLifecycleActive(operation: string): void {
9579
10275
  if (this.persistenceLifecycle !== "active") {
9580
10276
  throw new Error(
@@ -9584,9 +10280,7 @@ export class NativeBackboneCoordinatePersistence {
9584
10280
  }
9585
10281
 
9586
10282
  private assertActive(operation: string): void {
9587
- if (this.persistenceFailure !== undefined) {
9588
- throw this.persistenceFailure;
9589
- }
10283
+ this.assertPersistenceHealthy();
9590
10284
  this.assertLifecycleActive(operation);
9591
10285
  if (this.dropInitiatedOnGeneration) {
9592
10286
  throw new Error(
@@ -9595,13 +10289,44 @@ export class NativeBackboneCoordinatePersistence {
9595
10289
  }
9596
10290
  }
9597
10291
 
10292
+ private assertPersistenceHealthy(): void {
10293
+ if (this.persistenceFailure !== undefined) {
10294
+ throw this.persistenceFailure;
10295
+ }
10296
+ }
10297
+
9598
10298
  private resetJournalTracking(): void {
9599
10299
  this.journalInitialized = undefined;
10300
+ this.journalByteLength = 0;
10301
+ this.journalRecordCount = 0;
9600
10302
  this.documentJournalInitialized = undefined;
10303
+ this.documentJournalByteLength = 0;
10304
+ this.documentJournalRecordCount = 0;
9601
10305
  this.documentSignerJournalInitialized = undefined;
10306
+ this.documentSignerJournalByteLength = 0;
10307
+ this.documentSignerJournalRecordCount = 0;
10308
+ this.checkpointHighwater = 0n;
10309
+ this.checkpointCompleted = undefined;
10310
+ this.checkpointPending = undefined;
9602
10311
  this.lastFlushMs = Date.now();
9603
10312
  }
9604
10313
 
10314
+ private async persistDropTombstone(
10315
+ files: readonly string[],
10316
+ preferAtomicReplace = false,
10317
+ ): Promise<void> {
10318
+ const bytes = nativeBackboneCoordinateDropTombstoneBytes(files);
10319
+ if (preferAtomicReplace && this.store.atomicReplace) {
10320
+ await this.store.atomicReplace(
10321
+ nativeBackboneCoordinateDropTombstoneFile,
10322
+ bytes,
10323
+ );
10324
+ return;
10325
+ }
10326
+ await this.store.write(nativeBackboneCoordinateDropTombstoneFile, bytes);
10327
+ await this.barrierFile(nativeBackboneCoordinateDropTombstoneFile);
10328
+ }
10329
+
9605
10330
  private async eraseDropFiles(files: readonly string[]): Promise<void> {
9606
10331
  if (!this.supportsDrop || !this.store.remove) {
9607
10332
  throw new Error(
@@ -9651,17 +10376,7 @@ export class NativeBackboneCoordinatePersistence {
9651
10376
  this.dropInitiatedOnGeneration = true;
9652
10377
  this.persistenceLifecycle = "dropping";
9653
10378
  await this.enqueuePersistence(async () => {
9654
- await this.store.write(
9655
- nativeBackboneCoordinateDropTombstoneFile,
9656
- nativeBackboneCoordinateDropTombstoneBytes(files),
9657
- );
9658
- if (this.store.durableBarrier) {
9659
- await this.store.durableBarrier(
9660
- nativeBackboneCoordinateDropTombstoneFile,
9661
- );
9662
- } else {
9663
- await this.store.flush?.(nativeBackboneCoordinateDropTombstoneFile);
9664
- }
10379
+ await this.persistDropTombstone(files);
9665
10380
  await this.eraseDropFiles(files);
9666
10381
  this.resetJournalTracking();
9667
10382
  this.persistenceLifecycle = "dropped";
@@ -9699,6 +10414,57 @@ export class NativeBackboneCoordinatePersistence {
9699
10414
  });
9700
10415
  }
9701
10416
 
10417
+ private async barrierFile(name: string): Promise<void> {
10418
+ if (this.store.durableBarrier) {
10419
+ await this.store.durableBarrier(name);
10420
+ } else {
10421
+ await this.store.flush?.(name);
10422
+ }
10423
+ }
10424
+
10425
+ private async hasDurableLegacyMigrationSentinel(): Promise<boolean> {
10426
+ const existing = await this.store.read(this.journalFile);
10427
+ if (
10428
+ existing &&
10429
+ existing.byteLength ===
10430
+ nativeBackboneCoordinateLegacyMigrationSentinel.byteLength &&
10431
+ nativeBackboneCoordinateLegacyMigrationSentinel.every(
10432
+ (byte, index) => existing[index] === byte,
10433
+ )
10434
+ ) {
10435
+ // A previous process may have observed the bytes from page cache before
10436
+ // its durability barrier rejected. Cross a fresh barrier on every reopen
10437
+ // before any post-checkpoint WAL append can be admitted.
10438
+ await this.barrierFile(this.journalFile);
10439
+ return true;
10440
+ }
10441
+ return false;
10442
+ }
10443
+
10444
+ private async installLegacyMigrationSentinel(): Promise<void> {
10445
+ if (await this.hasDurableLegacyMigrationSentinel()) {
10446
+ return;
10447
+ }
10448
+ if (!this.store.atomicReplace) {
10449
+ throw new Error(
10450
+ "Native backbone completed checkpoint can not install its downgrade sentinel atomically",
10451
+ );
10452
+ }
10453
+ await this.store.atomicReplace(
10454
+ this.journalFile,
10455
+ nativeBackboneCoordinateLegacyMigrationSentinel,
10456
+ );
10457
+ }
10458
+
10459
+ private async requireLegacyMigrationSentinel(): Promise<void> {
10460
+ if (await this.hasDurableLegacyMigrationSentinel()) {
10461
+ return;
10462
+ }
10463
+ throw new Error(
10464
+ "Native backbone completed checkpoint downgrade sentinel is missing or has been replaced; refusing to overwrite possible legacy writes",
10465
+ );
10466
+ }
10467
+
9702
10468
  async hydrate(backbone: NativePeerbitBackbone): Promise<number> {
9703
10469
  this.assertActive("hydrate");
9704
10470
  // Claim the lifecycle synchronously. close() queues after this complete
@@ -9709,33 +10475,175 @@ export class NativeBackboneCoordinatePersistence {
9709
10475
  this.assertHydrating();
9710
10476
  await this.resumeDropInternal("hydrating");
9711
10477
  this.assertHydrating();
9712
- const [
9713
- snapshot,
9714
- journal,
9715
- documentSnapshot,
9716
- documentJournal,
9717
- documentSignerSnapshot,
9718
- documentSignerJournal,
9719
- ] = await Promise.all([
9720
- this.store.read(this.snapshotFile),
9721
- this.store.read(this.journalFile),
9722
- this.store.read(this.documentSnapshotFile),
9723
- this.store.read(this.documentJournalFile),
9724
- this.store.read(this.documentSignerSnapshotFile),
9725
- this.store.read(this.documentSignerJournalFile),
9726
- ]);
10478
+ const checkpointStateBytes = await this.store.read(
10479
+ this.checkpointStateFile,
10480
+ );
9727
10481
  this.assertHydrating();
9728
- try {
9729
- validateCoordinateJournal(journal, this.journalFile);
9730
- validateCoordinateJournal(documentJournal, this.documentJournalFile);
9731
- validateCoordinateJournal(
10482
+ let checkpointState:
10483
+ | NativeBackboneCoordinateCheckpointState
10484
+ | undefined;
10485
+ if (checkpointStateBytes) {
10486
+ try {
10487
+ checkpointState = parseNativeBackboneCoordinateCheckpointState(
10488
+ checkpointStateBytes,
10489
+ this.checkpointStateFile,
10490
+ );
10491
+ if (
10492
+ checkpointState.configurationChecksum !==
10493
+ this.checkpointConfigurationChecksum
10494
+ ) {
10495
+ throw new Error(
10496
+ "Native backbone checkpoint authority does not match the configured persistence files",
10497
+ );
10498
+ }
10499
+ } catch (error) {
10500
+ this.persistenceFailure ??= error;
10501
+ throw this.persistenceFailure;
10502
+ }
10503
+ }
10504
+ this.checkpointHighwater = checkpointState?.highwater ?? 0n;
10505
+ this.checkpointPending = checkpointState?.pending;
10506
+ this.checkpointCompleted = checkpointState?.completed;
10507
+ let checkpointAuthority = this.checkpointCompleted;
10508
+ let promotePendingCheckpoint = false;
10509
+ if (!checkpointAuthority && this.checkpointPending) {
10510
+ try {
10511
+ // On the first generation only, the sentinel is the durable stage
10512
+ // boundary: publication writes it after the pending checkpoint and its
10513
+ // WAL headers, but before completed authority. A pre-checkpoint reader
10514
+ // can no longer consume legacy state, so recovery may validate and
10515
+ // promote the staged generation. Once a completed generation exists the
10516
+ // sentinel predates later pending work and is not a promotion signal.
10517
+ if (await this.hasDurableLegacyMigrationSentinel()) {
10518
+ checkpointAuthority = this.checkpointPending;
10519
+ promotePendingCheckpoint = true;
10520
+ }
10521
+ } catch (error) {
10522
+ this.persistenceFailure ??= error;
10523
+ throw this.persistenceFailure;
10524
+ }
10525
+ }
10526
+
10527
+ let snapshot: Uint8Array | undefined;
10528
+ let journal: Uint8Array | undefined;
10529
+ let documentSnapshot: Uint8Array | undefined;
10530
+ let documentJournal: Uint8Array | undefined;
10531
+ let documentSignerSnapshot: Uint8Array | undefined;
10532
+ let documentSignerJournal: Uint8Array | undefined;
10533
+ let journalNames = {
10534
+ coordinate: this.journalFile,
10535
+ document: this.documentJournalFile,
10536
+ signer: this.documentSignerJournalFile,
10537
+ };
10538
+ if (checkpointAuthority) {
10539
+ const authority = checkpointAuthority;
10540
+ const authorityKind = promotePendingCheckpoint
10541
+ ? "promotable pending"
10542
+ : "completed";
10543
+ const slot = this.checkpointSlot(authority.generation);
10544
+ const checkpointFile = this.checkpointFiles[slot];
10545
+ journalNames = this.checkpointJournalFiles[slot];
10546
+ const [
10547
+ checkpointBytes,
10548
+ coordinateJournal,
10549
+ valueJournal,
10550
+ signerJournal,
10551
+ ] = await Promise.all([
10552
+ this.store.read(checkpointFile),
10553
+ this.store.read(journalNames.coordinate),
10554
+ this.store.read(journalNames.document),
10555
+ this.store.read(journalNames.signer),
10556
+ ]);
10557
+ this.assertHydrating();
10558
+ try {
10559
+ if (
10560
+ !checkpointBytes ||
10561
+ checkpointBytes.byteLength !== authority.byteLength ||
10562
+ coordinateJournalChecksum(checkpointBytes) !== authority.checksum
10563
+ ) {
10564
+ throw new Error(
10565
+ `Native backbone ${authorityKind} checkpoint authority is missing or corrupt`,
10566
+ );
10567
+ }
10568
+ if (!coordinateJournal || !valueJournal || !signerJournal) {
10569
+ throw new Error(
10570
+ `Native backbone ${authorityKind} checkpoint WAL generation is incomplete`,
10571
+ );
10572
+ }
10573
+ const checkpoint = parseNativeBackboneCoordinateCheckpoint(
10574
+ checkpointBytes,
10575
+ checkpointFile,
10576
+ );
10577
+ if (
10578
+ checkpoint.generation !== authority.generation ||
10579
+ checkpoint.configurationChecksum !==
10580
+ this.checkpointConfigurationChecksum
10581
+ ) {
10582
+ throw new Error(
10583
+ `Native backbone checkpoint generation or configuration does not match its ${authorityKind} authority`,
10584
+ );
10585
+ }
10586
+ snapshot = checkpoint.coordinateSnapshot;
10587
+ documentSnapshot = checkpoint.documentSnapshot;
10588
+ documentSignerSnapshot = checkpoint.documentSignerSnapshot;
10589
+ journal = coordinateJournal;
10590
+ documentJournal = valueJournal;
10591
+ documentSignerJournal = signerJournal;
10592
+ } catch (error) {
10593
+ this.persistenceFailure ??= error;
10594
+ throw this.persistenceFailure;
10595
+ }
10596
+ } else {
10597
+ [
10598
+ snapshot,
10599
+ journal,
10600
+ documentSnapshot,
10601
+ documentJournal,
10602
+ documentSignerSnapshot,
9732
10603
  documentSignerJournal,
9733
- this.documentSignerJournalFile,
9734
- );
10604
+ ] = await Promise.all([
10605
+ this.store.read(this.snapshotFile),
10606
+ this.store.read(this.journalFile),
10607
+ this.store.read(this.documentSnapshotFile),
10608
+ this.store.read(this.documentJournalFile),
10609
+ this.store.read(this.documentSignerSnapshotFile),
10610
+ this.store.read(this.documentSignerJournalFile),
10611
+ ]);
10612
+ this.assertHydrating();
10613
+ }
10614
+ try {
10615
+ if (
10616
+ checkpointAuthority &&
10617
+ (!journal ||
10618
+ !hasCoordinateJournalMagic(journal) ||
10619
+ !documentJournal ||
10620
+ !hasCoordinateJournalMagic(documentJournal) ||
10621
+ !documentSignerJournal ||
10622
+ !hasCoordinateJournalMagic(documentSignerJournal))
10623
+ ) {
10624
+ throw new Error(
10625
+ `Native backbone ${promotePendingCheckpoint ? "promotable pending" : "completed"} checkpoint WAL generation has a missing or invalid header`,
10626
+ );
10627
+ }
10628
+ validateCoordinateJournal(journal, journalNames.coordinate);
10629
+ validateCoordinateJournal(documentJournal, journalNames.document);
10630
+ validateCoordinateJournal(documentSignerJournal, journalNames.signer);
9735
10631
  } catch (error) {
9736
10632
  this.persistenceFailure ??= error;
9737
10633
  throw this.persistenceFailure;
9738
10634
  }
10635
+ if (checkpointAuthority && !promotePendingCheckpoint) {
10636
+ try {
10637
+ // Completed authority is never permission to overwrite an ordinary
10638
+ // legacy WAL: it may contain writes made by a downgraded process in the
10639
+ // publication cut. Require the exact durable sentinel before loading.
10640
+ await this.requireLegacyMigrationSentinel();
10641
+ this.assertHydrating();
10642
+ } catch (error) {
10643
+ this.persistenceFailure ??= error;
10644
+ throw this.persistenceFailure;
10645
+ }
10646
+ }
9739
10647
  let operations: number;
9740
10648
  let documentOperations: number;
9741
10649
  let documentSignerOperations: number;
@@ -9767,12 +10675,52 @@ export class NativeBackboneCoordinatePersistence {
9767
10675
  }
9768
10676
  throw error;
9769
10677
  }
10678
+ if (promotePendingCheckpoint) {
10679
+ try {
10680
+ const authority = checkpointAuthority!;
10681
+ if (!this.store.atomicReplace) {
10682
+ throw new Error(
10683
+ "Native backbone pending checkpoint can not publish completed authority atomically",
10684
+ );
10685
+ }
10686
+ await this.store.atomicReplace(
10687
+ this.checkpointStateFile,
10688
+ encodeNativeBackboneCoordinateCheckpointState({
10689
+ configurationChecksum: this.checkpointConfigurationChecksum,
10690
+ highwater: authority.generation,
10691
+ completed: authority,
10692
+ }),
10693
+ );
10694
+ this.checkpointCompleted = authority;
10695
+ this.checkpointPending = undefined;
10696
+ this.assertHydrating();
10697
+ } catch (error) {
10698
+ this.persistenceFailure ??= error;
10699
+ throw this.persistenceFailure;
10700
+ }
10701
+ }
10702
+ if (this.checkpointCompleted || this.checkpointPending) {
10703
+ try {
10704
+ await this.cleanupRetiredCheckpointFiles();
10705
+ this.assertHydrating();
10706
+ } catch (error) {
10707
+ this.persistenceFailure ??= error;
10708
+ throw this.persistenceFailure;
10709
+ }
10710
+ }
9770
10711
  this.assertHydrating();
9771
10712
  this.journalInitialized = !!journal && journal.byteLength > 0;
10713
+ this.journalByteLength = journal?.byteLength ?? 0;
10714
+ this.journalRecordCount = operations;
9772
10715
  this.documentJournalInitialized =
9773
10716
  !!documentJournal && documentJournal.byteLength > 0;
10717
+ this.documentJournalByteLength = documentJournal?.byteLength ?? 0;
10718
+ this.documentJournalRecordCount = documentOperations;
9774
10719
  this.documentSignerJournalInitialized =
9775
10720
  !!documentSignerJournal && documentSignerJournal.byteLength > 0;
10721
+ this.documentSignerJournalByteLength =
10722
+ documentSignerJournal?.byteLength ?? 0;
10723
+ this.documentSignerJournalRecordCount = documentSignerOperations;
9776
10724
  backbone.setCoordinateJournalEnabled(true);
9777
10725
  backbone.setDocumentJournalEnabled(true);
9778
10726
  backbone.setDocumentSignerJournalEnabled(true);
@@ -9815,15 +10763,19 @@ export class NativeBackboneCoordinatePersistence {
9815
10763
  return false;
9816
10764
  }
9817
10765
  let tombstone: NativeBackboneCoordinateDropTombstoneBody;
10766
+ let expandedFiles: string[];
9818
10767
  try {
9819
10768
  tombstone = parseNativeBackboneCoordinateDropTombstone(bytes);
9820
- for (const file of this.configuredFiles()) {
10769
+ for (const file of this.legacyConfiguredFiles()) {
9821
10770
  if (!tombstone.files.includes(file)) {
9822
10771
  throw new Error(
9823
10772
  "Native backbone drop tombstone does not cover the configured namespace",
9824
10773
  );
9825
10774
  }
9826
10775
  }
10776
+ expandedFiles = [
10777
+ ...new Set([...tombstone.files, ...this.configuredFiles()]),
10778
+ ];
9827
10779
  } catch (error) {
9828
10780
  // Corruption must never hydrate stale files, but an explicit drop must
9829
10781
  // remain available to overwrite the bad marker and erase the namespace.
@@ -9831,6 +10783,17 @@ export class NativeBackboneCoordinatePersistence {
9831
10783
  this.persistenceLifecycle = this.closePromise ? "closing" : "active";
9832
10784
  throw this.persistenceFailure;
9833
10785
  }
10786
+ if (expandedFiles.length !== tombstone.files.length) {
10787
+ // Older valid markers only knew the six snapshot/WAL files. Widen the
10788
+ // destructive intent durably before erasing any checkpoint generation,
10789
+ // while preserving additional namespace files recorded by that version.
10790
+ await this.persistDropTombstone(expandedFiles, true);
10791
+ tombstone = { ...tombstone, files: expandedFiles };
10792
+ }
10793
+ // A prior replacement attempt may have made the expanded bytes visible before
10794
+ // its durability barrier rejected. Cross a fresh barrier on every recovery
10795
+ // before allowing any target removal.
10796
+ await this.barrierFile(nativeBackboneCoordinateDropTombstoneFile);
9834
10797
  await this.eraseDropFiles(tombstone.files);
9835
10798
  this.resetJournalTracking();
9836
10799
  this.persistenceLifecycle =
@@ -9887,7 +10850,13 @@ export class NativeBackboneCoordinatePersistence {
9887
10850
  this.assertActive("flush");
9888
10851
  // Serialized with compact() so a flush never clears records appended to
9889
10852
  // the wasm journal while a previous flush was awaiting its disk write.
9890
- return this.enqueuePersistence(() => this.flushJournalInternal(backbone));
10853
+ return this.enqueuePersistence(() => {
10854
+ // A call can be admitted while an earlier queued write is still in flight.
10855
+ // Re-check the sticky failure at execution time so no suffix can ACK after
10856
+ // that earlier write or authority switch failed.
10857
+ this.assertPersistenceHealthy();
10858
+ return this.flushJournalInternal(backbone);
10859
+ });
9891
10860
  }
9892
10861
 
9893
10862
  private enqueuePersistence<T>(fn: () => Promise<T>): Promise<T> {
@@ -9908,9 +10877,185 @@ export class NativeBackboneCoordinatePersistence {
9908
10877
  return next;
9909
10878
  }
9910
10879
 
10880
+ private shouldCompactJournal(
10881
+ additionalBytes: number,
10882
+ additionalRecords: number,
10883
+ ): boolean {
10884
+ return (
10885
+ (this.compactMaxJournalBytes != null &&
10886
+ this.journalByteLength +
10887
+ this.documentJournalByteLength +
10888
+ this.documentSignerJournalByteLength +
10889
+ additionalBytes >=
10890
+ this.compactMaxJournalBytes) ||
10891
+ (this.compactMaxJournalRecords != null &&
10892
+ this.journalRecordCount +
10893
+ this.documentJournalRecordCount +
10894
+ this.documentSignerJournalRecordCount +
10895
+ additionalRecords >=
10896
+ this.compactMaxJournalRecords)
10897
+ );
10898
+ }
10899
+
10900
+ private nextCheckpointGeneration(): bigint {
10901
+ let generation = this.checkpointHighwater + 1n;
10902
+ const active = this.checkpointCompleted;
10903
+ if (
10904
+ active &&
10905
+ this.checkpointSlot(generation) === this.checkpointSlot(active.generation)
10906
+ ) {
10907
+ generation += 1n;
10908
+ }
10909
+ if (generation > nativeBackboneCoordinateMaxU64) {
10910
+ throw new RangeError(
10911
+ "Native backbone checkpoint generation highwater is exhausted",
10912
+ );
10913
+ }
10914
+ return generation;
10915
+ }
10916
+
10917
+ private async publishCheckpoint(checkpoint: {
10918
+ coordinateSnapshot: Uint8Array;
10919
+ documentSnapshot: Uint8Array;
10920
+ documentSignerSnapshot: Uint8Array;
10921
+ }): Promise<void> {
10922
+ if (
10923
+ !this.crashSafeCompaction ||
10924
+ !this.store.atomicReplace ||
10925
+ !this.store.remove
10926
+ ) {
10927
+ throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
10928
+ }
10929
+ const generation = this.nextCheckpointGeneration();
10930
+ const slot = this.checkpointSlot(generation);
10931
+ const checkpointBytes = encodeNativeBackboneCoordinateCheckpoint({
10932
+ configurationChecksum: this.checkpointConfigurationChecksum,
10933
+ generation,
10934
+ ...checkpoint,
10935
+ });
10936
+ const authority: NativeBackboneCoordinateCheckpointAuthority = {
10937
+ generation,
10938
+ byteLength: checkpointBytes.byteLength,
10939
+ checksum: coordinateJournalChecksum(checkpointBytes),
10940
+ };
10941
+ const pendingState: NativeBackboneCoordinateCheckpointState = {
10942
+ configurationChecksum: this.checkpointConfigurationChecksum,
10943
+ highwater: generation,
10944
+ pending: authority,
10945
+ completed: this.checkpointCompleted,
10946
+ };
10947
+ await this.store.atomicReplace(
10948
+ this.checkpointStateFile,
10949
+ encodeNativeBackboneCoordinateCheckpointState(pendingState),
10950
+ );
10951
+ this.checkpointHighwater = generation;
10952
+ this.checkpointPending = authority;
10953
+
10954
+ const targetJournals = this.checkpointJournalFiles[slot];
10955
+ for (const file of [
10956
+ targetJournals.coordinate,
10957
+ targetJournals.document,
10958
+ targetJournals.signer,
10959
+ ]) {
10960
+ await this.store.remove(file);
10961
+ }
10962
+ const journalHeaders = [
10963
+ [targetJournals.coordinate, nativeBackboneCoordinateJournalMagic],
10964
+ [targetJournals.document, nativeBackboneCoordinateJournalMagic],
10965
+ [targetJournals.signer, nativeBackboneCoordinateJournalMagic],
10966
+ ] as const;
10967
+ for (const [file, header] of journalHeaders) {
10968
+ await this.store.write(file, header);
10969
+ await this.barrierFile(file);
10970
+ }
10971
+ await this.store.atomicReplace(this.checkpointFiles[slot], checkpointBytes);
10972
+
10973
+ // A prior release does not understand checkpoint authority. The sentinel is
10974
+ // the first-generation publication boundary: install and durably revalidate
10975
+ // it before completed authority becomes visible. On later generations it
10976
+ // must already be exact; never overwrite possible writes from a downgrade.
10977
+ if (this.checkpointCompleted) {
10978
+ await this.requireLegacyMigrationSentinel();
10979
+ } else {
10980
+ await this.installLegacyMigrationSentinel();
10981
+ }
10982
+
10983
+ const completedState: NativeBackboneCoordinateCheckpointState = {
10984
+ configurationChecksum: this.checkpointConfigurationChecksum,
10985
+ highwater: generation,
10986
+ completed: authority,
10987
+ };
10988
+ await this.store.atomicReplace(
10989
+ this.checkpointStateFile,
10990
+ encodeNativeBackboneCoordinateCheckpointState(completedState),
10991
+ );
10992
+
10993
+ // The completed authority is now the only writable generation. Update all
10994
+ // in-memory routing synchronously before another queued flush can start.
10995
+ this.checkpointCompleted = authority;
10996
+ this.checkpointPending = undefined;
10997
+ this.journalInitialized = true;
10998
+ this.documentJournalInitialized = true;
10999
+ this.documentSignerJournalInitialized = true;
11000
+ this.journalByteLength = nativeBackboneCoordinateJournalMagic.byteLength;
11001
+ this.documentJournalByteLength =
11002
+ nativeBackboneCoordinateJournalMagic.byteLength;
11003
+ this.documentSignerJournalByteLength =
11004
+ nativeBackboneCoordinateJournalMagic.byteLength;
11005
+ this.journalRecordCount = 0;
11006
+ this.documentJournalRecordCount = 0;
11007
+ this.documentSignerJournalRecordCount = 0;
11008
+ }
11009
+
11010
+ private async cleanupRetiredCheckpointFiles(): Promise<void> {
11011
+ if (
11012
+ !this.store.remove ||
11013
+ (!this.checkpointCompleted && !this.checkpointPending)
11014
+ ) {
11015
+ return;
11016
+ }
11017
+ const retiredSlot: NativeBackboneCoordinateCheckpointSlot = this
11018
+ .checkpointCompleted
11019
+ ? this.checkpointSlot(this.checkpointCompleted.generation) === "a"
11020
+ ? "b"
11021
+ : "a"
11022
+ : this.checkpointSlot(this.checkpointPending!.generation);
11023
+ const retiredJournals = this.checkpointJournalFiles[retiredSlot];
11024
+ const files = [
11025
+ ...(this.checkpointCompleted
11026
+ ? [
11027
+ this.snapshotFile,
11028
+ this.documentSnapshotFile,
11029
+ this.documentJournalFile,
11030
+ this.documentSignerSnapshotFile,
11031
+ this.documentSignerJournalFile,
11032
+ ]
11033
+ : []),
11034
+ this.checkpointFiles[retiredSlot],
11035
+ `${this.checkpointFiles[retiredSlot]}.tmp`,
11036
+ retiredJournals.coordinate,
11037
+ retiredJournals.document,
11038
+ retiredJournals.signer,
11039
+ ];
11040
+ const removals = await Promise.allSettled(
11041
+ files.map((file) => this.store.remove!(file)),
11042
+ );
11043
+ if (removals.every((result) => result.status === "fulfilled")) {
11044
+ try {
11045
+ await this.store.durableBarrier?.();
11046
+ } catch {
11047
+ // Cleanup is retryable debt. Publication and the caller's ACK already
11048
+ // belong to the new generation, so never turn this into an ambiguous
11049
+ // append failure after the authority switch.
11050
+ }
11051
+ }
11052
+ }
11053
+
9911
11054
  private async flushJournalInternal(
9912
11055
  backbone: NativePeerbitBackbone,
11056
+ forceCheckpoint = false,
9913
11057
  ): Promise<number> {
11058
+ this.assertPersistenceHealthy();
9914
11059
  let written = 0;
9915
11060
  let persistenceMutationStarted = false;
9916
11061
  let coordinateBytes: Uint8Array | undefined;
@@ -9922,10 +11067,42 @@ export class NativeBackboneCoordinatePersistence {
9922
11067
  const documentRecordCount = backbone.documentPendingJournalLength;
9923
11068
  const signerRecords = backbone.documentSignerJournal();
9924
11069
  const signerRecordCount = backbone.documentSignerPendingJournalLength;
11070
+ const additionalBytes =
11071
+ coordinateRecords.byteLength +
11072
+ documentRecords.byteLength +
11073
+ signerRecords.byteLength +
11074
+ (coordinateRecords.byteLength > 0 && this.journalInitialized !== true
11075
+ ? backbone.coordinateJournalHeader().byteLength
11076
+ : 0) +
11077
+ (documentRecords.byteLength > 0 &&
11078
+ this.documentJournalInitialized !== true
11079
+ ? backbone.documentJournalHeader().byteLength
11080
+ : 0) +
11081
+ (signerRecords.byteLength > 0 &&
11082
+ this.documentSignerJournalInitialized !== true
11083
+ ? backbone.documentSignerJournalHeader().byteLength
11084
+ : 0);
11085
+ const additionalRecords =
11086
+ coordinateRecordCount + documentRecordCount + signerRecordCount;
11087
+ const checkpointRequested =
11088
+ forceCheckpoint ||
11089
+ (this.crashSafeCompaction &&
11090
+ this.shouldCompactJournal(additionalBytes, additionalRecords));
11091
+ // These three synchronous copies and the journal prefixes above are one
11092
+ // logical cut. No mutation can slip into the checkpoint without also being
11093
+ // represented by one of the captured prefixes.
11094
+ const checkpoint = checkpointRequested
11095
+ ? {
11096
+ coordinateSnapshot: backbone.coordinateSnapshot(),
11097
+ documentSnapshot: backbone.documentSnapshot(),
11098
+ documentSignerSnapshot: backbone.documentSignerSnapshot(),
11099
+ }
11100
+ : undefined;
9925
11101
  try {
11102
+ const activeJournals = this.activeJournalFiles();
9926
11103
  if (coordinateRecords.byteLength > 0) {
9927
11104
  if (this.journalInitialized === undefined) {
9928
- const existing = await this.store.read(this.journalFile);
11105
+ const existing = await this.store.read(activeJournals.coordinate);
9929
11106
  this.journalInitialized = !!existing && existing.byteLength > 0;
9930
11107
  }
9931
11108
  coordinateBytes = this.journalInitialized
@@ -9934,28 +11111,26 @@ export class NativeBackboneCoordinatePersistence {
9934
11111
  backbone.coordinateJournalHeader(),
9935
11112
  coordinateRecords,
9936
11113
  ]);
9937
- await this.store.append(this.journalFile, coordinateBytes);
11114
+ await this.store.append(activeJournals.coordinate, coordinateBytes);
9938
11115
  persistenceMutationStarted = true;
9939
11116
  written += coordinateRecords.byteLength;
9940
11117
  }
9941
11118
  if (documentRecords.byteLength > 0) {
9942
11119
  if (this.documentJournalInitialized === undefined) {
9943
- const existing = await this.store.read(this.documentJournalFile);
11120
+ const existing = await this.store.read(activeJournals.document);
9944
11121
  this.documentJournalInitialized =
9945
11122
  !!existing && existing.byteLength > 0;
9946
11123
  }
9947
11124
  documentBytes = this.documentJournalInitialized
9948
11125
  ? documentRecords
9949
11126
  : concatBytes([backbone.documentJournalHeader(), documentRecords]);
9950
- await this.store.append(this.documentJournalFile, documentBytes);
11127
+ await this.store.append(activeJournals.document, documentBytes);
9951
11128
  persistenceMutationStarted = true;
9952
11129
  written += documentRecords.byteLength;
9953
11130
  }
9954
11131
  if (signerRecords.byteLength > 0) {
9955
11132
  if (this.documentSignerJournalInitialized === undefined) {
9956
- const existing = await this.store.read(
9957
- this.documentSignerJournalFile,
9958
- );
11133
+ const existing = await this.store.read(activeJournals.signer);
9959
11134
  this.documentSignerJournalInitialized =
9960
11135
  !!existing && existing.byteLength > 0;
9961
11136
  }
@@ -9965,11 +11140,11 @@ export class NativeBackboneCoordinatePersistence {
9965
11140
  backbone.documentSignerJournalHeader(),
9966
11141
  signerRecords,
9967
11142
  ]);
9968
- await this.store.append(this.documentSignerJournalFile, signerBytes);
11143
+ await this.store.append(activeJournals.signer, signerBytes);
9969
11144
  persistenceMutationStarted = true;
9970
11145
  written += signerRecords.byteLength;
9971
11146
  }
9972
- if (written === 0) {
11147
+ if (written === 0 && !checkpoint) {
9973
11148
  this.lastFlushMs = Date.now();
9974
11149
  return 0;
9975
11150
  }
@@ -9977,27 +11152,34 @@ export class NativeBackboneCoordinatePersistence {
9977
11152
  // `append` may only enqueue bytes in a buffered store. Drain and fsync
9978
11153
  // every affected WAL before clearing its wasm prefix or returning an ACK.
9979
11154
  for (const file of [
9980
- coordinateBytes ? this.journalFile : undefined,
9981
- documentBytes ? this.documentJournalFile : undefined,
9982
- signerBytes ? this.documentSignerJournalFile : undefined,
11155
+ coordinateBytes ? activeJournals.coordinate : undefined,
11156
+ documentBytes ? activeJournals.document : undefined,
11157
+ signerBytes ? activeJournals.signer : undefined,
9983
11158
  ]) {
9984
11159
  if (file) {
9985
- if (this.store.durableBarrier) {
9986
- await this.store.durableBarrier(file);
9987
- } else {
9988
- await this.store.flush?.(file);
9989
- }
11160
+ await this.barrierFile(file);
9990
11161
  }
9991
11162
  }
9992
11163
 
9993
11164
  if (coordinateBytes) {
9994
11165
  this.journalInitialized = true;
11166
+ this.journalByteLength += coordinateBytes.byteLength;
11167
+ this.journalRecordCount += coordinateRecordCount;
9995
11168
  }
9996
11169
  if (documentBytes) {
9997
11170
  this.documentJournalInitialized = true;
11171
+ this.documentJournalByteLength += documentBytes.byteLength;
11172
+ this.documentJournalRecordCount += documentRecordCount;
9998
11173
  }
9999
11174
  if (signerBytes) {
10000
11175
  this.documentSignerJournalInitialized = true;
11176
+ this.documentSignerJournalByteLength += signerBytes.byteLength;
11177
+ this.documentSignerJournalRecordCount += signerRecordCount;
11178
+ }
11179
+
11180
+ if (checkpoint) {
11181
+ persistenceMutationStarted = true;
11182
+ await this.publishCheckpoint(checkpoint);
10001
11183
  }
10002
11184
 
10003
11185
  backbone.clearCoordinateJournalPrefix(
@@ -10012,6 +11194,9 @@ export class NativeBackboneCoordinatePersistence {
10012
11194
  signerRecords.byteLength,
10013
11195
  signerRecordCount,
10014
11196
  );
11197
+ if (checkpoint) {
11198
+ await this.cleanupRetiredCheckpointFiles();
11199
+ }
10015
11200
  this.lastFlushMs = Date.now();
10016
11201
  return written;
10017
11202
  } catch (error) {
@@ -10023,9 +11208,15 @@ export class NativeBackboneCoordinatePersistence {
10023
11208
  }
10024
11209
  }
10025
11210
 
10026
- async compact(_backbone: NativePeerbitBackbone): Promise<void> {
11211
+ async compact(backbone: NativePeerbitBackbone): Promise<void> {
10027
11212
  this.assertActive("compact");
10028
- throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
11213
+ if (!this.crashSafeCompaction) {
11214
+ throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
11215
+ }
11216
+ await this.enqueuePersistence(async () => {
11217
+ this.assertPersistenceHealthy();
11218
+ await this.flushJournalInternal(backbone, true);
11219
+ });
10029
11220
  }
10030
11221
 
10031
11222
  close(): Promise<void> {