@peerbit/native-backbone 0.2.8 → 0.2.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -9,6 +9,94 @@ export * from "./durability/storage.js";
9
9
 
10
10
  export type RangeResolution = "u32" | "u64";
11
11
 
12
+ type NativePrepareNoNextDocumentIndexCompactTrimFacts = (
13
+ wallTime: bigint,
14
+ logical: number,
15
+ gid: string,
16
+ type: number,
17
+ metaData: Uint8Array | undefined,
18
+ payloadData: Uint8Array,
19
+ trimLengthTo: number,
20
+ documentKey: string,
21
+ documentValuePrefixBytes: Uint8Array,
22
+ documentExistingCreated: string,
23
+ documentByteElementIndexLimit: number,
24
+ documentDeleteTrimmedHeads: boolean,
25
+ documentProjectionPlan:
26
+ | NativeBackboneSimpleDocumentProjectionPlan
27
+ | undefined,
28
+ documentProjectionEncodedDocument: Uint8Array | undefined,
29
+ documentProjectionSigner: Uint8Array | undefined,
30
+ ) => unknown[];
31
+
32
+ type NativePrepareNoNextCachedDocumentIndexCompactTrimFacts = (
33
+ wallTime: bigint,
34
+ logical: number,
35
+ gid: string,
36
+ type: number,
37
+ metaData: Uint8Array | undefined,
38
+ payloadData: Uint8Array,
39
+ trimLengthTo: number,
40
+ documentKey: string,
41
+ documentExistingCreated: string,
42
+ documentByteElementIndexLimit: number,
43
+ documentDeleteTrimmedHeads: boolean,
44
+ documentProjectionPlanId: number,
45
+ documentProjectionEncodedDocument: Uint8Array,
46
+ documentProjectionSigner: Uint8Array | undefined,
47
+ ) => unknown[];
48
+
49
+ type NativePrepareNoNextCachedPlainPutPayloadDocumentIndexCompactTrimFacts = (
50
+ wallTime: bigint,
51
+ logical: number,
52
+ gid: string,
53
+ type: number,
54
+ metaData: Uint8Array | undefined,
55
+ payloadData: Uint8Array,
56
+ trimLengthTo: number,
57
+ documentKey: string,
58
+ documentExistingCreated: string,
59
+ documentByteElementIndexLimit: number,
60
+ documentDeleteTrimmedHeads: boolean,
61
+ documentProjectionPlanId: number,
62
+ documentProjectionSigner: Uint8Array | undefined,
63
+ ) => unknown[];
64
+
65
+ type NativePrepareLatestDocumentIndexTrimFacts = (
66
+ wallTime: bigint,
67
+ logical: number,
68
+ gid: string,
69
+ type: number,
70
+ metaData: Uint8Array | undefined,
71
+ payloadData: Uint8Array,
72
+ trimLengthTo: number | undefined,
73
+ documentKey: string,
74
+ documentValuePrefixBytes: Uint8Array,
75
+ documentByteElementIndexLimit: number,
76
+ documentDeleteTrimmedHeads: boolean,
77
+ documentProjectionPlan:
78
+ | NativeBackboneSimpleDocumentProjectionPlan
79
+ | undefined,
80
+ documentProjectionEncodedDocument: Uint8Array | undefined,
81
+ documentProjectionSigner: Uint8Array | undefined,
82
+ ) => unknown[];
83
+
84
+ type NativePrepareLatestCachedDocumentIndexTrimFacts = (
85
+ wallTime: bigint,
86
+ logical: number,
87
+ gid: string,
88
+ type: number,
89
+ metaData: Uint8Array | undefined,
90
+ payloadData: Uint8Array,
91
+ trimLengthTo: number | undefined,
92
+ documentKey: string,
93
+ documentByteElementIndexLimit: number,
94
+ documentDeleteTrimmedHeads: boolean,
95
+ documentProjectionPlanId: number,
96
+ documentProjectionEncodedDocument: Uint8Array,
97
+ documentProjectionSigner: Uint8Array | undefined,
98
+ ) => unknown[];
99
+
12
100
  type NativePeerbitBackboneHandle = {
13
101
  log_len: () => number;
14
102
  block_len: () => number;
@@ -942,6 +1030,16 @@ type NativePeerbitBackboneHandle = {
942
1030
  payloadData: Uint8Array,
943
1031
  trimLengthTo: number | undefined,
944
1032
  ) => unknown[];
1033
+ prepare_plain_entry_commit_facts_trim_refs?: (
1034
+ wallTime: bigint,
1035
+ logical: number,
1036
+ gid: string,
1037
+ next: string[],
1038
+ type: number,
1039
+ metaData: Uint8Array | undefined,
1040
+ payloadData: Uint8Array,
1041
+ trimLengthTo: number,
1042
+ ) => unknown[];
945
1043
  prepare_plain_entry_commit_facts_document_index: (
946
1044
  wallTime: bigint,
947
1045
  logical: number,
@@ -1045,25 +1143,8 @@ type NativePeerbitBackboneHandle = {
1045
1143
  documentProjectionEncodedDocument: Uint8Array | undefined,
1046
1144
  documentProjectionSigner: Uint8Array | undefined,
1047
1145
  ) => unknown[];
1048
- prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_hashes: (
1049
- wallTime: bigint,
1050
- logical: number,
1051
- gid: string,
1052
- type: number,
1053
- metaData: Uint8Array | undefined,
1054
- payloadData: Uint8Array,
1055
- trimLengthTo: number,
1056
- documentKey: string,
1057
- documentValuePrefixBytes: Uint8Array,
1058
- documentExistingCreated: string,
1059
- documentByteElementIndexLimit: number,
1060
- documentDeleteTrimmedHeads: boolean,
1061
- documentProjectionPlan:
1062
- | NativeBackboneSimpleDocumentProjectionPlan
1063
- | undefined,
1064
- documentProjectionEncodedDocument: Uint8Array | undefined,
1065
- documentProjectionSigner: Uint8Array | undefined,
1066
- ) => unknown[];
1146
+ prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_hashes: NativePrepareNoNextDocumentIndexCompactTrimFacts;
1147
+ prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_refs?: NativePrepareNoNextDocumentIndexCompactTrimFacts;
1067
1148
  prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_trim_hashes: (
1068
1149
  wallTime: bigint,
1069
1150
  logical: number,
@@ -1080,70 +1161,14 @@ type NativePeerbitBackboneHandle = {
1080
1161
  documentProjectionEncodedDocument: Uint8Array,
1081
1162
  documentProjectionSigner: Uint8Array | undefined,
1082
1163
  ) => unknown[];
1083
- prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes: (
1084
- wallTime: bigint,
1085
- logical: number,
1086
- gid: string,
1087
- type: number,
1088
- metaData: Uint8Array | undefined,
1089
- payloadData: Uint8Array,
1090
- trimLengthTo: number,
1091
- documentKey: string,
1092
- documentExistingCreated: string,
1093
- documentByteElementIndexLimit: number,
1094
- documentDeleteTrimmedHeads: boolean,
1095
- documentProjectionPlanId: number,
1096
- documentProjectionEncodedDocument: Uint8Array,
1097
- documentProjectionSigner: Uint8Array | undefined,
1098
- ) => unknown[];
1099
- prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes_plain_put_payload?: (
1100
- wallTime: bigint,
1101
- logical: number,
1102
- gid: string,
1103
- type: number,
1104
- metaData: Uint8Array | undefined,
1105
- payloadData: Uint8Array,
1106
- trimLengthTo: number,
1107
- documentKey: string,
1108
- documentExistingCreated: string,
1109
- documentByteElementIndexLimit: number,
1110
- documentDeleteTrimmedHeads: boolean,
1111
- documentProjectionPlanId: number,
1112
- documentProjectionSigner: Uint8Array | undefined,
1113
- ) => unknown[];
1114
- prepare_plain_entry_commit_latest_facts_document_index_trim_hashes: (
1115
- wallTime: bigint,
1116
- logical: number,
1117
- gid: string,
1118
- type: number,
1119
- metaData: Uint8Array | undefined,
1120
- payloadData: Uint8Array,
1121
- trimLengthTo: number | undefined,
1122
- documentKey: string,
1123
- documentValuePrefixBytes: Uint8Array,
1124
- documentByteElementIndexLimit: number,
1125
- documentDeleteTrimmedHeads: boolean,
1126
- documentProjectionPlan:
1127
- | NativeBackboneSimpleDocumentProjectionPlan
1128
- | undefined,
1129
- documentProjectionEncodedDocument: Uint8Array | undefined,
1130
- documentProjectionSigner: Uint8Array | undefined,
1131
- ) => unknown[];
1132
- prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes: (
1133
- wallTime: bigint,
1134
- logical: number,
1135
- gid: string,
1136
- type: number,
1137
- metaData: Uint8Array | undefined,
1138
- payloadData: Uint8Array,
1139
- trimLengthTo: number | undefined,
1140
- documentKey: string,
1141
- documentByteElementIndexLimit: number,
1142
- documentDeleteTrimmedHeads: boolean,
1143
- documentProjectionPlanId: number,
1144
- documentProjectionEncodedDocument: Uint8Array,
1145
- documentProjectionSigner: Uint8Array | undefined,
1146
- ) => unknown[];
1164
+ prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes: NativePrepareNoNextCachedDocumentIndexCompactTrimFacts;
1165
+ prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs?: NativePrepareNoNextCachedDocumentIndexCompactTrimFacts;
1166
+ prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes_plain_put_payload?: NativePrepareNoNextCachedPlainPutPayloadDocumentIndexCompactTrimFacts;
1167
+ prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs_plain_put_payload?: NativePrepareNoNextCachedPlainPutPayloadDocumentIndexCompactTrimFacts;
1168
+ prepare_plain_entry_commit_latest_facts_document_index_trim_hashes: NativePrepareLatestDocumentIndexTrimFacts;
1169
+ prepare_plain_entry_commit_latest_facts_document_index_trim_refs?: NativePrepareLatestDocumentIndexTrimFacts;
1170
+ prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes: NativePrepareLatestCachedDocumentIndexTrimFacts;
1171
+ prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_refs?: NativePrepareLatestCachedDocumentIndexTrimFacts;
1147
1172
  prepare_plain_entry_storage_facts_and_put: (
1148
1173
  wallTime: bigint,
1149
1174
  logical: number,
@@ -2727,6 +2752,7 @@ export type NativeBackboneAppendResult = {
2727
2752
  assignedToRangeBoundary: boolean;
2728
2753
  trimmed: NativeBackboneTrimmedEntry[];
2729
2754
  trimmedHashes?: string[];
2755
+ trimmedGids?: string[];
2730
2756
  documentTrimmedHeadsProcessed?: boolean;
2731
2757
  documentPreviousContext?: NativeBackboneDocumentContextFacts;
2732
2758
  };
@@ -2739,6 +2765,7 @@ type NativeBackboneStorageAppendResult = {
2739
2765
  assignedToRangeBoundary: boolean;
2740
2766
  trimmed: NativeBackboneTrimmedEntry[];
2741
2767
  trimmedHashes?: string[];
2768
+ trimmedGids?: string[];
2742
2769
  documentTrimmedHeadsProcessed?: boolean;
2743
2770
  documentPreviousContext?: NativeBackboneDocumentContextFacts;
2744
2771
  };
@@ -2973,7 +3000,21 @@ export type NativeBackboneCoordinatePersistenceOptions =
2973
3000
 
2974
3001
  export type NativeBackboneCoordinatePersistenceStore = {
2975
3002
  read(name: string): Promise<Uint8Array | undefined>;
3003
+ /**
3004
+ * Read a named file without materializing more than `maxBytes`.
3005
+ *
3006
+ * The optional method is an explicit capability: callers that require a
3007
+ * pre-allocation bound must fail closed when it is absent and must not fall
3008
+ * back to `read`.
3009
+ */
3010
+ readLimited?(name: string, maxBytes: number): Promise<Uint8Array | undefined>;
2976
3011
  write(name: string, bytes: Uint8Array): Promise<void>;
3012
+ /**
3013
+ * Durably replace one named file without exposing a torn destination. The
3014
+ * replacement is not visible until its bytes and containing directory have
3015
+ * crossed their physical durability barriers.
3016
+ */
3017
+ atomicReplace?(name: string, bytes: Uint8Array): Promise<void>;
2977
3018
  /**
2978
3019
  * Appends `bytes` to the named file. Callers hand over ownership of
2979
3020
  * `bytes` and must not mutate it afterwards: buffering stores keep the
@@ -2991,6 +3032,21 @@ export type NativeBackboneCoordinatePersistenceStore = {
2991
3032
  close?(options?: { flush?: boolean }): Promise<void>;
2992
3033
  };
2993
3034
 
3035
+ export class NativeBackboneCoordinatePersistenceReadLimitError extends Error {
3036
+ readonly code = "ERR_NATIVE_BACKBONE_COORDINATE_READ_LIMIT";
3037
+
3038
+ constructor(
3039
+ readonly file: string,
3040
+ readonly maxBytes: number,
3041
+ readonly observedBytes: bigint,
3042
+ ) {
3043
+ super(
3044
+ `Native backbone coordinate persistence file ${file} exceeds the ${maxBytes} byte read limit (${observedBytes.toString()} bytes)`,
3045
+ );
3046
+ this.name = "NativeBackboneCoordinatePersistenceReadLimitError";
3047
+ }
3048
+ }
3049
+
2994
3050
  export type NativeBackboneCoordinatePersistenceAdapter = {
2995
3051
  /**
2996
3052
  * Optional durable capability used by higher layers for atomic operation
@@ -3074,12 +3130,12 @@ export const nativeBackboneCoordinateDropTombstoneFile =
3074
3130
  "native-backbone-drop.tombstone";
3075
3131
 
3076
3132
  export const defaultNativeBackboneCoordinateFlushMaxPendingBytes = 1024 * 1024;
3077
- /** @deprecated Built-in coordinate persistence compaction is currently disabled. */
3133
+ /** Recommended explicit byte threshold for crash-safe Node checkpointing. */
3078
3134
  export const defaultNativeBackboneCoordinateCompactMaxJournalBytes =
3079
3135
  64 * 1024 * 1024;
3080
3136
 
3081
3137
  const nativeBackboneCoordinateCompactionDisabledMessage =
3082
- "Native backbone coordinate persistence compaction is disabled until snapshots use a crash-safe generation protocol";
3138
+ "Native backbone coordinate persistence compaction is disabled for stores without crash-safe atomic replacement";
3083
3139
 
3084
3140
  const nativeBackboneCoordinateJournalMagic = Uint8Array.from([
3085
3141
  0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x57, 0x31,
@@ -3100,6 +3156,260 @@ const coordinateJournalChecksum = (bytes: Uint8Array): number => {
3100
3156
  return checksum;
3101
3157
  };
3102
3158
 
3159
+ type NativeBackboneCoordinateCheckpointSlot = "a" | "b";
3160
+
3161
+ type NativeBackboneCoordinateCheckpointAuthority = {
3162
+ generation: bigint;
3163
+ byteLength: number;
3164
+ checksum: number;
3165
+ };
3166
+
3167
+ type NativeBackboneCoordinateCheckpointState = {
3168
+ configurationChecksum: number;
3169
+ highwater: bigint;
3170
+ pending?: NativeBackboneCoordinateCheckpointAuthority;
3171
+ completed?: NativeBackboneCoordinateCheckpointAuthority;
3172
+ };
3173
+
3174
+ type NativeBackboneCoordinateCheckpoint = {
3175
+ configurationChecksum: number;
3176
+ generation: bigint;
3177
+ coordinateSnapshot: Uint8Array;
3178
+ documentSnapshot: Uint8Array;
3179
+ documentSignerSnapshot: Uint8Array;
3180
+ };
3181
+
3182
+ const nativeBackboneCoordinateCheckpointMagic = Uint8Array.from([
3183
+ 0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x43, 0x31,
3184
+ ]);
3185
+ const nativeBackboneCoordinateCheckpointStateMagic = Uint8Array.from([
3186
+ 0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x53, 0x31,
3187
+ ]);
3188
+ const nativeBackboneCoordinateLegacyMigrationSentinel = (() => {
3189
+ const payload = Uint8Array.of(0xff);
3190
+ const bytes = new Uint8Array(
3191
+ nativeBackboneCoordinateJournalMagic.byteLength + 8 + payload.byteLength,
3192
+ );
3193
+ bytes.set(nativeBackboneCoordinateJournalMagic, 0);
3194
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
3195
+ view.setUint32(nativeBackboneCoordinateJournalMagic.byteLength, 1, true);
3196
+ view.setUint32(
3197
+ nativeBackboneCoordinateJournalMagic.byteLength + 4,
3198
+ coordinateJournalChecksum(payload),
3199
+ true,
3200
+ );
3201
+ bytes.set(payload, nativeBackboneCoordinateJournalMagic.byteLength + 8);
3202
+ return bytes;
3203
+ })();
3204
+ const nativeBackboneCoordinateCheckpointVersion = 1;
3205
+ const nativeBackboneCoordinateCheckpointHeaderBytes = 56;
3206
+ const nativeBackboneCoordinateCheckpointStateBytes = 60;
3207
+ const nativeBackboneCoordinateMaxU64 = (1n << 64n) - 1n;
3208
+
3209
+ const hasBytesPrefix = (bytes: Uint8Array, prefix: Uint8Array): boolean =>
3210
+ bytes.byteLength >= prefix.byteLength &&
3211
+ prefix.every((byte, index) => bytes[index] === byte);
3212
+
3213
+ const writeCoordinateCheckpointU64 = (
3214
+ view: DataView,
3215
+ offset: number,
3216
+ value: bigint,
3217
+ ): void => {
3218
+ if (value < 0n || value > nativeBackboneCoordinateMaxU64) {
3219
+ throw new RangeError("Native backbone checkpoint generation exceeds u64");
3220
+ }
3221
+ view.setUint32(offset, Number(value & 0xffff_ffffn), true);
3222
+ view.setUint32(offset + 4, Number(value >> 32n), true);
3223
+ };
3224
+
3225
+ const readCoordinateCheckpointU64 = (view: DataView, offset: number): bigint =>
3226
+ BigInt(view.getUint32(offset, true)) |
3227
+ (BigInt(view.getUint32(offset + 4, true)) << 32n);
3228
+
3229
+ const encodeNativeBackboneCoordinateCheckpoint = (
3230
+ checkpoint: NativeBackboneCoordinateCheckpoint,
3231
+ ): Uint8Array => {
3232
+ const coordinateLength = checkpoint.coordinateSnapshot.byteLength;
3233
+ const documentLength = checkpoint.documentSnapshot.byteLength;
3234
+ const signerLength = checkpoint.documentSignerSnapshot.byteLength;
3235
+ const byteLength =
3236
+ nativeBackboneCoordinateCheckpointHeaderBytes +
3237
+ coordinateLength +
3238
+ documentLength +
3239
+ signerLength;
3240
+ if (!Number.isSafeInteger(byteLength) || byteLength > 0xffff_ffff) {
3241
+ throw new RangeError("Native backbone coordinate checkpoint is too large");
3242
+ }
3243
+ const bytes = new Uint8Array(byteLength);
3244
+ bytes.set(nativeBackboneCoordinateCheckpointMagic, 0);
3245
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
3246
+ view.setUint32(8, nativeBackboneCoordinateCheckpointVersion, true);
3247
+ view.setUint32(12, nativeBackboneCoordinateCheckpointHeaderBytes, true);
3248
+ writeCoordinateCheckpointU64(view, 16, checkpoint.generation);
3249
+ view.setUint32(24, coordinateLength, true);
3250
+ view.setUint32(28, documentLength, true);
3251
+ view.setUint32(32, signerLength, true);
3252
+ view.setUint32(
3253
+ 36,
3254
+ coordinateJournalChecksum(checkpoint.coordinateSnapshot),
3255
+ true,
3256
+ );
3257
+ view.setUint32(
3258
+ 40,
3259
+ coordinateJournalChecksum(checkpoint.documentSnapshot),
3260
+ true,
3261
+ );
3262
+ view.setUint32(
3263
+ 44,
3264
+ coordinateJournalChecksum(checkpoint.documentSignerSnapshot),
3265
+ true,
3266
+ );
3267
+ view.setUint32(48, checkpoint.configurationChecksum, true);
3268
+ view.setUint32(52, coordinateJournalChecksum(bytes.subarray(8, 52)), true);
3269
+ let offset = nativeBackboneCoordinateCheckpointHeaderBytes;
3270
+ bytes.set(checkpoint.coordinateSnapshot, offset);
3271
+ offset += coordinateLength;
3272
+ bytes.set(checkpoint.documentSnapshot, offset);
3273
+ offset += documentLength;
3274
+ bytes.set(checkpoint.documentSignerSnapshot, offset);
3275
+ return bytes;
3276
+ };
3277
+
3278
+ const parseNativeBackboneCoordinateCheckpoint = (
3279
+ bytes: Uint8Array,
3280
+ name: string,
3281
+ ): NativeBackboneCoordinateCheckpoint => {
3282
+ if (
3283
+ bytes.byteLength < nativeBackboneCoordinateCheckpointHeaderBytes ||
3284
+ !hasBytesPrefix(bytes, nativeBackboneCoordinateCheckpointMagic)
3285
+ ) {
3286
+ throw new Error(`Native backbone ${name} has an invalid checkpoint header`);
3287
+ }
3288
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
3289
+ if (
3290
+ view.getUint32(8, true) !== nativeBackboneCoordinateCheckpointVersion ||
3291
+ view.getUint32(12, true) !==
3292
+ nativeBackboneCoordinateCheckpointHeaderBytes ||
3293
+ view.getUint32(52, true) !==
3294
+ coordinateJournalChecksum(bytes.subarray(8, 52))
3295
+ ) {
3296
+ throw new Error(`Native backbone ${name} has invalid checkpoint metadata`);
3297
+ }
3298
+ const coordinateLength = view.getUint32(24, true);
3299
+ const documentLength = view.getUint32(28, true);
3300
+ const signerLength = view.getUint32(32, true);
3301
+ const expectedLength =
3302
+ nativeBackboneCoordinateCheckpointHeaderBytes +
3303
+ coordinateLength +
3304
+ documentLength +
3305
+ signerLength;
3306
+ if (
3307
+ !Number.isSafeInteger(expectedLength) ||
3308
+ expectedLength !== bytes.byteLength
3309
+ ) {
3310
+ throw new Error(`Native backbone ${name} has an invalid checkpoint length`);
3311
+ }
3312
+ let offset = nativeBackboneCoordinateCheckpointHeaderBytes;
3313
+ const coordinateSnapshot = bytes.subarray(offset, offset + coordinateLength);
3314
+ offset += coordinateLength;
3315
+ const documentSnapshot = bytes.subarray(offset, offset + documentLength);
3316
+ offset += documentLength;
3317
+ const documentSignerSnapshot = bytes.subarray(offset, offset + signerLength);
3318
+ if (
3319
+ coordinateJournalChecksum(coordinateSnapshot) !==
3320
+ view.getUint32(36, true) ||
3321
+ coordinateJournalChecksum(documentSnapshot) !== view.getUint32(40, true) ||
3322
+ coordinateJournalChecksum(documentSignerSnapshot) !==
3323
+ view.getUint32(44, true)
3324
+ ) {
3325
+ throw new Error(
3326
+ `Native backbone ${name} has a checkpoint checksum mismatch`,
3327
+ );
3328
+ }
3329
+ return {
3330
+ configurationChecksum: view.getUint32(48, true),
3331
+ generation: readCoordinateCheckpointU64(view, 16),
3332
+ coordinateSnapshot,
3333
+ documentSnapshot,
3334
+ documentSignerSnapshot,
3335
+ };
3336
+ };
3337
+
3338
+ const encodeNativeBackboneCoordinateCheckpointState = (
3339
+ state: NativeBackboneCoordinateCheckpointState,
3340
+ ): Uint8Array => {
3341
+ const bytes = new Uint8Array(nativeBackboneCoordinateCheckpointStateBytes);
3342
+ bytes.set(nativeBackboneCoordinateCheckpointStateMagic, 0);
3343
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
3344
+ view.setUint32(8, nativeBackboneCoordinateCheckpointVersion, true);
3345
+ view.setUint32(12, state.configurationChecksum, true);
3346
+ writeCoordinateCheckpointU64(view, 16, state.highwater);
3347
+ writeCoordinateCheckpointU64(view, 24, state.pending?.generation ?? 0n);
3348
+ view.setUint32(32, state.pending?.byteLength ?? 0, true);
3349
+ view.setUint32(36, state.pending?.checksum ?? 0, true);
3350
+ writeCoordinateCheckpointU64(view, 40, state.completed?.generation ?? 0n);
3351
+ view.setUint32(48, state.completed?.byteLength ?? 0, true);
3352
+ view.setUint32(52, state.completed?.checksum ?? 0, true);
3353
+ view.setUint32(56, coordinateJournalChecksum(bytes.subarray(8, 56)), true);
3354
+ return bytes;
3355
+ };
3356
+
3357
+ const parseNativeBackboneCoordinateCheckpointState = (
3358
+ bytes: Uint8Array,
3359
+ name: string,
3360
+ ): NativeBackboneCoordinateCheckpointState => {
3361
+ if (
3362
+ bytes.byteLength !== nativeBackboneCoordinateCheckpointStateBytes ||
3363
+ !hasBytesPrefix(bytes, nativeBackboneCoordinateCheckpointStateMagic)
3364
+ ) {
3365
+ throw new Error(
3366
+ `Native backbone ${name} has an invalid checkpoint authority`,
3367
+ );
3368
+ }
3369
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
3370
+ if (
3371
+ view.getUint32(8, true) !== nativeBackboneCoordinateCheckpointVersion ||
3372
+ view.getUint32(56, true) !==
3373
+ coordinateJournalChecksum(bytes.subarray(8, 56))
3374
+ ) {
3375
+ throw new Error(`Native backbone ${name} has corrupt checkpoint authority`);
3376
+ }
3377
+ const configurationChecksum = view.getUint32(12, true);
3378
+ const highwater = readCoordinateCheckpointU64(view, 16);
3379
+ const pendingGeneration = readCoordinateCheckpointU64(view, 24);
3380
+ const completedGeneration = readCoordinateCheckpointU64(view, 40);
3381
+ const authority = (
3382
+ generation: bigint,
3383
+ lengthOffset: number,
3384
+ checksumOffset: number,
3385
+ ): NativeBackboneCoordinateCheckpointAuthority | undefined => {
3386
+ const byteLength = view.getUint32(lengthOffset, true);
3387
+ const checksum = view.getUint32(checksumOffset, true);
3388
+ if (generation === 0n) {
3389
+ if (byteLength !== 0 || checksum !== 0) {
3390
+ throw new Error(`Native backbone ${name} has empty authority metadata`);
3391
+ }
3392
+ return undefined;
3393
+ }
3394
+ if (byteLength < nativeBackboneCoordinateCheckpointHeaderBytes) {
3395
+ throw new Error(`Native backbone ${name} has invalid authority length`);
3396
+ }
3397
+ return { generation, byteLength, checksum };
3398
+ };
3399
+ const pending = authority(pendingGeneration, 32, 36);
3400
+ const completed = authority(completedGeneration, 48, 52);
3401
+ if (
3402
+ (completed && completed.generation > highwater) ||
3403
+ (pending && pending.generation !== highwater) ||
3404
+ (pending && completed && pending.generation <= completed.generation) ||
3405
+ (!pending && completed && completed.generation !== highwater) ||
3406
+ (!pending && !completed && highwater !== 0n)
3407
+ ) {
3408
+ throw new Error(`Native backbone ${name} has invalid generation authority`);
3409
+ }
3410
+ return { configurationChecksum, highwater, pending, completed };
3411
+ };
3412
+
3103
3413
  const hasCoordinateJournalMagic = (bytes: Uint8Array): boolean =>
3104
3414
  bytes.byteLength >= nativeBackboneCoordinateJournalMagic.byteLength &&
3105
3415
  nativeBackboneCoordinateJournalMagic.every(
@@ -3149,11 +3459,28 @@ const resolveCoordinateFlushMaxPendingBytes = (
3149
3459
  ? defaultNativeBackboneCoordinateFlushMaxPendingBytes
3150
3460
  : undefined;
3151
3461
 
3462
+ const validateCoordinateCompactionThreshold = (
3463
+ value: number,
3464
+ name: string,
3465
+ ): number => {
3466
+ if (!Number.isSafeInteger(value) || value <= 0) {
3467
+ throw new RangeError(
3468
+ `Native backbone ${name} must be a positive safe integer`,
3469
+ );
3470
+ }
3471
+ return value;
3472
+ };
3473
+
3152
3474
  type NativeBackboneNodeFs = {
3153
3475
  mkdir(path: string, options?: { recursive?: boolean }): Promise<unknown>;
3154
3476
  readFile(path: string): Promise<Uint8Array>;
3155
3477
  writeFile(path: string, data: Uint8Array): Promise<unknown>;
3156
3478
  appendFile(path: string, data: Uint8Array): Promise<unknown>;
3479
+ rename?(from: string, to: string): Promise<unknown>;
3480
+ /** Explicit capability for pre-allocation-bounded positional reads. */
3481
+ openBoundedRead?(
3482
+ path: string,
3483
+ ): Promise<NativeBackboneNodeBoundedReadFileHandle>;
3157
3484
  open?(
3158
3485
  path: string,
3159
3486
  flags: string,
@@ -3163,10 +3490,28 @@ type NativeBackboneNodeFs = {
3163
3490
 
3164
3491
  type NativeBackboneNodeAppendFileHandle = {
3165
3492
  write(data: Uint8Array): Promise<{ bytesWritten: number }>;
3493
+ read?(
3494
+ buffer: Uint8Array,
3495
+ offset: number,
3496
+ length: number,
3497
+ position: number,
3498
+ ): Promise<{ bytesRead: number }>;
3499
+ stat?(options: { bigint: true }): Promise<{ size: bigint }>;
3166
3500
  sync?(): Promise<unknown>;
3167
3501
  close(): Promise<unknown>;
3168
3502
  };
3169
3503
 
3504
+ type NativeBackboneNodeBoundedReadFileHandle = {
3505
+ read(
3506
+ buffer: Uint8Array,
3507
+ offset: number,
3508
+ length: number,
3509
+ position: number,
3510
+ ): Promise<{ bytesRead: number }>;
3511
+ stat(options: { bigint: true }): Promise<{ size: bigint }>;
3512
+ close(): Promise<unknown>;
3513
+ };
3514
+
3170
3515
  type NativeBackboneOPFSFile = {
3171
3516
  arrayBuffer(): Promise<ArrayBuffer>;
3172
3517
  size: number;
@@ -3274,6 +3619,31 @@ const validateCoordinatePersistenceName = (name: string): string => {
3274
3619
  return name;
3275
3620
  };
3276
3621
 
3622
+ const validateCoordinatePersistenceReadMaxBytes = (
3623
+ maxBytes: number,
3624
+ ): number => {
3625
+ if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) {
3626
+ throw new RangeError(
3627
+ "Native backbone coordinate persistence read limit must be a non-negative safe integer",
3628
+ );
3629
+ }
3630
+ return maxBytes;
3631
+ };
3632
+
3633
+ const assertCoordinatePersistenceReadWithinLimit = (
3634
+ name: string,
3635
+ maxBytes: number,
3636
+ observedBytes: number | bigint,
3637
+ ): void => {
3638
+ if (BigInt(observedBytes) > BigInt(maxBytes)) {
3639
+ throw new NativeBackboneCoordinatePersistenceReadLimitError(
3640
+ name,
3641
+ maxBytes,
3642
+ BigInt(observedBytes),
3643
+ );
3644
+ }
3645
+ };
3646
+
3277
3647
  type NativeBackboneCoordinateDropTombstoneBody = {
3278
3648
  format: "peerbit-native-backbone-coordinate-drop";
3279
3649
  version: 1;
@@ -3877,6 +4247,8 @@ const storageAppendResultFromRow = (
3877
4247
  trimHashRows,
3878
4248
  documentTrimmedHeadsProcessed,
3879
4249
  documentPreviousContextRow,
4250
+ _trimGidExtension,
4251
+ trimGidRows,
3880
4252
  ] = row as [
3881
4253
  unknown[],
3882
4254
  unknown[] | undefined,
@@ -3887,6 +4259,8 @@ const storageAppendResultFromRow = (
3887
4259
  unknown[] | undefined,
3888
4260
  boolean | undefined,
3889
4261
  unknown[] | undefined,
4262
+ undefined,
4263
+ string[] | undefined,
3890
4264
  ];
3891
4265
  return {
3892
4266
  entry: storageFactsEntryFromRow(entryRow),
@@ -3895,6 +4269,8 @@ const storageAppendResultFromRow = (
3895
4269
  assignedToRangeBoundary,
3896
4270
  coordinate: appendCoordinatePlanFromRow(resolution, coordinateRow),
3897
4271
  ...trimmedRowsAndHashesResult(trimRows, trimHashRows),
4272
+ trimmedGids:
4273
+ trimGidRows ?? (trimRows as unknown[][]).map((trim) => trim[1] as string),
3898
4274
  documentTrimmedHeadsProcessed,
3899
4275
  documentPreviousContext: documentContextFactsFromRow(
3900
4276
  documentPreviousContextRow,
@@ -3916,6 +4292,8 @@ const committedStorageAppendResultFromRow = (
3916
4292
  trimHashRows,
3917
4293
  documentTrimmedHeadsProcessed,
3918
4294
  documentPreviousContextRow,
4295
+ _trimGidExtension,
4296
+ trimGidRows,
3919
4297
  ] = row as [
3920
4298
  unknown[],
3921
4299
  unknown[] | undefined,
@@ -3926,6 +4304,8 @@ const committedStorageAppendResultFromRow = (
3926
4304
  unknown[] | undefined,
3927
4305
  boolean | undefined,
3928
4306
  unknown[] | undefined,
4307
+ undefined,
4308
+ string[] | undefined,
3929
4309
  ];
3930
4310
  return {
3931
4311
  entry: committedStorageFactsEntryFromRow(entryRow),
@@ -3934,6 +4314,8 @@ const committedStorageAppendResultFromRow = (
3934
4314
  assignedToRangeBoundary,
3935
4315
  coordinate: appendCoordinatePlanFromRow(resolution, coordinateRow),
3936
4316
  ...trimmedRowsAndHashesResult(trimRows, trimHashRows),
4317
+ trimmedGids:
4318
+ trimGidRows ?? (trimRows as unknown[][]).map((trim) => trim[1] as string),
3937
4319
  documentTrimmedHeadsProcessed,
3938
4320
  documentPreviousContext: documentContextFactsFromRow(
3939
4321
  documentPreviousContextRow,
@@ -3945,7 +4327,14 @@ const compactCommittedNoNextStorageAppendResultFromRow = (
3945
4327
  resolution: RangeResolution,
3946
4328
  row: unknown[],
3947
4329
  ): NativeBackboneAppendResult => {
3948
- const [hash, byteLength, metaBytes, fourth] = row as [
4330
+ const appendedTrimGids =
4331
+ row.length >= 12 &&
4332
+ row[row.length - 2] === undefined &&
4333
+ Array.isArray(row[row.length - 1])
4334
+ ? (row[row.length - 1] as string[])
4335
+ : undefined;
4336
+ const baseRow = appendedTrimGids ? row.slice(0, -2) : row;
4337
+ const [hash, byteLength, metaBytes, fourth] = baseRow as [
3949
4338
  string,
3950
4339
  number,
3951
4340
  Uint8Array | undefined,
@@ -3953,7 +4342,7 @@ const compactCommittedNoNextStorageAppendResultFromRow = (
3953
4342
  ];
3954
4343
  const hasDigestRow = fourth instanceof Uint8Array;
3955
4344
  const hashDigestBytes = hasDigestRow ? (fourth as Uint8Array) : undefined;
3956
- const rest = row.slice(hasDigestRow ? 4 : 3);
4345
+ const rest = baseRow.slice(hasDigestRow ? 4 : 3);
3957
4346
  const usesNestedCoordinateRow = Array.isArray(rest[0]);
3958
4347
  let coordinate: NativeBackboneCoordinatePlan;
3959
4348
  let leaderRows: unknown[] | undefined;
@@ -4036,6 +4425,7 @@ const compactCommittedNoNextStorageAppendResultFromRow = (
4036
4425
  coordinate,
4037
4426
  trimmed: [],
4038
4427
  trimmedHashes: trimHashRows ?? [],
4428
+ trimmedGids: appendedTrimGids,
4039
4429
  documentTrimmedHeadsProcessed,
4040
4430
  };
4041
4431
  };
@@ -4044,6 +4434,13 @@ const compactCommittedLatestStorageAppendResultFromRow = (
4044
4434
  resolution: RangeResolution,
4045
4435
  row: unknown[],
4046
4436
  ): NativeBackboneAppendResult => {
4437
+ const appendedTrimGids =
4438
+ row.length >= 13 &&
4439
+ row[row.length - 2] === undefined &&
4440
+ Array.isArray(row[row.length - 1])
4441
+ ? (row[row.length - 1] as string[])
4442
+ : undefined;
4443
+ const baseRow = appendedTrimGids ? row.slice(0, -2) : row;
4047
4444
  const [
4048
4445
  hash,
4049
4446
  byteLength,
@@ -4057,7 +4454,7 @@ const compactCommittedLatestStorageAppendResultFromRow = (
4057
4454
  tenth,
4058
4455
  eleventh,
4059
4456
  twelfth,
4060
- ] = row as [
4457
+ ] = baseRow as [
4061
4458
  string,
4062
4459
  number,
4063
4460
  Uint8Array | undefined,
@@ -4102,6 +4499,7 @@ const compactCommittedLatestStorageAppendResultFromRow = (
4102
4499
  coordinate,
4103
4500
  trimmed: [],
4104
4501
  trimmedHashes: trimHashRows ?? [],
4502
+ trimmedGids: appendedTrimGids,
4105
4503
  documentTrimmedHeadsProcessed,
4106
4504
  documentPreviousContext: documentContextFactsFromRow(
4107
4505
  documentPreviousContextRow,
@@ -4130,6 +4528,20 @@ const preparedCommitFactsFromRow = (
4130
4528
  return prepared;
4131
4529
  };
4132
4530
 
4531
+ const preparedCommitFactsWithTrimRefsFromRow = (
4532
+ row: unknown[],
4533
+ ): NativeBackboneCommittedEntry & {
4534
+ trimmedEntryHashes: string[];
4535
+ trimmedEntryGids: string[];
4536
+ } => {
4537
+ const prepared = committedStorageFactsEntryFromRow(row[0] as unknown[]);
4538
+ return {
4539
+ ...prepared,
4540
+ trimmedEntryHashes: row[1] as string[],
4541
+ trimmedEntryGids: row[2] as string[],
4542
+ };
4543
+ };
4544
+
4133
4545
  const preparedCommitFactsWithLatestDocumentContextFromRow = (
4134
4546
  row: unknown[],
4135
4547
  ): NativeBackboneCommittedEntry & {
@@ -4152,12 +4564,39 @@ const preparedCommitFactsWithLatestDocumentContextFromRow = (
4152
4564
  };
4153
4565
  };
4154
4566
 
4155
- const compactPreparedCommitFactsWithTrimHashesFromRow = (
4567
+ const preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow = (
4156
4568
  row: unknown[],
4157
4569
  ): NativeBackboneCommittedEntry & {
4158
- trimmedEntryHashes?: string[];
4570
+ trimmedEntryHashes: string[];
4571
+ trimmedEntryGids: string[];
4159
4572
  documentTrimmedHeadsProcessed?: boolean;
4573
+ documentPreviousContext?: NativeBackboneDocumentContextFacts;
4160
4574
  } => {
4575
+ const [
4576
+ entryRow,
4577
+ trimHashRows,
4578
+ trimGidRows,
4579
+ documentTrimmedHeadsProcessed,
4580
+ contextRow,
4581
+ ] = row as [
4582
+ unknown[],
4583
+ string[],
4584
+ string[],
4585
+ boolean | undefined,
4586
+ unknown[] | undefined,
4587
+ ];
4588
+ return {
4589
+ ...committedStorageFactsEntryFromRow(entryRow),
4590
+ trimmedEntryHashes: trimHashRows,
4591
+ trimmedEntryGids: trimGidRows,
4592
+ documentTrimmedHeadsProcessed,
4593
+ documentPreviousContext: documentContextFactsFromRow(contextRow),
4594
+ };
4595
+ };
4596
+
4597
+ const compactPreparedCommitFactsBaseFromRow = (
4598
+ row: unknown[],
4599
+ ): { entry: NativeBackboneCommittedEntry; trimRowOffset: number } => {
4161
4600
  const [hash, byteLength, metaBytes, fourth] = row as [
4162
4601
  string,
4163
4602
  number,
@@ -4165,32 +4604,66 @@ const compactPreparedCommitFactsWithTrimHashesFromRow = (
4165
4604
  Uint8Array | string[] | undefined,
4166
4605
  ];
4167
4606
  const hasDigestRow = fourth instanceof Uint8Array;
4168
- const hashDigestBytes = hasDigestRow ? fourth : undefined;
4169
- const trimHashOffset = hasDigestRow ? 4 : 3;
4170
- const trimHashRows = row[trimHashOffset] as string[] | undefined;
4171
- const documentTrimmedHeadsProcessed = row[trimHashOffset + 1] as
4607
+ return {
4608
+ entry: {
4609
+ cid: hash,
4610
+ hash,
4611
+ next: [],
4612
+ metaBytes,
4613
+ byteLength,
4614
+ hashDigestBytes: hasDigestRow ? fourth : undefined,
4615
+ },
4616
+ trimRowOffset: hasDigestRow ? 4 : 3,
4617
+ };
4618
+ };
4619
+
4620
+ const compactPreparedCommitFactsWithTrimHashesFromRow = (
4621
+ row: unknown[],
4622
+ ): NativeBackboneCommittedEntry & {
4623
+ trimmedEntryHashes?: string[];
4624
+ documentTrimmedHeadsProcessed?: boolean;
4625
+ } => {
4626
+ const { entry, trimRowOffset } = compactPreparedCommitFactsBaseFromRow(row);
4627
+ const trimHashRows = row[trimRowOffset] as string[] | undefined;
4628
+ const documentTrimmedHeadsProcessed = row[trimRowOffset + 1] as
4172
4629
  | boolean
4173
4630
  | undefined;
4174
4631
  return {
4175
- cid: hash,
4176
- hash,
4177
- next: [],
4178
- metaBytes,
4179
- byteLength,
4180
- hashDigestBytes,
4632
+ ...entry,
4181
4633
  trimmedEntryHashes: trimHashRows ?? [],
4182
4634
  documentTrimmedHeadsProcessed,
4183
4635
  };
4184
4636
  };
4185
4637
 
4186
- const nativeLogCommitEntryColumns = (
4187
- entries: NativeBackboneLogCommitEntry[],
4188
- ) => {
4189
- const hashes = new Array<string>(entries.length);
4190
- const blockBytes = new Array<Uint8Array>(entries.length);
4191
- const gids = new Array<string>(entries.length);
4192
- const nexts = new Array<string[]>(entries.length);
4193
- const entryTypes = new Uint8Array(entries.length);
4638
+ const compactPreparedCommitFactsWithTrimRefsFromRow = (
4639
+ row: unknown[],
4640
+ ): NativeBackboneCommittedEntry & {
4641
+ trimmedEntryHashes: string[];
4642
+ trimmedEntryGids: string[];
4643
+ documentTrimmedHeadsProcessed?: boolean;
4644
+ } => {
4645
+ const { entry, trimRowOffset } = compactPreparedCommitFactsBaseFromRow(row);
4646
+ const trimHashRows = row[trimRowOffset] as string[];
4647
+ const trimGidRows = row[trimRowOffset + 1] as string[];
4648
+ const documentTrimmedHeadsProcessed = row[trimRowOffset + 2] as
4649
+ | boolean
4650
+ | undefined;
4651
+ return {
4652
+ ...entry,
4653
+ trimmedEntryHashes: trimHashRows,
4654
+ trimmedEntryGids: trimGidRows,
4655
+ documentTrimmedHeadsProcessed,
4656
+ };
4657
+ };
4658
+
4659
+ const nativeLogCommitEntryColumns = (
4660
+ entries: NativeBackboneLogCommitEntry[],
4661
+ ) => {
4662
+ const hashes = new Array<string>(entries.length);
4663
+ const blockBytes = new Array<Uint8Array>(entries.length);
4664
+ const gids = new Array<string>(entries.length);
4665
+ const nexts = new Array<string[]>(entries.length);
4666
+ const entryTypes = new Uint8Array(entries.length);
4194
4667
  const wallTimes = new BigUint64Array(entries.length);
4195
4668
  const logicals = new Uint32Array(entries.length);
4196
4669
  const payloadSizes = new Uint32Array(entries.length);
@@ -5010,7 +5483,9 @@ class NativeBackboneLogGraph {
5010
5483
  | (NativeBackboneCommittedEntry & {
5011
5484
  trimmedEntries?: NativeBackboneLogEntry[];
5012
5485
  trimmedEntryHashes?: string[];
5486
+ trimmedEntryGids?: string[];
5013
5487
  documentTrimmedHeadsProcessed?: boolean;
5488
+ documentPreviousContext?: NativeBackboneDocumentContextFacts;
5014
5489
  })
5015
5490
  | undefined {
5016
5491
  if (this.options?.commitBlocks === false) {
@@ -5029,32 +5504,32 @@ class NativeBackboneLogGraph {
5029
5504
  const documentIndex = input.documentIndex;
5030
5505
  const documentIndexArgs = nativeDocumentIndexArgs(documentIndex);
5031
5506
  const projection = documentIndex?.projection;
5507
+ if (
5508
+ !documentIndexArgs &&
5509
+ input.resolveTrimmedEntries === false &&
5510
+ input.trimLengthTo != null &&
5511
+ this.native.prepare_plain_entry_commit_facts_trim_refs
5512
+ ) {
5513
+ return preparedCommitFactsWithTrimRefsFromRow(
5514
+ this.native.prepare_plain_entry_commit_facts_trim_refs(
5515
+ wallTime,
5516
+ logical,
5517
+ input.gid,
5518
+ input.next ?? [],
5519
+ entryType,
5520
+ input.metaData,
5521
+ input.payloadData,
5522
+ input.trimLengthTo,
5523
+ ),
5524
+ );
5525
+ }
5032
5526
  if (
5033
5527
  documentIndex?.useLatestContext &&
5034
5528
  documentIndexArgs &&
5035
5529
  input.resolveTrimmedEntries === false
5036
5530
  ) {
5037
5531
  if (projection && this.options?.documentProjectionPlanId) {
5038
- return preparedCommitFactsWithLatestDocumentContextFromRow(
5039
- this.native.prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes(
5040
- wallTime,
5041
- logical,
5042
- input.gid,
5043
- entryType,
5044
- input.metaData,
5045
- input.payloadData,
5046
- input.trimLengthTo,
5047
- documentIndex.key,
5048
- documentIndex.byteElementIndexLimit ?? 0,
5049
- documentIndex.deleteTrimmedHeads === true,
5050
- this.options.documentProjectionPlanId(projection.plan),
5051
- projection.encodedDocument,
5052
- projection.signer,
5053
- ),
5054
- );
5055
- }
5056
- return preparedCommitFactsWithLatestDocumentContextFromRow(
5057
- this.native.prepare_plain_entry_commit_latest_facts_document_index_trim_hashes(
5532
+ const args = [
5058
5533
  wallTime,
5059
5534
  logical,
5060
5535
  input.gid,
@@ -5063,12 +5538,53 @@ class NativeBackboneLogGraph {
5063
5538
  input.payloadData,
5064
5539
  input.trimLengthTo,
5065
5540
  documentIndex.key,
5066
- documentIndex.valuePrefixBytes ?? EMPTY_UINT8_ARRAY,
5067
5541
  documentIndex.byteElementIndexLimit ?? 0,
5068
5542
  documentIndex.deleteTrimmedHeads === true,
5069
- projection?.plan,
5070
- projection?.encodedDocument,
5071
- projection?.signer,
5543
+ this.options.documentProjectionPlanId(projection.plan),
5544
+ projection.encodedDocument,
5545
+ projection.signer,
5546
+ ] as const;
5547
+ const prepareTrimRefs =
5548
+ this.native
5549
+ .prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_refs;
5550
+ if (prepareTrimRefs) {
5551
+ return preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow(
5552
+ prepareTrimRefs.call(this.native, ...args),
5553
+ );
5554
+ }
5555
+ return preparedCommitFactsWithLatestDocumentContextFromRow(
5556
+ this.native.prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes(
5557
+ ...args,
5558
+ ),
5559
+ );
5560
+ }
5561
+ const args = [
5562
+ wallTime,
5563
+ logical,
5564
+ input.gid,
5565
+ entryType,
5566
+ input.metaData,
5567
+ input.payloadData,
5568
+ input.trimLengthTo,
5569
+ documentIndex.key,
5570
+ documentIndex.valuePrefixBytes ?? EMPTY_UINT8_ARRAY,
5571
+ documentIndex.byteElementIndexLimit ?? 0,
5572
+ documentIndex.deleteTrimmedHeads === true,
5573
+ projection?.plan,
5574
+ projection?.encodedDocument,
5575
+ projection?.signer,
5576
+ ] as const;
5577
+ const prepareTrimRefs =
5578
+ this.native
5579
+ .prepare_plain_entry_commit_latest_facts_document_index_trim_refs;
5580
+ if (prepareTrimRefs) {
5581
+ return preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow(
5582
+ prepareTrimRefs.call(this.native, ...args),
5583
+ );
5584
+ }
5585
+ return preparedCommitFactsWithLatestDocumentContextFromRow(
5586
+ this.native.prepare_plain_entry_commit_latest_facts_document_index_trim_hashes(
5587
+ ...args,
5072
5588
  ),
5073
5589
  );
5074
5590
  }
@@ -5083,49 +5599,71 @@ class NativeBackboneLogGraph {
5083
5599
  const projectionPlanId = this.options.documentProjectionPlanId(
5084
5600
  projection.plan,
5085
5601
  );
5602
+ const plainPutPayloadArgs = [
5603
+ wallTime,
5604
+ logical,
5605
+ input.gid,
5606
+ entryType,
5607
+ input.metaData,
5608
+ input.payloadData,
5609
+ input.trimLengthTo,
5610
+ documentIndex.key,
5611
+ documentIndex.existingCreated == null
5612
+ ? ""
5613
+ : integerString(documentIndex.existingCreated),
5614
+ documentIndex.byteElementIndexLimit ?? 0,
5615
+ documentIndex.deleteTrimmedHeads === true,
5616
+ projectionPlanId,
5617
+ projection.signer,
5618
+ ] as const;
5619
+ const plainPutPayloadCommitWithTrimRefs =
5620
+ this.native
5621
+ .prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs_plain_put_payload;
5622
+ if (plainPutPayloadCommitWithTrimRefs) {
5623
+ return compactPreparedCommitFactsWithTrimRefsFromRow(
5624
+ plainPutPayloadCommitWithTrimRefs.call(
5625
+ this.native,
5626
+ ...plainPutPayloadArgs,
5627
+ ),
5628
+ );
5629
+ }
5086
5630
  const plainPutPayloadCommit =
5087
5631
  this.native
5088
5632
  .prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes_plain_put_payload;
5089
5633
  if (plainPutPayloadCommit) {
5090
5634
  return compactPreparedCommitFactsWithTrimHashesFromRow(
5091
- plainPutPayloadCommit.call(
5092
- this.native,
5093
- wallTime,
5094
- logical,
5095
- input.gid,
5096
- entryType,
5097
- input.metaData,
5098
- input.payloadData,
5099
- input.trimLengthTo,
5100
- documentIndex.key,
5101
- documentIndex.existingCreated == null
5102
- ? ""
5103
- : integerString(documentIndex.existingCreated),
5104
- documentIndex.byteElementIndexLimit ?? 0,
5105
- documentIndex.deleteTrimmedHeads === true,
5106
- projectionPlanId,
5107
- projection.signer,
5108
- ),
5635
+ plainPutPayloadCommit.call(this.native, ...plainPutPayloadArgs),
5636
+ );
5637
+ }
5638
+ const args = [
5639
+ wallTime,
5640
+ logical,
5641
+ input.gid,
5642
+ entryType,
5643
+ input.metaData,
5644
+ input.payloadData,
5645
+ input.trimLengthTo,
5646
+ documentIndex.key,
5647
+ documentIndex.existingCreated == null
5648
+ ? ""
5649
+ : integerString(documentIndex.existingCreated),
5650
+ documentIndex.byteElementIndexLimit ?? 0,
5651
+ documentIndex.deleteTrimmedHeads === true,
5652
+ projectionPlanId,
5653
+ projection.encodedDocument,
5654
+ projection.signer,
5655
+ ] as const;
5656
+ const prepareTrimRefs =
5657
+ this.native
5658
+ .prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs;
5659
+ if (prepareTrimRefs) {
5660
+ return compactPreparedCommitFactsWithTrimRefsFromRow(
5661
+ prepareTrimRefs.call(this.native, ...args),
5109
5662
  );
5110
5663
  }
5111
5664
  return compactPreparedCommitFactsWithTrimHashesFromRow(
5112
5665
  this.native.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes(
5113
- wallTime,
5114
- logical,
5115
- input.gid,
5116
- entryType,
5117
- input.metaData,
5118
- input.payloadData,
5119
- input.trimLengthTo,
5120
- documentIndex.key,
5121
- documentIndex.existingCreated == null
5122
- ? ""
5123
- : integerString(documentIndex.existingCreated),
5124
- documentIndex.byteElementIndexLimit ?? 0,
5125
- documentIndex.deleteTrimmedHeads === true,
5126
- projectionPlanId,
5127
- projection.encodedDocument,
5128
- projection.signer,
5666
+ ...args,
5129
5667
  ),
5130
5668
  );
5131
5669
  }
@@ -5232,6 +5770,24 @@ class NativeBackboneLogGraph {
5232
5770
  input.trimLengthTo != null &&
5233
5771
  hasNoNext
5234
5772
  ) {
5773
+ const prepareTrimRefs =
5774
+ this.native
5775
+ .prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_refs;
5776
+ if (prepareTrimRefs) {
5777
+ return compactPreparedCommitFactsWithTrimRefsFromRow(
5778
+ prepareTrimRefs.call(
5779
+ this.native,
5780
+ wallTime,
5781
+ logical,
5782
+ input.gid,
5783
+ entryType,
5784
+ input.metaData,
5785
+ input.payloadData,
5786
+ input.trimLengthTo,
5787
+ ...documentIndexArgs,
5788
+ ),
5789
+ );
5790
+ }
5235
5791
  return compactPreparedCommitFactsWithTrimHashesFromRow(
5236
5792
  this.native.prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_hashes(
5237
5793
  wallTime,
@@ -6385,9 +6941,7 @@ export class NativePeerbitBackbone {
6385
6941
  end2: bigint | number | string;
6386
6942
  limit: number;
6387
6943
  }): bigint[] | undefined {
6388
- if (
6389
- typeof this.native.entry_hash_numbers_in_range_limited !== "function"
6390
- ) {
6944
+ if (typeof this.native.entry_hash_numbers_in_range_limited !== "function") {
6391
6945
  return undefined;
6392
6946
  }
6393
6947
  return rowsToNumbers(
@@ -8646,6 +9200,23 @@ export class NativeBackboneMemoryCoordinatePersistenceStore
8646
9200
  return file ? copyBytes(file) : undefined;
8647
9201
  }
8648
9202
 
9203
+ async readLimited(
9204
+ name: string,
9205
+ maxBytes: number,
9206
+ ): Promise<Uint8Array | undefined> {
9207
+ const validName = validateCoordinatePersistenceName(name);
9208
+ const limit = validateCoordinatePersistenceReadMaxBytes(maxBytes);
9209
+ const file = this.files.get(validName);
9210
+ if (file) {
9211
+ assertCoordinatePersistenceReadWithinLimit(
9212
+ validName,
9213
+ limit,
9214
+ file.byteLength,
9215
+ );
9216
+ }
9217
+ return file ? copyBytes(file) : undefined;
9218
+ }
9219
+
8649
9220
  async write(name: string, bytes: Uint8Array): Promise<void> {
8650
9221
  this.files.set(validateCoordinatePersistenceName(name), copyBytes(bytes));
8651
9222
  }
@@ -8676,7 +9247,12 @@ export class NativeBackboneNodeCoordinatePersistenceStore
8676
9247
  private readonly filePaths = new Map<string, string>();
8677
9248
  private directoryEnsured = false;
8678
9249
  private appendFailure: unknown;
9250
+ readonly readLimited?: (
9251
+ name: string,
9252
+ maxBytes: number,
9253
+ ) => Promise<Uint8Array | undefined>;
8679
9254
  readonly durableBarrier?: (name?: string) => Promise<void>;
9255
+ readonly atomicReplace?: (name: string, bytes: Uint8Array) => Promise<void>;
8680
9256
 
8681
9257
  constructor(
8682
9258
  private readonly directory: string,
@@ -8688,6 +9264,16 @@ export class NativeBackboneNodeCoordinatePersistenceStore
8688
9264
  if (!fs || typeof fs.open === "function") {
8689
9265
  this.durableBarrier = (name) => this.syncDurably(name);
8690
9266
  }
9267
+ if (!fs || typeof fs.openBoundedRead === "function") {
9268
+ this.readLimited = (name, maxBytes) =>
9269
+ this.readWithinLimit(name, maxBytes);
9270
+ }
9271
+ if (
9272
+ !fs ||
9273
+ (typeof fs.open === "function" && typeof fs.rename === "function")
9274
+ ) {
9275
+ this.atomicReplace = (name, bytes) => this.replaceAtomically(name, bytes);
9276
+ }
8691
9277
  }
8692
9278
 
8693
9279
  private async nodeFs(): Promise<NativeBackboneNodeFs> {
@@ -8749,6 +9335,120 @@ export class NativeBackboneNodeCoordinatePersistenceStore
8749
9335
  }
8750
9336
  }
8751
9337
 
9338
+ private async readWithinLimit(
9339
+ name: string,
9340
+ maxBytes: number,
9341
+ ): Promise<Uint8Array | undefined> {
9342
+ const validName = validateCoordinatePersistenceName(name);
9343
+ const limit = validateCoordinatePersistenceReadMaxBytes(maxBytes);
9344
+ const fs = await this.nodeFs();
9345
+ const path = await this.filePath(validName);
9346
+ let handle: NativeBackboneNodeBoundedReadFileHandle | undefined;
9347
+ try {
9348
+ if (this.fs) {
9349
+ handle = await this.fs.openBoundedRead!(path);
9350
+ } else {
9351
+ if (!fs.open) {
9352
+ throw new Error(
9353
+ "Default Node coordinate persistence does not expose FileHandle.open",
9354
+ );
9355
+ }
9356
+ const opened = await fs.open(path, "r");
9357
+ if (!opened.stat || !opened.read) {
9358
+ await opened.close();
9359
+ throw new Error(
9360
+ "Default Node coordinate persistence bounded reads require FileHandle.stat and FileHandle.read",
9361
+ );
9362
+ }
9363
+ handle = opened as NativeBackboneNodeBoundedReadFileHandle;
9364
+ }
9365
+ } catch (error) {
9366
+ if (isNotFoundError(error)) {
9367
+ return undefined;
9368
+ }
9369
+ throw error;
9370
+ }
9371
+ try {
9372
+ const initial = await handle.stat({ bigint: true });
9373
+ if (typeof initial.size !== "bigint" || initial.size < 0n) {
9374
+ throw new Error(
9375
+ "Node coordinate persistence returned an invalid bigint file size",
9376
+ );
9377
+ }
9378
+ assertCoordinatePersistenceReadWithinLimit(
9379
+ validName,
9380
+ limit,
9381
+ initial.size,
9382
+ );
9383
+ if (initial.size > BigInt(Number.MAX_SAFE_INTEGER)) {
9384
+ throw new RangeError(
9385
+ "Node coordinate persistence file is too large to materialize safely",
9386
+ );
9387
+ }
9388
+ const byteLength = Number(initial.size);
9389
+ const bytes = new Uint8Array(byteLength);
9390
+ let offset = 0;
9391
+ while (offset < byteLength) {
9392
+ const { bytesRead } = await handle.read(
9393
+ bytes,
9394
+ offset,
9395
+ byteLength - offset,
9396
+ offset,
9397
+ );
9398
+ if (
9399
+ !Number.isSafeInteger(bytesRead) ||
9400
+ bytesRead <= 0 ||
9401
+ bytesRead > byteLength - offset
9402
+ ) {
9403
+ throw new Error(
9404
+ "Node coordinate persistence file changed during a bounded read",
9405
+ );
9406
+ }
9407
+ offset += bytesRead;
9408
+ }
9409
+ const growthProbe = new Uint8Array(1);
9410
+ const { bytesRead: growthBytes } = await handle.read(
9411
+ growthProbe,
9412
+ 0,
9413
+ 1,
9414
+ byteLength,
9415
+ );
9416
+ if (
9417
+ !Number.isSafeInteger(growthBytes) ||
9418
+ growthBytes < 0 ||
9419
+ growthBytes > 1
9420
+ ) {
9421
+ throw new Error(
9422
+ "Node coordinate persistence returned invalid bounded read progress",
9423
+ );
9424
+ }
9425
+ const final = await handle.stat({ bigint: true });
9426
+ if (typeof final.size !== "bigint" || final.size < 0n) {
9427
+ throw new Error(
9428
+ "Node coordinate persistence returned an invalid bigint file size",
9429
+ );
9430
+ }
9431
+ if (growthBytes !== 0 || final.size > initial.size) {
9432
+ assertCoordinatePersistenceReadWithinLimit(
9433
+ validName,
9434
+ limit,
9435
+ final.size > initial.size ? final.size : initial.size + 1n,
9436
+ );
9437
+ throw new Error(
9438
+ "Node coordinate persistence file changed during a bounded read",
9439
+ );
9440
+ }
9441
+ if (final.size !== initial.size) {
9442
+ throw new Error(
9443
+ "Node coordinate persistence file changed during a bounded read",
9444
+ );
9445
+ }
9446
+ return bytes;
9447
+ } finally {
9448
+ await handle.close();
9449
+ }
9450
+ }
9451
+
8752
9452
  async write(name: string, bytes: Uint8Array): Promise<void> {
8753
9453
  const fs = await this.ensureDirectory();
8754
9454
  const path = await this.filePath(name);
@@ -8756,6 +9456,51 @@ export class NativeBackboneNodeCoordinatePersistenceStore
8756
9456
  await fs.writeFile(path, bytes);
8757
9457
  }
8758
9458
 
9459
+ private async replaceAtomically(
9460
+ name: string,
9461
+ bytes: Uint8Array,
9462
+ ): Promise<void> {
9463
+ const fs = await this.ensureDirectory();
9464
+ if (!fs.open || !fs.rename) {
9465
+ throw new Error(
9466
+ "Node coordinate persistence does not expose atomic file replacement",
9467
+ );
9468
+ }
9469
+ const validName = validateCoordinatePersistenceName(name);
9470
+ const temporaryName = validateCoordinatePersistenceName(`${validName}.tmp`);
9471
+ const path = await this.filePath(validName);
9472
+ const temporaryPath = await this.filePath(temporaryName);
9473
+ await this.closeAppendHandle(path);
9474
+ await this.closeAppendHandle(temporaryPath);
9475
+ await fs.rm(temporaryPath, { force: true });
9476
+ await fs.writeFile(temporaryPath, bytes);
9477
+ let temporaryHandle: NativeBackboneNodeAppendFileHandle | undefined;
9478
+ try {
9479
+ temporaryHandle = await fs.open(temporaryPath, "r");
9480
+ if (typeof temporaryHandle.sync !== "function") {
9481
+ throw new Error(
9482
+ "Node coordinate persistence atomic replacement does not expose FileHandle.sync",
9483
+ );
9484
+ }
9485
+ await temporaryHandle.sync();
9486
+ } finally {
9487
+ await temporaryHandle?.close();
9488
+ }
9489
+ await fs.rename(temporaryPath, path);
9490
+ let directoryHandle: NativeBackboneNodeAppendFileHandle | undefined;
9491
+ try {
9492
+ directoryHandle = await fs.open(this.directory, "r");
9493
+ if (typeof directoryHandle.sync !== "function") {
9494
+ throw new Error(
9495
+ "Node coordinate persistence atomic replacement does not expose directory sync",
9496
+ );
9497
+ }
9498
+ await directoryHandle.sync();
9499
+ } finally {
9500
+ await directoryHandle?.close();
9501
+ }
9502
+ }
9503
+
8759
9504
  async append(name: string, bytes: Uint8Array): Promise<void> {
8760
9505
  if (this.appendFailure !== undefined) {
8761
9506
  throw this.appendFailure;
@@ -8949,6 +9694,38 @@ export class NativeBackboneOPFSCoordinatePersistenceStore
8949
9694
  }
8950
9695
  }
8951
9696
 
9697
+ async readLimited(
9698
+ name: string,
9699
+ maxBytes: number,
9700
+ ): Promise<Uint8Array | undefined> {
9701
+ const validName = validateCoordinatePersistenceName(name);
9702
+ const limit = validateCoordinatePersistenceReadMaxBytes(maxBytes);
9703
+ try {
9704
+ const handle = await this.directory.getFileHandle(validName, {
9705
+ create: false,
9706
+ });
9707
+ const file = await handle.getFile();
9708
+ if (!Number.isSafeInteger(file.size) || file.size < 0) {
9709
+ throw new Error(
9710
+ "OPFS coordinate persistence returned an invalid file size",
9711
+ );
9712
+ }
9713
+ assertCoordinatePersistenceReadWithinLimit(validName, limit, file.size);
9714
+ const buffer = await file.arrayBuffer();
9715
+ if (buffer.byteLength !== file.size) {
9716
+ throw new Error(
9717
+ "OPFS coordinate persistence file changed during a bounded read",
9718
+ );
9719
+ }
9720
+ return new Uint8Array(buffer);
9721
+ } catch (error) {
9722
+ if (isNotFoundError(error)) {
9723
+ return undefined;
9724
+ }
9725
+ throw error;
9726
+ }
9727
+ }
9728
+
8952
9729
  async write(name: string, bytes: Uint8Array): Promise<void> {
8953
9730
  const handle = await this.directory.getFileHandle(
8954
9731
  validateCoordinatePersistenceName(name),
@@ -9070,8 +9847,13 @@ export class NativeBackboneBufferedCoordinatePersistenceStore
9070
9847
  {
9071
9848
  private readonly buffers = new Map<string, Uint8Array[]>();
9072
9849
  private bufferedBytes = 0;
9850
+ readonly readLimited?: (
9851
+ name: string,
9852
+ maxBytes: number,
9853
+ ) => Promise<Uint8Array | undefined>;
9073
9854
  readonly supportsRemoval: boolean;
9074
9855
  readonly durableBarrier?: (name?: string) => Promise<void>;
9856
+ readonly atomicReplace?: (name: string, bytes: Uint8Array) => Promise<void>;
9075
9857
 
9076
9858
  constructor(
9077
9859
  private readonly inner: NativeBackboneCoordinatePersistenceStore,
@@ -9079,12 +9861,23 @@ export class NativeBackboneBufferedCoordinatePersistenceStore
9079
9861
  ) {
9080
9862
  this.supportsRemoval =
9081
9863
  inner.supportsRemoval ?? typeof inner.remove === "function";
9864
+ if (typeof inner.readLimited === "function") {
9865
+ const innerReadLimited = inner.readLimited.bind(inner);
9866
+ this.readLimited = (name, maxBytes) =>
9867
+ this.readWithinLimit(name, maxBytes, innerReadLimited);
9868
+ }
9082
9869
  if (typeof inner.durableBarrier === "function") {
9083
9870
  this.durableBarrier = async (name) => {
9084
9871
  await this.flush(name);
9085
9872
  await inner.durableBarrier!(name);
9086
9873
  };
9087
9874
  }
9875
+ if (typeof inner.atomicReplace === "function") {
9876
+ this.atomicReplace = async (name, bytes) => {
9877
+ await this.flush(name);
9878
+ await inner.atomicReplace!(name, bytes);
9879
+ };
9880
+ }
9088
9881
  }
9089
9882
 
9090
9883
  private buffer(name: string): Uint8Array[] {
@@ -9102,6 +9895,63 @@ export class NativeBackboneBufferedCoordinatePersistenceStore
9102
9895
  return this.inner.read(name);
9103
9896
  }
9104
9897
 
9898
+ private async readWithinLimit(
9899
+ name: string,
9900
+ maxBytes: number,
9901
+ innerReadLimited: (
9902
+ name: string,
9903
+ maxBytes: number,
9904
+ ) => Promise<Uint8Array | undefined>,
9905
+ ): Promise<Uint8Array | undefined> {
9906
+ const validName = validateCoordinatePersistenceName(name);
9907
+ const limit = validateCoordinatePersistenceReadMaxBytes(maxBytes);
9908
+ const pending = this.buffers.get(validName);
9909
+ if (!pending || pending.length === 0) {
9910
+ return innerReadLimited(validName, limit);
9911
+ }
9912
+ const pendingLength = pending.length;
9913
+ let pendingBytes = 0n;
9914
+ for (const chunk of pending) {
9915
+ pendingBytes += BigInt(chunk.byteLength);
9916
+ }
9917
+ assertCoordinatePersistenceReadWithinLimit(validName, limit, pendingBytes);
9918
+ const remaining = limit - Number(pendingBytes);
9919
+ let existingBytes: number | undefined;
9920
+ try {
9921
+ existingBytes = (await innerReadLimited(validName, remaining))
9922
+ ?.byteLength;
9923
+ } catch (error) {
9924
+ if (error instanceof NativeBackboneCoordinatePersistenceReadLimitError) {
9925
+ throw new NativeBackboneCoordinatePersistenceReadLimitError(
9926
+ validName,
9927
+ limit,
9928
+ pendingBytes + error.observedBytes,
9929
+ );
9930
+ }
9931
+ throw error;
9932
+ }
9933
+ let confirmedPendingBytes = 0n;
9934
+ for (const chunk of pending) {
9935
+ confirmedPendingBytes += BigInt(chunk.byteLength);
9936
+ }
9937
+ if (
9938
+ this.buffers.get(validName) !== pending ||
9939
+ pending.length !== pendingLength ||
9940
+ confirmedPendingBytes !== pendingBytes
9941
+ ) {
9942
+ throw new Error(
9943
+ "Native backbone coordinate persistence pending bytes changed during a bounded read",
9944
+ );
9945
+ }
9946
+ assertCoordinatePersistenceReadWithinLimit(
9947
+ validName,
9948
+ limit,
9949
+ pendingBytes + BigInt(existingBytes ?? 0),
9950
+ );
9951
+ await this.flush(validName);
9952
+ return innerReadLimited(validName, limit);
9953
+ }
9954
+
9105
9955
  async write(name: string, bytes: Uint8Array): Promise<void> {
9106
9956
  await this.flush(name);
9107
9957
  await this.inner.write(name, bytes);
@@ -9173,7 +10023,7 @@ export class NativeBackboneCoordinatePersistence {
9173
10023
  readonly flushIntervalMs?: number;
9174
10024
  readonly compactMaxJournalBytes?: number;
9175
10025
  readonly compactMaxJournalRecords?: number;
9176
- readonly crashSafeCompaction = false;
10026
+ readonly crashSafeCompaction: boolean;
9177
10027
  readonly durableBarrier: boolean;
9178
10028
  readonly supportsDrop: boolean;
9179
10029
  readonly dropIsTerminal = true;
@@ -9183,9 +10033,32 @@ export class NativeBackboneCoordinatePersistence {
9183
10033
  private readonly documentJournalFile: string;
9184
10034
  private readonly documentSignerSnapshotFile: string;
9185
10035
  private readonly documentSignerJournalFile: string;
10036
+ private readonly checkpointConfigurationChecksum: number;
10037
+ private readonly checkpointStateFile: string;
10038
+ private readonly checkpointFiles: Record<
10039
+ NativeBackboneCoordinateCheckpointSlot,
10040
+ string
10041
+ >;
10042
+ private readonly checkpointJournalFiles: Record<
10043
+ NativeBackboneCoordinateCheckpointSlot,
10044
+ {
10045
+ coordinate: string;
10046
+ document: string;
10047
+ signer: string;
10048
+ }
10049
+ >;
9186
10050
  private journalInitialized: boolean | undefined;
10051
+ private journalByteLength = 0;
10052
+ private journalRecordCount = 0;
9187
10053
  private documentJournalInitialized: boolean | undefined;
10054
+ private documentJournalByteLength = 0;
10055
+ private documentJournalRecordCount = 0;
9188
10056
  private documentSignerJournalInitialized: boolean | undefined;
10057
+ private documentSignerJournalByteLength = 0;
10058
+ private documentSignerJournalRecordCount = 0;
10059
+ private checkpointHighwater = 0n;
10060
+ private checkpointCompleted?: NativeBackboneCoordinateCheckpointAuthority;
10061
+ private checkpointPending?: NativeBackboneCoordinateCheckpointAuthority;
9189
10062
  private lastFlushMs = Date.now();
9190
10063
  private persistenceQueue: Promise<unknown> | undefined;
9191
10064
  private persistenceLifecycle:
@@ -9232,24 +10105,87 @@ export class NativeBackboneCoordinatePersistence {
9232
10105
  options.documentSignerJournal ??
9233
10106
  nativeBackboneCoordinatePersistenceFiles.documentSignerJournal,
9234
10107
  );
10108
+ this.checkpointConfigurationChecksum = coordinateJournalChecksum(
10109
+ new TextEncoder().encode(
10110
+ JSON.stringify([
10111
+ this.snapshotFile,
10112
+ this.journalFile,
10113
+ this.documentSnapshotFile,
10114
+ this.documentJournalFile,
10115
+ this.documentSignerSnapshotFile,
10116
+ this.documentSignerJournalFile,
10117
+ ]),
10118
+ ),
10119
+ );
10120
+ this.checkpointStateFile = validateCoordinatePersistenceName(
10121
+ `${this.snapshotFile}.checkpoint-state`,
10122
+ );
10123
+ this.checkpointFiles = {
10124
+ a: validateCoordinatePersistenceName(`${this.snapshotFile}.checkpoint-a`),
10125
+ b: validateCoordinatePersistenceName(`${this.snapshotFile}.checkpoint-b`),
10126
+ };
10127
+ this.checkpointJournalFiles = {
10128
+ a: {
10129
+ coordinate: validateCoordinatePersistenceName(
10130
+ `${this.journalFile}.checkpoint-a`,
10131
+ ),
10132
+ document: validateCoordinatePersistenceName(
10133
+ `${this.documentJournalFile}.checkpoint-a`,
10134
+ ),
10135
+ signer: validateCoordinatePersistenceName(
10136
+ `${this.documentSignerJournalFile}.checkpoint-a`,
10137
+ ),
10138
+ },
10139
+ b: {
10140
+ coordinate: validateCoordinatePersistenceName(
10141
+ `${this.journalFile}.checkpoint-b`,
10142
+ ),
10143
+ document: validateCoordinatePersistenceName(
10144
+ `${this.documentJournalFile}.checkpoint-b`,
10145
+ ),
10146
+ signer: validateCoordinatePersistenceName(
10147
+ `${this.documentSignerJournalFile}.checkpoint-b`,
10148
+ ),
10149
+ },
10150
+ };
9235
10151
  if (
9236
- this.configuredFiles().includes(nativeBackboneCoordinateDropTombstoneFile)
10152
+ this.configuredFiles().includes(
10153
+ nativeBackboneCoordinateDropTombstoneFile,
10154
+ ) ||
10155
+ new Set(this.configuredFiles()).size !== this.configuredFiles().length
9237
10156
  ) {
9238
10157
  throw new Error(
9239
- "Native backbone coordinate persistence file conflicts with its drop tombstone",
10158
+ "Native backbone coordinate persistence files conflict with one another or with its drop tombstone",
9240
10159
  );
9241
10160
  }
10161
+ this.crashSafeCompaction =
10162
+ typeof store.atomicReplace === "function" &&
10163
+ this.supportsDrop &&
10164
+ this.durableBarrier;
9242
10165
  this.flushOnAppend = options.flushOnAppend ?? true;
9243
10166
  this.flushMaxPendingBytes = resolveCoordinateFlushMaxPendingBytes(options);
9244
10167
  if (options.flushIntervalMs != null) {
9245
10168
  this.flushIntervalMs = Math.max(0, options.flushIntervalMs);
9246
10169
  }
9247
10170
  if (
9248
- options.compactMaxJournalBytes != null ||
9249
- options.compactMaxJournalRecords != null
10171
+ (options.compactMaxJournalBytes != null ||
10172
+ options.compactMaxJournalRecords != null) &&
10173
+ !this.crashSafeCompaction
9250
10174
  ) {
9251
10175
  throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
9252
10176
  }
10177
+ if (options.compactMaxJournalBytes != null) {
10178
+ this.compactMaxJournalBytes = validateCoordinateCompactionThreshold(
10179
+ options.compactMaxJournalBytes,
10180
+ "compactMaxJournalBytes",
10181
+ );
10182
+ }
10183
+ if (options.compactMaxJournalRecords != null) {
10184
+ this.compactMaxJournalRecords = validateCoordinateCompactionThreshold(
10185
+ options.compactMaxJournalRecords,
10186
+ "compactMaxJournalRecords",
10187
+ );
10188
+ }
9253
10189
  }
9254
10190
 
9255
10191
  /** Durable operation-intent capability for strict shared-log transactions. */
@@ -9257,7 +10193,7 @@ export class NativeBackboneCoordinatePersistence {
9257
10193
  return this.store;
9258
10194
  }
9259
10195
 
9260
- private configuredFiles(): string[] {
10196
+ private legacyConfiguredFiles(): string[] {
9261
10197
  return [
9262
10198
  this.snapshotFile,
9263
10199
  this.journalFile,
@@ -9268,6 +10204,47 @@ export class NativeBackboneCoordinatePersistence {
9268
10204
  ];
9269
10205
  }
9270
10206
 
10207
+ private configuredFiles(): string[] {
10208
+ return [
10209
+ ...this.legacyConfiguredFiles(),
10210
+ this.checkpointStateFile,
10211
+ this.checkpointFiles.a,
10212
+ this.checkpointFiles.b,
10213
+ this.checkpointJournalFiles.a.coordinate,
10214
+ this.checkpointJournalFiles.a.document,
10215
+ this.checkpointJournalFiles.a.signer,
10216
+ this.checkpointJournalFiles.b.coordinate,
10217
+ this.checkpointJournalFiles.b.document,
10218
+ this.checkpointJournalFiles.b.signer,
10219
+ `${this.checkpointStateFile}.tmp`,
10220
+ `${this.checkpointFiles.a}.tmp`,
10221
+ `${this.checkpointFiles.b}.tmp`,
10222
+ `${this.journalFile}.tmp`,
10223
+ `${nativeBackboneCoordinateDropTombstoneFile}.tmp`,
10224
+ ];
10225
+ }
10226
+
10227
+ private checkpointSlot(
10228
+ generation: bigint,
10229
+ ): NativeBackboneCoordinateCheckpointSlot {
10230
+ return generation % 2n === 1n ? "a" : "b";
10231
+ }
10232
+
10233
+ private activeJournalFiles(): {
10234
+ coordinate: string;
10235
+ document: string;
10236
+ signer: string;
10237
+ } {
10238
+ const completed = this.checkpointCompleted;
10239
+ return completed
10240
+ ? this.checkpointJournalFiles[this.checkpointSlot(completed.generation)]
10241
+ : {
10242
+ coordinate: this.journalFile,
10243
+ document: this.documentJournalFile,
10244
+ signer: this.documentSignerJournalFile,
10245
+ };
10246
+ }
10247
+
9271
10248
  private assertLifecycleActive(operation: string): void {
9272
10249
  if (this.persistenceLifecycle !== "active") {
9273
10250
  throw new Error(
@@ -9277,9 +10254,7 @@ export class NativeBackboneCoordinatePersistence {
9277
10254
  }
9278
10255
 
9279
10256
  private assertActive(operation: string): void {
9280
- if (this.persistenceFailure !== undefined) {
9281
- throw this.persistenceFailure;
9282
- }
10257
+ this.assertPersistenceHealthy();
9283
10258
  this.assertLifecycleActive(operation);
9284
10259
  if (this.dropInitiatedOnGeneration) {
9285
10260
  throw new Error(
@@ -9288,13 +10263,44 @@ export class NativeBackboneCoordinatePersistence {
9288
10263
  }
9289
10264
  }
9290
10265
 
10266
+ private assertPersistenceHealthy(): void {
10267
+ if (this.persistenceFailure !== undefined) {
10268
+ throw this.persistenceFailure;
10269
+ }
10270
+ }
10271
+
9291
10272
  private resetJournalTracking(): void {
9292
10273
  this.journalInitialized = undefined;
10274
+ this.journalByteLength = 0;
10275
+ this.journalRecordCount = 0;
9293
10276
  this.documentJournalInitialized = undefined;
10277
+ this.documentJournalByteLength = 0;
10278
+ this.documentJournalRecordCount = 0;
9294
10279
  this.documentSignerJournalInitialized = undefined;
10280
+ this.documentSignerJournalByteLength = 0;
10281
+ this.documentSignerJournalRecordCount = 0;
10282
+ this.checkpointHighwater = 0n;
10283
+ this.checkpointCompleted = undefined;
10284
+ this.checkpointPending = undefined;
9295
10285
  this.lastFlushMs = Date.now();
9296
10286
  }
9297
10287
 
10288
+ private async persistDropTombstone(
10289
+ files: readonly string[],
10290
+ preferAtomicReplace = false,
10291
+ ): Promise<void> {
10292
+ const bytes = nativeBackboneCoordinateDropTombstoneBytes(files);
10293
+ if (preferAtomicReplace && this.store.atomicReplace) {
10294
+ await this.store.atomicReplace(
10295
+ nativeBackboneCoordinateDropTombstoneFile,
10296
+ bytes,
10297
+ );
10298
+ return;
10299
+ }
10300
+ await this.store.write(nativeBackboneCoordinateDropTombstoneFile, bytes);
10301
+ await this.barrierFile(nativeBackboneCoordinateDropTombstoneFile);
10302
+ }
10303
+
9298
10304
  private async eraseDropFiles(files: readonly string[]): Promise<void> {
9299
10305
  if (!this.supportsDrop || !this.store.remove) {
9300
10306
  throw new Error(
@@ -9344,17 +10350,7 @@ export class NativeBackboneCoordinatePersistence {
9344
10350
  this.dropInitiatedOnGeneration = true;
9345
10351
  this.persistenceLifecycle = "dropping";
9346
10352
  await this.enqueuePersistence(async () => {
9347
- await this.store.write(
9348
- nativeBackboneCoordinateDropTombstoneFile,
9349
- nativeBackboneCoordinateDropTombstoneBytes(files),
9350
- );
9351
- if (this.store.durableBarrier) {
9352
- await this.store.durableBarrier(
9353
- nativeBackboneCoordinateDropTombstoneFile,
9354
- );
9355
- } else {
9356
- await this.store.flush?.(nativeBackboneCoordinateDropTombstoneFile);
9357
- }
10353
+ await this.persistDropTombstone(files);
9358
10354
  await this.eraseDropFiles(files);
9359
10355
  this.resetJournalTracking();
9360
10356
  this.persistenceLifecycle = "dropped";
@@ -9392,6 +10388,57 @@ export class NativeBackboneCoordinatePersistence {
9392
10388
  });
9393
10389
  }
9394
10390
 
10391
+ private async barrierFile(name: string): Promise<void> {
10392
+ if (this.store.durableBarrier) {
10393
+ await this.store.durableBarrier(name);
10394
+ } else {
10395
+ await this.store.flush?.(name);
10396
+ }
10397
+ }
10398
+
10399
+ private async hasDurableLegacyMigrationSentinel(): Promise<boolean> {
10400
+ const existing = await this.store.read(this.journalFile);
10401
+ if (
10402
+ existing &&
10403
+ existing.byteLength ===
10404
+ nativeBackboneCoordinateLegacyMigrationSentinel.byteLength &&
10405
+ nativeBackboneCoordinateLegacyMigrationSentinel.every(
10406
+ (byte, index) => existing[index] === byte,
10407
+ )
10408
+ ) {
10409
+ // A previous process may have observed the bytes from page cache before
10410
+ // its durability barrier rejected. Cross a fresh barrier on every reopen
10411
+ // before any post-checkpoint WAL append can be admitted.
10412
+ await this.barrierFile(this.journalFile);
10413
+ return true;
10414
+ }
10415
+ return false;
10416
+ }
10417
+
10418
+ private async installLegacyMigrationSentinel(): Promise<void> {
10419
+ if (await this.hasDurableLegacyMigrationSentinel()) {
10420
+ return;
10421
+ }
10422
+ if (!this.store.atomicReplace) {
10423
+ throw new Error(
10424
+ "Native backbone completed checkpoint can not install its downgrade sentinel atomically",
10425
+ );
10426
+ }
10427
+ await this.store.atomicReplace(
10428
+ this.journalFile,
10429
+ nativeBackboneCoordinateLegacyMigrationSentinel,
10430
+ );
10431
+ }
10432
+
10433
+ private async requireLegacyMigrationSentinel(): Promise<void> {
10434
+ if (await this.hasDurableLegacyMigrationSentinel()) {
10435
+ return;
10436
+ }
10437
+ throw new Error(
10438
+ "Native backbone completed checkpoint downgrade sentinel is missing or has been replaced; refusing to overwrite possible legacy writes",
10439
+ );
10440
+ }
10441
+
9395
10442
  async hydrate(backbone: NativePeerbitBackbone): Promise<number> {
9396
10443
  this.assertActive("hydrate");
9397
10444
  // Claim the lifecycle synchronously. close() queues after this complete
@@ -9402,33 +10449,175 @@ export class NativeBackboneCoordinatePersistence {
9402
10449
  this.assertHydrating();
9403
10450
  await this.resumeDropInternal("hydrating");
9404
10451
  this.assertHydrating();
9405
- const [
9406
- snapshot,
9407
- journal,
9408
- documentSnapshot,
9409
- documentJournal,
9410
- documentSignerSnapshot,
9411
- documentSignerJournal,
9412
- ] = await Promise.all([
9413
- this.store.read(this.snapshotFile),
9414
- this.store.read(this.journalFile),
9415
- this.store.read(this.documentSnapshotFile),
9416
- this.store.read(this.documentJournalFile),
9417
- this.store.read(this.documentSignerSnapshotFile),
9418
- this.store.read(this.documentSignerJournalFile),
9419
- ]);
10452
+ const checkpointStateBytes = await this.store.read(
10453
+ this.checkpointStateFile,
10454
+ );
9420
10455
  this.assertHydrating();
9421
- try {
9422
- validateCoordinateJournal(journal, this.journalFile);
9423
- validateCoordinateJournal(documentJournal, this.documentJournalFile);
9424
- validateCoordinateJournal(
10456
+ let checkpointState:
10457
+ | NativeBackboneCoordinateCheckpointState
10458
+ | undefined;
10459
+ if (checkpointStateBytes) {
10460
+ try {
10461
+ checkpointState = parseNativeBackboneCoordinateCheckpointState(
10462
+ checkpointStateBytes,
10463
+ this.checkpointStateFile,
10464
+ );
10465
+ if (
10466
+ checkpointState.configurationChecksum !==
10467
+ this.checkpointConfigurationChecksum
10468
+ ) {
10469
+ throw new Error(
10470
+ "Native backbone checkpoint authority does not match the configured persistence files",
10471
+ );
10472
+ }
10473
+ } catch (error) {
10474
+ this.persistenceFailure ??= error;
10475
+ throw this.persistenceFailure;
10476
+ }
10477
+ }
10478
+ this.checkpointHighwater = checkpointState?.highwater ?? 0n;
10479
+ this.checkpointPending = checkpointState?.pending;
10480
+ this.checkpointCompleted = checkpointState?.completed;
10481
+ let checkpointAuthority = this.checkpointCompleted;
10482
+ let promotePendingCheckpoint = false;
10483
+ if (!checkpointAuthority && this.checkpointPending) {
10484
+ try {
10485
+ // On the first generation only, the sentinel is the durable stage
10486
+ // boundary: publication writes it after the pending checkpoint and its
10487
+ // WAL headers, but before completed authority. A pre-checkpoint reader
10488
+ // can no longer consume legacy state, so recovery may validate and
10489
+ // promote the staged generation. Once a completed generation exists the
10490
+ // sentinel predates later pending work and is not a promotion signal.
10491
+ if (await this.hasDurableLegacyMigrationSentinel()) {
10492
+ checkpointAuthority = this.checkpointPending;
10493
+ promotePendingCheckpoint = true;
10494
+ }
10495
+ } catch (error) {
10496
+ this.persistenceFailure ??= error;
10497
+ throw this.persistenceFailure;
10498
+ }
10499
+ }
10500
+
10501
+ let snapshot: Uint8Array | undefined;
10502
+ let journal: Uint8Array | undefined;
10503
+ let documentSnapshot: Uint8Array | undefined;
10504
+ let documentJournal: Uint8Array | undefined;
10505
+ let documentSignerSnapshot: Uint8Array | undefined;
10506
+ let documentSignerJournal: Uint8Array | undefined;
10507
+ let journalNames = {
10508
+ coordinate: this.journalFile,
10509
+ document: this.documentJournalFile,
10510
+ signer: this.documentSignerJournalFile,
10511
+ };
10512
+ if (checkpointAuthority) {
10513
+ const authority = checkpointAuthority;
10514
+ const authorityKind = promotePendingCheckpoint
10515
+ ? "promotable pending"
10516
+ : "completed";
10517
+ const slot = this.checkpointSlot(authority.generation);
10518
+ const checkpointFile = this.checkpointFiles[slot];
10519
+ journalNames = this.checkpointJournalFiles[slot];
10520
+ const [
10521
+ checkpointBytes,
10522
+ coordinateJournal,
10523
+ valueJournal,
10524
+ signerJournal,
10525
+ ] = await Promise.all([
10526
+ this.store.read(checkpointFile),
10527
+ this.store.read(journalNames.coordinate),
10528
+ this.store.read(journalNames.document),
10529
+ this.store.read(journalNames.signer),
10530
+ ]);
10531
+ this.assertHydrating();
10532
+ try {
10533
+ if (
10534
+ !checkpointBytes ||
10535
+ checkpointBytes.byteLength !== authority.byteLength ||
10536
+ coordinateJournalChecksum(checkpointBytes) !== authority.checksum
10537
+ ) {
10538
+ throw new Error(
10539
+ `Native backbone ${authorityKind} checkpoint authority is missing or corrupt`,
10540
+ );
10541
+ }
10542
+ if (!coordinateJournal || !valueJournal || !signerJournal) {
10543
+ throw new Error(
10544
+ `Native backbone ${authorityKind} checkpoint WAL generation is incomplete`,
10545
+ );
10546
+ }
10547
+ const checkpoint = parseNativeBackboneCoordinateCheckpoint(
10548
+ checkpointBytes,
10549
+ checkpointFile,
10550
+ );
10551
+ if (
10552
+ checkpoint.generation !== authority.generation ||
10553
+ checkpoint.configurationChecksum !==
10554
+ this.checkpointConfigurationChecksum
10555
+ ) {
10556
+ throw new Error(
10557
+ `Native backbone checkpoint generation or configuration does not match its ${authorityKind} authority`,
10558
+ );
10559
+ }
10560
+ snapshot = checkpoint.coordinateSnapshot;
10561
+ documentSnapshot = checkpoint.documentSnapshot;
10562
+ documentSignerSnapshot = checkpoint.documentSignerSnapshot;
10563
+ journal = coordinateJournal;
10564
+ documentJournal = valueJournal;
10565
+ documentSignerJournal = signerJournal;
10566
+ } catch (error) {
10567
+ this.persistenceFailure ??= error;
10568
+ throw this.persistenceFailure;
10569
+ }
10570
+ } else {
10571
+ [
10572
+ snapshot,
10573
+ journal,
10574
+ documentSnapshot,
10575
+ documentJournal,
10576
+ documentSignerSnapshot,
9425
10577
  documentSignerJournal,
9426
- this.documentSignerJournalFile,
9427
- );
10578
+ ] = await Promise.all([
10579
+ this.store.read(this.snapshotFile),
10580
+ this.store.read(this.journalFile),
10581
+ this.store.read(this.documentSnapshotFile),
10582
+ this.store.read(this.documentJournalFile),
10583
+ this.store.read(this.documentSignerSnapshotFile),
10584
+ this.store.read(this.documentSignerJournalFile),
10585
+ ]);
10586
+ this.assertHydrating();
10587
+ }
10588
+ try {
10589
+ if (
10590
+ checkpointAuthority &&
10591
+ (!journal ||
10592
+ !hasCoordinateJournalMagic(journal) ||
10593
+ !documentJournal ||
10594
+ !hasCoordinateJournalMagic(documentJournal) ||
10595
+ !documentSignerJournal ||
10596
+ !hasCoordinateJournalMagic(documentSignerJournal))
10597
+ ) {
10598
+ throw new Error(
10599
+ `Native backbone ${promotePendingCheckpoint ? "promotable pending" : "completed"} checkpoint WAL generation has a missing or invalid header`,
10600
+ );
10601
+ }
10602
+ validateCoordinateJournal(journal, journalNames.coordinate);
10603
+ validateCoordinateJournal(documentJournal, journalNames.document);
10604
+ validateCoordinateJournal(documentSignerJournal, journalNames.signer);
9428
10605
  } catch (error) {
9429
10606
  this.persistenceFailure ??= error;
9430
10607
  throw this.persistenceFailure;
9431
10608
  }
10609
+ if (checkpointAuthority && !promotePendingCheckpoint) {
10610
+ try {
10611
+ // Completed authority is never permission to overwrite an ordinary
10612
+ // legacy WAL: it may contain writes made by a downgraded process in the
10613
+ // publication cut. Require the exact durable sentinel before loading.
10614
+ await this.requireLegacyMigrationSentinel();
10615
+ this.assertHydrating();
10616
+ } catch (error) {
10617
+ this.persistenceFailure ??= error;
10618
+ throw this.persistenceFailure;
10619
+ }
10620
+ }
9432
10621
  let operations: number;
9433
10622
  let documentOperations: number;
9434
10623
  let documentSignerOperations: number;
@@ -9460,12 +10649,52 @@ export class NativeBackboneCoordinatePersistence {
9460
10649
  }
9461
10650
  throw error;
9462
10651
  }
10652
+ if (promotePendingCheckpoint) {
10653
+ try {
10654
+ const authority = checkpointAuthority!;
10655
+ if (!this.store.atomicReplace) {
10656
+ throw new Error(
10657
+ "Native backbone pending checkpoint can not publish completed authority atomically",
10658
+ );
10659
+ }
10660
+ await this.store.atomicReplace(
10661
+ this.checkpointStateFile,
10662
+ encodeNativeBackboneCoordinateCheckpointState({
10663
+ configurationChecksum: this.checkpointConfigurationChecksum,
10664
+ highwater: authority.generation,
10665
+ completed: authority,
10666
+ }),
10667
+ );
10668
+ this.checkpointCompleted = authority;
10669
+ this.checkpointPending = undefined;
10670
+ this.assertHydrating();
10671
+ } catch (error) {
10672
+ this.persistenceFailure ??= error;
10673
+ throw this.persistenceFailure;
10674
+ }
10675
+ }
10676
+ if (this.checkpointCompleted || this.checkpointPending) {
10677
+ try {
10678
+ await this.cleanupRetiredCheckpointFiles();
10679
+ this.assertHydrating();
10680
+ } catch (error) {
10681
+ this.persistenceFailure ??= error;
10682
+ throw this.persistenceFailure;
10683
+ }
10684
+ }
9463
10685
  this.assertHydrating();
9464
10686
  this.journalInitialized = !!journal && journal.byteLength > 0;
10687
+ this.journalByteLength = journal?.byteLength ?? 0;
10688
+ this.journalRecordCount = operations;
9465
10689
  this.documentJournalInitialized =
9466
10690
  !!documentJournal && documentJournal.byteLength > 0;
10691
+ this.documentJournalByteLength = documentJournal?.byteLength ?? 0;
10692
+ this.documentJournalRecordCount = documentOperations;
9467
10693
  this.documentSignerJournalInitialized =
9468
10694
  !!documentSignerJournal && documentSignerJournal.byteLength > 0;
10695
+ this.documentSignerJournalByteLength =
10696
+ documentSignerJournal?.byteLength ?? 0;
10697
+ this.documentSignerJournalRecordCount = documentSignerOperations;
9469
10698
  backbone.setCoordinateJournalEnabled(true);
9470
10699
  backbone.setDocumentJournalEnabled(true);
9471
10700
  backbone.setDocumentSignerJournalEnabled(true);
@@ -9508,15 +10737,19 @@ export class NativeBackboneCoordinatePersistence {
9508
10737
  return false;
9509
10738
  }
9510
10739
  let tombstone: NativeBackboneCoordinateDropTombstoneBody;
10740
+ let expandedFiles: string[];
9511
10741
  try {
9512
10742
  tombstone = parseNativeBackboneCoordinateDropTombstone(bytes);
9513
- for (const file of this.configuredFiles()) {
10743
+ for (const file of this.legacyConfiguredFiles()) {
9514
10744
  if (!tombstone.files.includes(file)) {
9515
10745
  throw new Error(
9516
10746
  "Native backbone drop tombstone does not cover the configured namespace",
9517
10747
  );
9518
10748
  }
9519
10749
  }
10750
+ expandedFiles = [
10751
+ ...new Set([...tombstone.files, ...this.configuredFiles()]),
10752
+ ];
9520
10753
  } catch (error) {
9521
10754
  // Corruption must never hydrate stale files, but an explicit drop must
9522
10755
  // remain available to overwrite the bad marker and erase the namespace.
@@ -9524,6 +10757,17 @@ export class NativeBackboneCoordinatePersistence {
9524
10757
  this.persistenceLifecycle = this.closePromise ? "closing" : "active";
9525
10758
  throw this.persistenceFailure;
9526
10759
  }
10760
+ if (expandedFiles.length !== tombstone.files.length) {
10761
+ // Older valid markers only knew the six snapshot/WAL files. Widen the
10762
+ // destructive intent durably before erasing any checkpoint generation,
10763
+ // while preserving additional namespace files recorded by that version.
10764
+ await this.persistDropTombstone(expandedFiles, true);
10765
+ tombstone = { ...tombstone, files: expandedFiles };
10766
+ }
10767
+ // A prior replacement attempt may have made the expanded bytes visible before
10768
+ // its durability barrier rejected. Cross a fresh barrier on every recovery
10769
+ // before allowing any target removal.
10770
+ await this.barrierFile(nativeBackboneCoordinateDropTombstoneFile);
9527
10771
  await this.eraseDropFiles(tombstone.files);
9528
10772
  this.resetJournalTracking();
9529
10773
  this.persistenceLifecycle =
@@ -9580,7 +10824,13 @@ export class NativeBackboneCoordinatePersistence {
9580
10824
  this.assertActive("flush");
9581
10825
  // Serialized with compact() so a flush never clears records appended to
9582
10826
  // the wasm journal while a previous flush was awaiting its disk write.
9583
- return this.enqueuePersistence(() => this.flushJournalInternal(backbone));
10827
+ return this.enqueuePersistence(() => {
10828
+ // A call can be admitted while an earlier queued write is still in flight.
10829
+ // Re-check the sticky failure at execution time so no suffix can ACK after
10830
+ // that earlier write or authority switch failed.
10831
+ this.assertPersistenceHealthy();
10832
+ return this.flushJournalInternal(backbone);
10833
+ });
9584
10834
  }
9585
10835
 
9586
10836
  private enqueuePersistence<T>(fn: () => Promise<T>): Promise<T> {
@@ -9601,9 +10851,185 @@ export class NativeBackboneCoordinatePersistence {
9601
10851
  return next;
9602
10852
  }
9603
10853
 
10854
+ private shouldCompactJournal(
10855
+ additionalBytes: number,
10856
+ additionalRecords: number,
10857
+ ): boolean {
10858
+ return (
10859
+ (this.compactMaxJournalBytes != null &&
10860
+ this.journalByteLength +
10861
+ this.documentJournalByteLength +
10862
+ this.documentSignerJournalByteLength +
10863
+ additionalBytes >=
10864
+ this.compactMaxJournalBytes) ||
10865
+ (this.compactMaxJournalRecords != null &&
10866
+ this.journalRecordCount +
10867
+ this.documentJournalRecordCount +
10868
+ this.documentSignerJournalRecordCount +
10869
+ additionalRecords >=
10870
+ this.compactMaxJournalRecords)
10871
+ );
10872
+ }
10873
+
10874
+ private nextCheckpointGeneration(): bigint {
10875
+ let generation = this.checkpointHighwater + 1n;
10876
+ const active = this.checkpointCompleted;
10877
+ if (
10878
+ active &&
10879
+ this.checkpointSlot(generation) === this.checkpointSlot(active.generation)
10880
+ ) {
10881
+ generation += 1n;
10882
+ }
10883
+ if (generation > nativeBackboneCoordinateMaxU64) {
10884
+ throw new RangeError(
10885
+ "Native backbone checkpoint generation highwater is exhausted",
10886
+ );
10887
+ }
10888
+ return generation;
10889
+ }
10890
+
10891
+ private async publishCheckpoint(checkpoint: {
10892
+ coordinateSnapshot: Uint8Array;
10893
+ documentSnapshot: Uint8Array;
10894
+ documentSignerSnapshot: Uint8Array;
10895
+ }): Promise<void> {
10896
+ if (
10897
+ !this.crashSafeCompaction ||
10898
+ !this.store.atomicReplace ||
10899
+ !this.store.remove
10900
+ ) {
10901
+ throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
10902
+ }
10903
+ const generation = this.nextCheckpointGeneration();
10904
+ const slot = this.checkpointSlot(generation);
10905
+ const checkpointBytes = encodeNativeBackboneCoordinateCheckpoint({
10906
+ configurationChecksum: this.checkpointConfigurationChecksum,
10907
+ generation,
10908
+ ...checkpoint,
10909
+ });
10910
+ const authority: NativeBackboneCoordinateCheckpointAuthority = {
10911
+ generation,
10912
+ byteLength: checkpointBytes.byteLength,
10913
+ checksum: coordinateJournalChecksum(checkpointBytes),
10914
+ };
10915
+ const pendingState: NativeBackboneCoordinateCheckpointState = {
10916
+ configurationChecksum: this.checkpointConfigurationChecksum,
10917
+ highwater: generation,
10918
+ pending: authority,
10919
+ completed: this.checkpointCompleted,
10920
+ };
10921
+ await this.store.atomicReplace(
10922
+ this.checkpointStateFile,
10923
+ encodeNativeBackboneCoordinateCheckpointState(pendingState),
10924
+ );
10925
+ this.checkpointHighwater = generation;
10926
+ this.checkpointPending = authority;
10927
+
10928
+ const targetJournals = this.checkpointJournalFiles[slot];
10929
+ for (const file of [
10930
+ targetJournals.coordinate,
10931
+ targetJournals.document,
10932
+ targetJournals.signer,
10933
+ ]) {
10934
+ await this.store.remove(file);
10935
+ }
10936
+ const journalHeaders = [
10937
+ [targetJournals.coordinate, nativeBackboneCoordinateJournalMagic],
10938
+ [targetJournals.document, nativeBackboneCoordinateJournalMagic],
10939
+ [targetJournals.signer, nativeBackboneCoordinateJournalMagic],
10940
+ ] as const;
10941
+ for (const [file, header] of journalHeaders) {
10942
+ await this.store.write(file, header);
10943
+ await this.barrierFile(file);
10944
+ }
10945
+ await this.store.atomicReplace(this.checkpointFiles[slot], checkpointBytes);
10946
+
10947
+ // A prior release does not understand checkpoint authority. The sentinel is
10948
+ // the first-generation publication boundary: install and durably revalidate
10949
+ // it before completed authority becomes visible. On later generations it
10950
+ // must already be exact; never overwrite possible writes from a downgrade.
10951
+ if (this.checkpointCompleted) {
10952
+ await this.requireLegacyMigrationSentinel();
10953
+ } else {
10954
+ await this.installLegacyMigrationSentinel();
10955
+ }
10956
+
10957
+ const completedState: NativeBackboneCoordinateCheckpointState = {
10958
+ configurationChecksum: this.checkpointConfigurationChecksum,
10959
+ highwater: generation,
10960
+ completed: authority,
10961
+ };
10962
+ await this.store.atomicReplace(
10963
+ this.checkpointStateFile,
10964
+ encodeNativeBackboneCoordinateCheckpointState(completedState),
10965
+ );
10966
+
10967
+ // The completed authority is now the only writable generation. Update all
10968
+ // in-memory routing synchronously before another queued flush can start.
10969
+ this.checkpointCompleted = authority;
10970
+ this.checkpointPending = undefined;
10971
+ this.journalInitialized = true;
10972
+ this.documentJournalInitialized = true;
10973
+ this.documentSignerJournalInitialized = true;
10974
+ this.journalByteLength = nativeBackboneCoordinateJournalMagic.byteLength;
10975
+ this.documentJournalByteLength =
10976
+ nativeBackboneCoordinateJournalMagic.byteLength;
10977
+ this.documentSignerJournalByteLength =
10978
+ nativeBackboneCoordinateJournalMagic.byteLength;
10979
+ this.journalRecordCount = 0;
10980
+ this.documentJournalRecordCount = 0;
10981
+ this.documentSignerJournalRecordCount = 0;
10982
+ }
10983
+
10984
+ private async cleanupRetiredCheckpointFiles(): Promise<void> {
10985
+ if (
10986
+ !this.store.remove ||
10987
+ (!this.checkpointCompleted && !this.checkpointPending)
10988
+ ) {
10989
+ return;
10990
+ }
10991
+ const retiredSlot: NativeBackboneCoordinateCheckpointSlot = this
10992
+ .checkpointCompleted
10993
+ ? this.checkpointSlot(this.checkpointCompleted.generation) === "a"
10994
+ ? "b"
10995
+ : "a"
10996
+ : this.checkpointSlot(this.checkpointPending!.generation);
10997
+ const retiredJournals = this.checkpointJournalFiles[retiredSlot];
10998
+ const files = [
10999
+ ...(this.checkpointCompleted
11000
+ ? [
11001
+ this.snapshotFile,
11002
+ this.documentSnapshotFile,
11003
+ this.documentJournalFile,
11004
+ this.documentSignerSnapshotFile,
11005
+ this.documentSignerJournalFile,
11006
+ ]
11007
+ : []),
11008
+ this.checkpointFiles[retiredSlot],
11009
+ `${this.checkpointFiles[retiredSlot]}.tmp`,
11010
+ retiredJournals.coordinate,
11011
+ retiredJournals.document,
11012
+ retiredJournals.signer,
11013
+ ];
11014
+ const removals = await Promise.allSettled(
11015
+ files.map((file) => this.store.remove!(file)),
11016
+ );
11017
+ if (removals.every((result) => result.status === "fulfilled")) {
11018
+ try {
11019
+ await this.store.durableBarrier?.();
11020
+ } catch {
11021
+ // Cleanup is retryable debt. Publication and the caller's ACK already
11022
+ // belong to the new generation, so never turn this into an ambiguous
11023
+ // append failure after the authority switch.
11024
+ }
11025
+ }
11026
+ }
11027
+
9604
11028
  private async flushJournalInternal(
9605
11029
  backbone: NativePeerbitBackbone,
11030
+ forceCheckpoint = false,
9606
11031
  ): Promise<number> {
11032
+ this.assertPersistenceHealthy();
9607
11033
  let written = 0;
9608
11034
  let persistenceMutationStarted = false;
9609
11035
  let coordinateBytes: Uint8Array | undefined;
@@ -9615,10 +11041,42 @@ export class NativeBackboneCoordinatePersistence {
9615
11041
  const documentRecordCount = backbone.documentPendingJournalLength;
9616
11042
  const signerRecords = backbone.documentSignerJournal();
9617
11043
  const signerRecordCount = backbone.documentSignerPendingJournalLength;
11044
+ const additionalBytes =
11045
+ coordinateRecords.byteLength +
11046
+ documentRecords.byteLength +
11047
+ signerRecords.byteLength +
11048
+ (coordinateRecords.byteLength > 0 && this.journalInitialized !== true
11049
+ ? backbone.coordinateJournalHeader().byteLength
11050
+ : 0) +
11051
+ (documentRecords.byteLength > 0 &&
11052
+ this.documentJournalInitialized !== true
11053
+ ? backbone.documentJournalHeader().byteLength
11054
+ : 0) +
11055
+ (signerRecords.byteLength > 0 &&
11056
+ this.documentSignerJournalInitialized !== true
11057
+ ? backbone.documentSignerJournalHeader().byteLength
11058
+ : 0);
11059
+ const additionalRecords =
11060
+ coordinateRecordCount + documentRecordCount + signerRecordCount;
11061
+ const checkpointRequested =
11062
+ forceCheckpoint ||
11063
+ (this.crashSafeCompaction &&
11064
+ this.shouldCompactJournal(additionalBytes, additionalRecords));
11065
+ // These three synchronous copies and the journal prefixes above are one
11066
+ // logical cut. No mutation can slip into the checkpoint without also being
11067
+ // represented by one of the captured prefixes.
11068
+ const checkpoint = checkpointRequested
11069
+ ? {
11070
+ coordinateSnapshot: backbone.coordinateSnapshot(),
11071
+ documentSnapshot: backbone.documentSnapshot(),
11072
+ documentSignerSnapshot: backbone.documentSignerSnapshot(),
11073
+ }
11074
+ : undefined;
9618
11075
  try {
11076
+ const activeJournals = this.activeJournalFiles();
9619
11077
  if (coordinateRecords.byteLength > 0) {
9620
11078
  if (this.journalInitialized === undefined) {
9621
- const existing = await this.store.read(this.journalFile);
11079
+ const existing = await this.store.read(activeJournals.coordinate);
9622
11080
  this.journalInitialized = !!existing && existing.byteLength > 0;
9623
11081
  }
9624
11082
  coordinateBytes = this.journalInitialized
@@ -9627,28 +11085,26 @@ export class NativeBackboneCoordinatePersistence {
9627
11085
  backbone.coordinateJournalHeader(),
9628
11086
  coordinateRecords,
9629
11087
  ]);
9630
- await this.store.append(this.journalFile, coordinateBytes);
11088
+ await this.store.append(activeJournals.coordinate, coordinateBytes);
9631
11089
  persistenceMutationStarted = true;
9632
11090
  written += coordinateRecords.byteLength;
9633
11091
  }
9634
11092
  if (documentRecords.byteLength > 0) {
9635
11093
  if (this.documentJournalInitialized === undefined) {
9636
- const existing = await this.store.read(this.documentJournalFile);
11094
+ const existing = await this.store.read(activeJournals.document);
9637
11095
  this.documentJournalInitialized =
9638
11096
  !!existing && existing.byteLength > 0;
9639
11097
  }
9640
11098
  documentBytes = this.documentJournalInitialized
9641
11099
  ? documentRecords
9642
11100
  : concatBytes([backbone.documentJournalHeader(), documentRecords]);
9643
- await this.store.append(this.documentJournalFile, documentBytes);
11101
+ await this.store.append(activeJournals.document, documentBytes);
9644
11102
  persistenceMutationStarted = true;
9645
11103
  written += documentRecords.byteLength;
9646
11104
  }
9647
11105
  if (signerRecords.byteLength > 0) {
9648
11106
  if (this.documentSignerJournalInitialized === undefined) {
9649
- const existing = await this.store.read(
9650
- this.documentSignerJournalFile,
9651
- );
11107
+ const existing = await this.store.read(activeJournals.signer);
9652
11108
  this.documentSignerJournalInitialized =
9653
11109
  !!existing && existing.byteLength > 0;
9654
11110
  }
@@ -9658,11 +11114,11 @@ export class NativeBackboneCoordinatePersistence {
9658
11114
  backbone.documentSignerJournalHeader(),
9659
11115
  signerRecords,
9660
11116
  ]);
9661
- await this.store.append(this.documentSignerJournalFile, signerBytes);
11117
+ await this.store.append(activeJournals.signer, signerBytes);
9662
11118
  persistenceMutationStarted = true;
9663
11119
  written += signerRecords.byteLength;
9664
11120
  }
9665
- if (written === 0) {
11121
+ if (written === 0 && !checkpoint) {
9666
11122
  this.lastFlushMs = Date.now();
9667
11123
  return 0;
9668
11124
  }
@@ -9670,27 +11126,34 @@ export class NativeBackboneCoordinatePersistence {
9670
11126
  // `append` may only enqueue bytes in a buffered store. Drain and fsync
9671
11127
  // every affected WAL before clearing its wasm prefix or returning an ACK.
9672
11128
  for (const file of [
9673
- coordinateBytes ? this.journalFile : undefined,
9674
- documentBytes ? this.documentJournalFile : undefined,
9675
- signerBytes ? this.documentSignerJournalFile : undefined,
11129
+ coordinateBytes ? activeJournals.coordinate : undefined,
11130
+ documentBytes ? activeJournals.document : undefined,
11131
+ signerBytes ? activeJournals.signer : undefined,
9676
11132
  ]) {
9677
11133
  if (file) {
9678
- if (this.store.durableBarrier) {
9679
- await this.store.durableBarrier(file);
9680
- } else {
9681
- await this.store.flush?.(file);
9682
- }
11134
+ await this.barrierFile(file);
9683
11135
  }
9684
11136
  }
9685
11137
 
9686
11138
  if (coordinateBytes) {
9687
11139
  this.journalInitialized = true;
11140
+ this.journalByteLength += coordinateBytes.byteLength;
11141
+ this.journalRecordCount += coordinateRecordCount;
9688
11142
  }
9689
11143
  if (documentBytes) {
9690
11144
  this.documentJournalInitialized = true;
11145
+ this.documentJournalByteLength += documentBytes.byteLength;
11146
+ this.documentJournalRecordCount += documentRecordCount;
9691
11147
  }
9692
11148
  if (signerBytes) {
9693
11149
  this.documentSignerJournalInitialized = true;
11150
+ this.documentSignerJournalByteLength += signerBytes.byteLength;
11151
+ this.documentSignerJournalRecordCount += signerRecordCount;
11152
+ }
11153
+
11154
+ if (checkpoint) {
11155
+ persistenceMutationStarted = true;
11156
+ await this.publishCheckpoint(checkpoint);
9694
11157
  }
9695
11158
 
9696
11159
  backbone.clearCoordinateJournalPrefix(
@@ -9705,6 +11168,9 @@ export class NativeBackboneCoordinatePersistence {
9705
11168
  signerRecords.byteLength,
9706
11169
  signerRecordCount,
9707
11170
  );
11171
+ if (checkpoint) {
11172
+ await this.cleanupRetiredCheckpointFiles();
11173
+ }
9708
11174
  this.lastFlushMs = Date.now();
9709
11175
  return written;
9710
11176
  } catch (error) {
@@ -9716,9 +11182,15 @@ export class NativeBackboneCoordinatePersistence {
9716
11182
  }
9717
11183
  }
9718
11184
 
9719
- async compact(_backbone: NativePeerbitBackbone): Promise<void> {
11185
+ async compact(backbone: NativePeerbitBackbone): Promise<void> {
9720
11186
  this.assertActive("compact");
9721
- throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
11187
+ if (!this.crashSafeCompaction) {
11188
+ throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
11189
+ }
11190
+ await this.enqueuePersistence(async () => {
11191
+ this.assertPersistenceHealthy();
11192
+ await this.flushJournalInternal(backbone, true);
11193
+ });
9722
11194
  }
9723
11195
 
9724
11196
  close(): Promise<void> {