@peerbit/native-backbone 0.2.9 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -20
- package/dist/src/index.d.ts +61 -8
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +948 -74
- package/dist/src/index.js.map +1 -1
- package/dist/wasm/README.md +55 -20
- package/dist/wasm/native_backbone.d.ts +124 -79
- package/dist/wasm/native_backbone.js +297 -0
- package/dist/wasm/native_backbone_bg.wasm +0 -0
- package/dist/wasm/native_backbone_bg.wasm.d.ts +90 -79
- package/package.json +2 -2
- package/src/append_tx/committed_latest.rs +10 -5
- package/src/append_tx/committed_no_next.rs +6 -3
- package/src/append_tx/facts.rs +427 -1
- package/src/append_tx/mod.rs +51 -18
- package/src/append_tx/storage.rs +12 -2
- package/src/index.ts +1418 -227
- package/src/shared_log_plan.rs +30 -0
package/src/index.ts
CHANGED
|
@@ -9,6 +9,94 @@ export * from "./durability/storage.js";
|
|
|
9
9
|
|
|
10
10
|
export type RangeResolution = "u32" | "u64";
|
|
11
11
|
|
|
12
|
+
type NativePrepareNoNextDocumentIndexCompactTrimFacts = (
|
|
13
|
+
wallTime: bigint,
|
|
14
|
+
logical: number,
|
|
15
|
+
gid: string,
|
|
16
|
+
type: number,
|
|
17
|
+
metaData: Uint8Array | undefined,
|
|
18
|
+
payloadData: Uint8Array,
|
|
19
|
+
trimLengthTo: number,
|
|
20
|
+
documentKey: string,
|
|
21
|
+
documentValuePrefixBytes: Uint8Array,
|
|
22
|
+
documentExistingCreated: string,
|
|
23
|
+
documentByteElementIndexLimit: number,
|
|
24
|
+
documentDeleteTrimmedHeads: boolean,
|
|
25
|
+
documentProjectionPlan:
|
|
26
|
+
| NativeBackboneSimpleDocumentProjectionPlan
|
|
27
|
+
| undefined,
|
|
28
|
+
documentProjectionEncodedDocument: Uint8Array | undefined,
|
|
29
|
+
documentProjectionSigner: Uint8Array | undefined,
|
|
30
|
+
) => unknown[];
|
|
31
|
+
|
|
32
|
+
type NativePrepareNoNextCachedDocumentIndexCompactTrimFacts = (
|
|
33
|
+
wallTime: bigint,
|
|
34
|
+
logical: number,
|
|
35
|
+
gid: string,
|
|
36
|
+
type: number,
|
|
37
|
+
metaData: Uint8Array | undefined,
|
|
38
|
+
payloadData: Uint8Array,
|
|
39
|
+
trimLengthTo: number,
|
|
40
|
+
documentKey: string,
|
|
41
|
+
documentExistingCreated: string,
|
|
42
|
+
documentByteElementIndexLimit: number,
|
|
43
|
+
documentDeleteTrimmedHeads: boolean,
|
|
44
|
+
documentProjectionPlanId: number,
|
|
45
|
+
documentProjectionEncodedDocument: Uint8Array,
|
|
46
|
+
documentProjectionSigner: Uint8Array | undefined,
|
|
47
|
+
) => unknown[];
|
|
48
|
+
|
|
49
|
+
type NativePrepareNoNextCachedPlainPutPayloadDocumentIndexCompactTrimFacts = (
|
|
50
|
+
wallTime: bigint,
|
|
51
|
+
logical: number,
|
|
52
|
+
gid: string,
|
|
53
|
+
type: number,
|
|
54
|
+
metaData: Uint8Array | undefined,
|
|
55
|
+
payloadData: Uint8Array,
|
|
56
|
+
trimLengthTo: number,
|
|
57
|
+
documentKey: string,
|
|
58
|
+
documentExistingCreated: string,
|
|
59
|
+
documentByteElementIndexLimit: number,
|
|
60
|
+
documentDeleteTrimmedHeads: boolean,
|
|
61
|
+
documentProjectionPlanId: number,
|
|
62
|
+
documentProjectionSigner: Uint8Array | undefined,
|
|
63
|
+
) => unknown[];
|
|
64
|
+
|
|
65
|
+
type NativePrepareLatestDocumentIndexTrimFacts = (
|
|
66
|
+
wallTime: bigint,
|
|
67
|
+
logical: number,
|
|
68
|
+
gid: string,
|
|
69
|
+
type: number,
|
|
70
|
+
metaData: Uint8Array | undefined,
|
|
71
|
+
payloadData: Uint8Array,
|
|
72
|
+
trimLengthTo: number | undefined,
|
|
73
|
+
documentKey: string,
|
|
74
|
+
documentValuePrefixBytes: Uint8Array,
|
|
75
|
+
documentByteElementIndexLimit: number,
|
|
76
|
+
documentDeleteTrimmedHeads: boolean,
|
|
77
|
+
documentProjectionPlan:
|
|
78
|
+
| NativeBackboneSimpleDocumentProjectionPlan
|
|
79
|
+
| undefined,
|
|
80
|
+
documentProjectionEncodedDocument: Uint8Array | undefined,
|
|
81
|
+
documentProjectionSigner: Uint8Array | undefined,
|
|
82
|
+
) => unknown[];
|
|
83
|
+
|
|
84
|
+
type NativePrepareLatestCachedDocumentIndexTrimFacts = (
|
|
85
|
+
wallTime: bigint,
|
|
86
|
+
logical: number,
|
|
87
|
+
gid: string,
|
|
88
|
+
type: number,
|
|
89
|
+
metaData: Uint8Array | undefined,
|
|
90
|
+
payloadData: Uint8Array,
|
|
91
|
+
trimLengthTo: number | undefined,
|
|
92
|
+
documentKey: string,
|
|
93
|
+
documentByteElementIndexLimit: number,
|
|
94
|
+
documentDeleteTrimmedHeads: boolean,
|
|
95
|
+
documentProjectionPlanId: number,
|
|
96
|
+
documentProjectionEncodedDocument: Uint8Array,
|
|
97
|
+
documentProjectionSigner: Uint8Array | undefined,
|
|
98
|
+
) => unknown[];
|
|
99
|
+
|
|
12
100
|
type NativePeerbitBackboneHandle = {
|
|
13
101
|
log_len: () => number;
|
|
14
102
|
block_len: () => number;
|
|
@@ -618,6 +706,17 @@ type NativePeerbitBackboneHandle = {
|
|
|
618
706
|
minReplicas: number,
|
|
619
707
|
selfHash: string,
|
|
620
708
|
) => string[];
|
|
709
|
+
get_routing_full_replica_leaders?: (
|
|
710
|
+
replicas: number,
|
|
711
|
+
roleAgeMs: number,
|
|
712
|
+
now: string,
|
|
713
|
+
peerFilter: string[] | undefined,
|
|
714
|
+
expandPeerFilter: boolean,
|
|
715
|
+
selfHash: string,
|
|
716
|
+
includeSelf: boolean,
|
|
717
|
+
fullReplicaFallback: boolean,
|
|
718
|
+
includeStrictFullReplica: boolean,
|
|
719
|
+
) => unknown[] | undefined;
|
|
621
720
|
put_range: (
|
|
622
721
|
id: string,
|
|
623
722
|
hash: string,
|
|
@@ -942,6 +1041,16 @@ type NativePeerbitBackboneHandle = {
|
|
|
942
1041
|
payloadData: Uint8Array,
|
|
943
1042
|
trimLengthTo: number | undefined,
|
|
944
1043
|
) => unknown[];
|
|
1044
|
+
prepare_plain_entry_commit_facts_trim_refs?: (
|
|
1045
|
+
wallTime: bigint,
|
|
1046
|
+
logical: number,
|
|
1047
|
+
gid: string,
|
|
1048
|
+
next: string[],
|
|
1049
|
+
type: number,
|
|
1050
|
+
metaData: Uint8Array | undefined,
|
|
1051
|
+
payloadData: Uint8Array,
|
|
1052
|
+
trimLengthTo: number,
|
|
1053
|
+
) => unknown[];
|
|
945
1054
|
prepare_plain_entry_commit_facts_document_index: (
|
|
946
1055
|
wallTime: bigint,
|
|
947
1056
|
logical: number,
|
|
@@ -1045,25 +1154,8 @@ type NativePeerbitBackboneHandle = {
|
|
|
1045
1154
|
documentProjectionEncodedDocument: Uint8Array | undefined,
|
|
1046
1155
|
documentProjectionSigner: Uint8Array | undefined,
|
|
1047
1156
|
) => unknown[];
|
|
1048
|
-
prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_hashes:
|
|
1049
|
-
|
|
1050
|
-
logical: number,
|
|
1051
|
-
gid: string,
|
|
1052
|
-
type: number,
|
|
1053
|
-
metaData: Uint8Array | undefined,
|
|
1054
|
-
payloadData: Uint8Array,
|
|
1055
|
-
trimLengthTo: number,
|
|
1056
|
-
documentKey: string,
|
|
1057
|
-
documentValuePrefixBytes: Uint8Array,
|
|
1058
|
-
documentExistingCreated: string,
|
|
1059
|
-
documentByteElementIndexLimit: number,
|
|
1060
|
-
documentDeleteTrimmedHeads: boolean,
|
|
1061
|
-
documentProjectionPlan:
|
|
1062
|
-
| NativeBackboneSimpleDocumentProjectionPlan
|
|
1063
|
-
| undefined,
|
|
1064
|
-
documentProjectionEncodedDocument: Uint8Array | undefined,
|
|
1065
|
-
documentProjectionSigner: Uint8Array | undefined,
|
|
1066
|
-
) => unknown[];
|
|
1157
|
+
prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_hashes: NativePrepareNoNextDocumentIndexCompactTrimFacts;
|
|
1158
|
+
prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_refs?: NativePrepareNoNextDocumentIndexCompactTrimFacts;
|
|
1067
1159
|
prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_trim_hashes: (
|
|
1068
1160
|
wallTime: bigint,
|
|
1069
1161
|
logical: number,
|
|
@@ -1080,70 +1172,14 @@ type NativePeerbitBackboneHandle = {
|
|
|
1080
1172
|
documentProjectionEncodedDocument: Uint8Array,
|
|
1081
1173
|
documentProjectionSigner: Uint8Array | undefined,
|
|
1082
1174
|
) => unknown[];
|
|
1083
|
-
prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes:
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
documentKey: string,
|
|
1092
|
-
documentExistingCreated: string,
|
|
1093
|
-
documentByteElementIndexLimit: number,
|
|
1094
|
-
documentDeleteTrimmedHeads: boolean,
|
|
1095
|
-
documentProjectionPlanId: number,
|
|
1096
|
-
documentProjectionEncodedDocument: Uint8Array,
|
|
1097
|
-
documentProjectionSigner: Uint8Array | undefined,
|
|
1098
|
-
) => unknown[];
|
|
1099
|
-
prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes_plain_put_payload?: (
|
|
1100
|
-
wallTime: bigint,
|
|
1101
|
-
logical: number,
|
|
1102
|
-
gid: string,
|
|
1103
|
-
type: number,
|
|
1104
|
-
metaData: Uint8Array | undefined,
|
|
1105
|
-
payloadData: Uint8Array,
|
|
1106
|
-
trimLengthTo: number,
|
|
1107
|
-
documentKey: string,
|
|
1108
|
-
documentExistingCreated: string,
|
|
1109
|
-
documentByteElementIndexLimit: number,
|
|
1110
|
-
documentDeleteTrimmedHeads: boolean,
|
|
1111
|
-
documentProjectionPlanId: number,
|
|
1112
|
-
documentProjectionSigner: Uint8Array | undefined,
|
|
1113
|
-
) => unknown[];
|
|
1114
|
-
prepare_plain_entry_commit_latest_facts_document_index_trim_hashes: (
|
|
1115
|
-
wallTime: bigint,
|
|
1116
|
-
logical: number,
|
|
1117
|
-
gid: string,
|
|
1118
|
-
type: number,
|
|
1119
|
-
metaData: Uint8Array | undefined,
|
|
1120
|
-
payloadData: Uint8Array,
|
|
1121
|
-
trimLengthTo: number | undefined,
|
|
1122
|
-
documentKey: string,
|
|
1123
|
-
documentValuePrefixBytes: Uint8Array,
|
|
1124
|
-
documentByteElementIndexLimit: number,
|
|
1125
|
-
documentDeleteTrimmedHeads: boolean,
|
|
1126
|
-
documentProjectionPlan:
|
|
1127
|
-
| NativeBackboneSimpleDocumentProjectionPlan
|
|
1128
|
-
| undefined,
|
|
1129
|
-
documentProjectionEncodedDocument: Uint8Array | undefined,
|
|
1130
|
-
documentProjectionSigner: Uint8Array | undefined,
|
|
1131
|
-
) => unknown[];
|
|
1132
|
-
prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes: (
|
|
1133
|
-
wallTime: bigint,
|
|
1134
|
-
logical: number,
|
|
1135
|
-
gid: string,
|
|
1136
|
-
type: number,
|
|
1137
|
-
metaData: Uint8Array | undefined,
|
|
1138
|
-
payloadData: Uint8Array,
|
|
1139
|
-
trimLengthTo: number | undefined,
|
|
1140
|
-
documentKey: string,
|
|
1141
|
-
documentByteElementIndexLimit: number,
|
|
1142
|
-
documentDeleteTrimmedHeads: boolean,
|
|
1143
|
-
documentProjectionPlanId: number,
|
|
1144
|
-
documentProjectionEncodedDocument: Uint8Array,
|
|
1145
|
-
documentProjectionSigner: Uint8Array | undefined,
|
|
1146
|
-
) => unknown[];
|
|
1175
|
+
prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes: NativePrepareNoNextCachedDocumentIndexCompactTrimFacts;
|
|
1176
|
+
prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs?: NativePrepareNoNextCachedDocumentIndexCompactTrimFacts;
|
|
1177
|
+
prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes_plain_put_payload?: NativePrepareNoNextCachedPlainPutPayloadDocumentIndexCompactTrimFacts;
|
|
1178
|
+
prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs_plain_put_payload?: NativePrepareNoNextCachedPlainPutPayloadDocumentIndexCompactTrimFacts;
|
|
1179
|
+
prepare_plain_entry_commit_latest_facts_document_index_trim_hashes: NativePrepareLatestDocumentIndexTrimFacts;
|
|
1180
|
+
prepare_plain_entry_commit_latest_facts_document_index_trim_refs?: NativePrepareLatestDocumentIndexTrimFacts;
|
|
1181
|
+
prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes: NativePrepareLatestCachedDocumentIndexTrimFacts;
|
|
1182
|
+
prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_refs?: NativePrepareLatestCachedDocumentIndexTrimFacts;
|
|
1147
1183
|
prepare_plain_entry_storage_facts_and_put: (
|
|
1148
1184
|
wallTime: bigint,
|
|
1149
1185
|
logical: number,
|
|
@@ -2727,6 +2763,7 @@ export type NativeBackboneAppendResult = {
|
|
|
2727
2763
|
assignedToRangeBoundary: boolean;
|
|
2728
2764
|
trimmed: NativeBackboneTrimmedEntry[];
|
|
2729
2765
|
trimmedHashes?: string[];
|
|
2766
|
+
trimmedGids?: string[];
|
|
2730
2767
|
documentTrimmedHeadsProcessed?: boolean;
|
|
2731
2768
|
documentPreviousContext?: NativeBackboneDocumentContextFacts;
|
|
2732
2769
|
};
|
|
@@ -2739,6 +2776,7 @@ type NativeBackboneStorageAppendResult = {
|
|
|
2739
2776
|
assignedToRangeBoundary: boolean;
|
|
2740
2777
|
trimmed: NativeBackboneTrimmedEntry[];
|
|
2741
2778
|
trimmedHashes?: string[];
|
|
2779
|
+
trimmedGids?: string[];
|
|
2742
2780
|
documentTrimmedHeadsProcessed?: boolean;
|
|
2743
2781
|
documentPreviousContext?: NativeBackboneDocumentContextFacts;
|
|
2744
2782
|
};
|
|
@@ -2982,6 +3020,12 @@ export type NativeBackboneCoordinatePersistenceStore = {
|
|
|
2982
3020
|
*/
|
|
2983
3021
|
readLimited?(name: string, maxBytes: number): Promise<Uint8Array | undefined>;
|
|
2984
3022
|
write(name: string, bytes: Uint8Array): Promise<void>;
|
|
3023
|
+
/**
|
|
3024
|
+
* Durably replace one named file without exposing a torn destination. The
|
|
3025
|
+
* replacement is not visible until its bytes and containing directory have
|
|
3026
|
+
* crossed their physical durability barriers.
|
|
3027
|
+
*/
|
|
3028
|
+
atomicReplace?(name: string, bytes: Uint8Array): Promise<void>;
|
|
2985
3029
|
/**
|
|
2986
3030
|
* Appends `bytes` to the named file. Callers hand over ownership of
|
|
2987
3031
|
* `bytes` and must not mutate it afterwards: buffering stores keep the
|
|
@@ -3097,12 +3141,12 @@ export const nativeBackboneCoordinateDropTombstoneFile =
|
|
|
3097
3141
|
"native-backbone-drop.tombstone";
|
|
3098
3142
|
|
|
3099
3143
|
export const defaultNativeBackboneCoordinateFlushMaxPendingBytes = 1024 * 1024;
|
|
3100
|
-
/**
|
|
3144
|
+
/** Recommended explicit byte threshold for crash-safe Node checkpointing. */
|
|
3101
3145
|
export const defaultNativeBackboneCoordinateCompactMaxJournalBytes =
|
|
3102
3146
|
64 * 1024 * 1024;
|
|
3103
3147
|
|
|
3104
3148
|
const nativeBackboneCoordinateCompactionDisabledMessage =
|
|
3105
|
-
"Native backbone coordinate persistence compaction is disabled
|
|
3149
|
+
"Native backbone coordinate persistence compaction is disabled for stores without crash-safe atomic replacement";
|
|
3106
3150
|
|
|
3107
3151
|
const nativeBackboneCoordinateJournalMagic = Uint8Array.from([
|
|
3108
3152
|
0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x57, 0x31,
|
|
@@ -3123,6 +3167,260 @@ const coordinateJournalChecksum = (bytes: Uint8Array): number => {
|
|
|
3123
3167
|
return checksum;
|
|
3124
3168
|
};
|
|
3125
3169
|
|
|
3170
|
+
type NativeBackboneCoordinateCheckpointSlot = "a" | "b";
|
|
3171
|
+
|
|
3172
|
+
type NativeBackboneCoordinateCheckpointAuthority = {
|
|
3173
|
+
generation: bigint;
|
|
3174
|
+
byteLength: number;
|
|
3175
|
+
checksum: number;
|
|
3176
|
+
};
|
|
3177
|
+
|
|
3178
|
+
type NativeBackboneCoordinateCheckpointState = {
|
|
3179
|
+
configurationChecksum: number;
|
|
3180
|
+
highwater: bigint;
|
|
3181
|
+
pending?: NativeBackboneCoordinateCheckpointAuthority;
|
|
3182
|
+
completed?: NativeBackboneCoordinateCheckpointAuthority;
|
|
3183
|
+
};
|
|
3184
|
+
|
|
3185
|
+
type NativeBackboneCoordinateCheckpoint = {
|
|
3186
|
+
configurationChecksum: number;
|
|
3187
|
+
generation: bigint;
|
|
3188
|
+
coordinateSnapshot: Uint8Array;
|
|
3189
|
+
documentSnapshot: Uint8Array;
|
|
3190
|
+
documentSignerSnapshot: Uint8Array;
|
|
3191
|
+
};
|
|
3192
|
+
|
|
3193
|
+
const nativeBackboneCoordinateCheckpointMagic = Uint8Array.from([
|
|
3194
|
+
0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x43, 0x31,
|
|
3195
|
+
]);
|
|
3196
|
+
const nativeBackboneCoordinateCheckpointStateMagic = Uint8Array.from([
|
|
3197
|
+
0x50, 0x42, 0x52, 0x49, 0x44, 0x58, 0x53, 0x31,
|
|
3198
|
+
]);
|
|
3199
|
+
const nativeBackboneCoordinateLegacyMigrationSentinel = (() => {
|
|
3200
|
+
const payload = Uint8Array.of(0xff);
|
|
3201
|
+
const bytes = new Uint8Array(
|
|
3202
|
+
nativeBackboneCoordinateJournalMagic.byteLength + 8 + payload.byteLength,
|
|
3203
|
+
);
|
|
3204
|
+
bytes.set(nativeBackboneCoordinateJournalMagic, 0);
|
|
3205
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
3206
|
+
view.setUint32(nativeBackboneCoordinateJournalMagic.byteLength, 1, true);
|
|
3207
|
+
view.setUint32(
|
|
3208
|
+
nativeBackboneCoordinateJournalMagic.byteLength + 4,
|
|
3209
|
+
coordinateJournalChecksum(payload),
|
|
3210
|
+
true,
|
|
3211
|
+
);
|
|
3212
|
+
bytes.set(payload, nativeBackboneCoordinateJournalMagic.byteLength + 8);
|
|
3213
|
+
return bytes;
|
|
3214
|
+
})();
|
|
3215
|
+
const nativeBackboneCoordinateCheckpointVersion = 1;
|
|
3216
|
+
const nativeBackboneCoordinateCheckpointHeaderBytes = 56;
|
|
3217
|
+
const nativeBackboneCoordinateCheckpointStateBytes = 60;
|
|
3218
|
+
const nativeBackboneCoordinateMaxU64 = (1n << 64n) - 1n;
|
|
3219
|
+
|
|
3220
|
+
const hasBytesPrefix = (bytes: Uint8Array, prefix: Uint8Array): boolean =>
|
|
3221
|
+
bytes.byteLength >= prefix.byteLength &&
|
|
3222
|
+
prefix.every((byte, index) => bytes[index] === byte);
|
|
3223
|
+
|
|
3224
|
+
const writeCoordinateCheckpointU64 = (
|
|
3225
|
+
view: DataView,
|
|
3226
|
+
offset: number,
|
|
3227
|
+
value: bigint,
|
|
3228
|
+
): void => {
|
|
3229
|
+
if (value < 0n || value > nativeBackboneCoordinateMaxU64) {
|
|
3230
|
+
throw new RangeError("Native backbone checkpoint generation exceeds u64");
|
|
3231
|
+
}
|
|
3232
|
+
view.setUint32(offset, Number(value & 0xffff_ffffn), true);
|
|
3233
|
+
view.setUint32(offset + 4, Number(value >> 32n), true);
|
|
3234
|
+
};
|
|
3235
|
+
|
|
3236
|
+
const readCoordinateCheckpointU64 = (view: DataView, offset: number): bigint =>
|
|
3237
|
+
BigInt(view.getUint32(offset, true)) |
|
|
3238
|
+
(BigInt(view.getUint32(offset + 4, true)) << 32n);
|
|
3239
|
+
|
|
3240
|
+
const encodeNativeBackboneCoordinateCheckpoint = (
|
|
3241
|
+
checkpoint: NativeBackboneCoordinateCheckpoint,
|
|
3242
|
+
): Uint8Array => {
|
|
3243
|
+
const coordinateLength = checkpoint.coordinateSnapshot.byteLength;
|
|
3244
|
+
const documentLength = checkpoint.documentSnapshot.byteLength;
|
|
3245
|
+
const signerLength = checkpoint.documentSignerSnapshot.byteLength;
|
|
3246
|
+
const byteLength =
|
|
3247
|
+
nativeBackboneCoordinateCheckpointHeaderBytes +
|
|
3248
|
+
coordinateLength +
|
|
3249
|
+
documentLength +
|
|
3250
|
+
signerLength;
|
|
3251
|
+
if (!Number.isSafeInteger(byteLength) || byteLength > 0xffff_ffff) {
|
|
3252
|
+
throw new RangeError("Native backbone coordinate checkpoint is too large");
|
|
3253
|
+
}
|
|
3254
|
+
const bytes = new Uint8Array(byteLength);
|
|
3255
|
+
bytes.set(nativeBackboneCoordinateCheckpointMagic, 0);
|
|
3256
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
3257
|
+
view.setUint32(8, nativeBackboneCoordinateCheckpointVersion, true);
|
|
3258
|
+
view.setUint32(12, nativeBackboneCoordinateCheckpointHeaderBytes, true);
|
|
3259
|
+
writeCoordinateCheckpointU64(view, 16, checkpoint.generation);
|
|
3260
|
+
view.setUint32(24, coordinateLength, true);
|
|
3261
|
+
view.setUint32(28, documentLength, true);
|
|
3262
|
+
view.setUint32(32, signerLength, true);
|
|
3263
|
+
view.setUint32(
|
|
3264
|
+
36,
|
|
3265
|
+
coordinateJournalChecksum(checkpoint.coordinateSnapshot),
|
|
3266
|
+
true,
|
|
3267
|
+
);
|
|
3268
|
+
view.setUint32(
|
|
3269
|
+
40,
|
|
3270
|
+
coordinateJournalChecksum(checkpoint.documentSnapshot),
|
|
3271
|
+
true,
|
|
3272
|
+
);
|
|
3273
|
+
view.setUint32(
|
|
3274
|
+
44,
|
|
3275
|
+
coordinateJournalChecksum(checkpoint.documentSignerSnapshot),
|
|
3276
|
+
true,
|
|
3277
|
+
);
|
|
3278
|
+
view.setUint32(48, checkpoint.configurationChecksum, true);
|
|
3279
|
+
view.setUint32(52, coordinateJournalChecksum(bytes.subarray(8, 52)), true);
|
|
3280
|
+
let offset = nativeBackboneCoordinateCheckpointHeaderBytes;
|
|
3281
|
+
bytes.set(checkpoint.coordinateSnapshot, offset);
|
|
3282
|
+
offset += coordinateLength;
|
|
3283
|
+
bytes.set(checkpoint.documentSnapshot, offset);
|
|
3284
|
+
offset += documentLength;
|
|
3285
|
+
bytes.set(checkpoint.documentSignerSnapshot, offset);
|
|
3286
|
+
return bytes;
|
|
3287
|
+
};
|
|
3288
|
+
|
|
3289
|
+
const parseNativeBackboneCoordinateCheckpoint = (
|
|
3290
|
+
bytes: Uint8Array,
|
|
3291
|
+
name: string,
|
|
3292
|
+
): NativeBackboneCoordinateCheckpoint => {
|
|
3293
|
+
if (
|
|
3294
|
+
bytes.byteLength < nativeBackboneCoordinateCheckpointHeaderBytes ||
|
|
3295
|
+
!hasBytesPrefix(bytes, nativeBackboneCoordinateCheckpointMagic)
|
|
3296
|
+
) {
|
|
3297
|
+
throw new Error(`Native backbone ${name} has an invalid checkpoint header`);
|
|
3298
|
+
}
|
|
3299
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
3300
|
+
if (
|
|
3301
|
+
view.getUint32(8, true) !== nativeBackboneCoordinateCheckpointVersion ||
|
|
3302
|
+
view.getUint32(12, true) !==
|
|
3303
|
+
nativeBackboneCoordinateCheckpointHeaderBytes ||
|
|
3304
|
+
view.getUint32(52, true) !==
|
|
3305
|
+
coordinateJournalChecksum(bytes.subarray(8, 52))
|
|
3306
|
+
) {
|
|
3307
|
+
throw new Error(`Native backbone ${name} has invalid checkpoint metadata`);
|
|
3308
|
+
}
|
|
3309
|
+
const coordinateLength = view.getUint32(24, true);
|
|
3310
|
+
const documentLength = view.getUint32(28, true);
|
|
3311
|
+
const signerLength = view.getUint32(32, true);
|
|
3312
|
+
const expectedLength =
|
|
3313
|
+
nativeBackboneCoordinateCheckpointHeaderBytes +
|
|
3314
|
+
coordinateLength +
|
|
3315
|
+
documentLength +
|
|
3316
|
+
signerLength;
|
|
3317
|
+
if (
|
|
3318
|
+
!Number.isSafeInteger(expectedLength) ||
|
|
3319
|
+
expectedLength !== bytes.byteLength
|
|
3320
|
+
) {
|
|
3321
|
+
throw new Error(`Native backbone ${name} has an invalid checkpoint length`);
|
|
3322
|
+
}
|
|
3323
|
+
let offset = nativeBackboneCoordinateCheckpointHeaderBytes;
|
|
3324
|
+
const coordinateSnapshot = bytes.subarray(offset, offset + coordinateLength);
|
|
3325
|
+
offset += coordinateLength;
|
|
3326
|
+
const documentSnapshot = bytes.subarray(offset, offset + documentLength);
|
|
3327
|
+
offset += documentLength;
|
|
3328
|
+
const documentSignerSnapshot = bytes.subarray(offset, offset + signerLength);
|
|
3329
|
+
if (
|
|
3330
|
+
coordinateJournalChecksum(coordinateSnapshot) !==
|
|
3331
|
+
view.getUint32(36, true) ||
|
|
3332
|
+
coordinateJournalChecksum(documentSnapshot) !== view.getUint32(40, true) ||
|
|
3333
|
+
coordinateJournalChecksum(documentSignerSnapshot) !==
|
|
3334
|
+
view.getUint32(44, true)
|
|
3335
|
+
) {
|
|
3336
|
+
throw new Error(
|
|
3337
|
+
`Native backbone ${name} has a checkpoint checksum mismatch`,
|
|
3338
|
+
);
|
|
3339
|
+
}
|
|
3340
|
+
return {
|
|
3341
|
+
configurationChecksum: view.getUint32(48, true),
|
|
3342
|
+
generation: readCoordinateCheckpointU64(view, 16),
|
|
3343
|
+
coordinateSnapshot,
|
|
3344
|
+
documentSnapshot,
|
|
3345
|
+
documentSignerSnapshot,
|
|
3346
|
+
};
|
|
3347
|
+
};
|
|
3348
|
+
|
|
3349
|
+
const encodeNativeBackboneCoordinateCheckpointState = (
|
|
3350
|
+
state: NativeBackboneCoordinateCheckpointState,
|
|
3351
|
+
): Uint8Array => {
|
|
3352
|
+
const bytes = new Uint8Array(nativeBackboneCoordinateCheckpointStateBytes);
|
|
3353
|
+
bytes.set(nativeBackboneCoordinateCheckpointStateMagic, 0);
|
|
3354
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
3355
|
+
view.setUint32(8, nativeBackboneCoordinateCheckpointVersion, true);
|
|
3356
|
+
view.setUint32(12, state.configurationChecksum, true);
|
|
3357
|
+
writeCoordinateCheckpointU64(view, 16, state.highwater);
|
|
3358
|
+
writeCoordinateCheckpointU64(view, 24, state.pending?.generation ?? 0n);
|
|
3359
|
+
view.setUint32(32, state.pending?.byteLength ?? 0, true);
|
|
3360
|
+
view.setUint32(36, state.pending?.checksum ?? 0, true);
|
|
3361
|
+
writeCoordinateCheckpointU64(view, 40, state.completed?.generation ?? 0n);
|
|
3362
|
+
view.setUint32(48, state.completed?.byteLength ?? 0, true);
|
|
3363
|
+
view.setUint32(52, state.completed?.checksum ?? 0, true);
|
|
3364
|
+
view.setUint32(56, coordinateJournalChecksum(bytes.subarray(8, 56)), true);
|
|
3365
|
+
return bytes;
|
|
3366
|
+
};
|
|
3367
|
+
|
|
3368
|
+
const parseNativeBackboneCoordinateCheckpointState = (
|
|
3369
|
+
bytes: Uint8Array,
|
|
3370
|
+
name: string,
|
|
3371
|
+
): NativeBackboneCoordinateCheckpointState => {
|
|
3372
|
+
if (
|
|
3373
|
+
bytes.byteLength !== nativeBackboneCoordinateCheckpointStateBytes ||
|
|
3374
|
+
!hasBytesPrefix(bytes, nativeBackboneCoordinateCheckpointStateMagic)
|
|
3375
|
+
) {
|
|
3376
|
+
throw new Error(
|
|
3377
|
+
`Native backbone ${name} has an invalid checkpoint authority`,
|
|
3378
|
+
);
|
|
3379
|
+
}
|
|
3380
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
3381
|
+
if (
|
|
3382
|
+
view.getUint32(8, true) !== nativeBackboneCoordinateCheckpointVersion ||
|
|
3383
|
+
view.getUint32(56, true) !==
|
|
3384
|
+
coordinateJournalChecksum(bytes.subarray(8, 56))
|
|
3385
|
+
) {
|
|
3386
|
+
throw new Error(`Native backbone ${name} has corrupt checkpoint authority`);
|
|
3387
|
+
}
|
|
3388
|
+
const configurationChecksum = view.getUint32(12, true);
|
|
3389
|
+
const highwater = readCoordinateCheckpointU64(view, 16);
|
|
3390
|
+
const pendingGeneration = readCoordinateCheckpointU64(view, 24);
|
|
3391
|
+
const completedGeneration = readCoordinateCheckpointU64(view, 40);
|
|
3392
|
+
const authority = (
|
|
3393
|
+
generation: bigint,
|
|
3394
|
+
lengthOffset: number,
|
|
3395
|
+
checksumOffset: number,
|
|
3396
|
+
): NativeBackboneCoordinateCheckpointAuthority | undefined => {
|
|
3397
|
+
const byteLength = view.getUint32(lengthOffset, true);
|
|
3398
|
+
const checksum = view.getUint32(checksumOffset, true);
|
|
3399
|
+
if (generation === 0n) {
|
|
3400
|
+
if (byteLength !== 0 || checksum !== 0) {
|
|
3401
|
+
throw new Error(`Native backbone ${name} has empty authority metadata`);
|
|
3402
|
+
}
|
|
3403
|
+
return undefined;
|
|
3404
|
+
}
|
|
3405
|
+
if (byteLength < nativeBackboneCoordinateCheckpointHeaderBytes) {
|
|
3406
|
+
throw new Error(`Native backbone ${name} has invalid authority length`);
|
|
3407
|
+
}
|
|
3408
|
+
return { generation, byteLength, checksum };
|
|
3409
|
+
};
|
|
3410
|
+
const pending = authority(pendingGeneration, 32, 36);
|
|
3411
|
+
const completed = authority(completedGeneration, 48, 52);
|
|
3412
|
+
if (
|
|
3413
|
+
(completed && completed.generation > highwater) ||
|
|
3414
|
+
(pending && pending.generation !== highwater) ||
|
|
3415
|
+
(pending && completed && pending.generation <= completed.generation) ||
|
|
3416
|
+
(!pending && completed && completed.generation !== highwater) ||
|
|
3417
|
+
(!pending && !completed && highwater !== 0n)
|
|
3418
|
+
) {
|
|
3419
|
+
throw new Error(`Native backbone ${name} has invalid generation authority`);
|
|
3420
|
+
}
|
|
3421
|
+
return { configurationChecksum, highwater, pending, completed };
|
|
3422
|
+
};
|
|
3423
|
+
|
|
3126
3424
|
const hasCoordinateJournalMagic = (bytes: Uint8Array): boolean =>
|
|
3127
3425
|
bytes.byteLength >= nativeBackboneCoordinateJournalMagic.byteLength &&
|
|
3128
3426
|
nativeBackboneCoordinateJournalMagic.every(
|
|
@@ -3172,11 +3470,24 @@ const resolveCoordinateFlushMaxPendingBytes = (
|
|
|
3172
3470
|
? defaultNativeBackboneCoordinateFlushMaxPendingBytes
|
|
3173
3471
|
: undefined;
|
|
3174
3472
|
|
|
3473
|
+
const validateCoordinateCompactionThreshold = (
|
|
3474
|
+
value: number,
|
|
3475
|
+
name: string,
|
|
3476
|
+
): number => {
|
|
3477
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
3478
|
+
throw new RangeError(
|
|
3479
|
+
`Native backbone ${name} must be a positive safe integer`,
|
|
3480
|
+
);
|
|
3481
|
+
}
|
|
3482
|
+
return value;
|
|
3483
|
+
};
|
|
3484
|
+
|
|
3175
3485
|
type NativeBackboneNodeFs = {
|
|
3176
3486
|
mkdir(path: string, options?: { recursive?: boolean }): Promise<unknown>;
|
|
3177
3487
|
readFile(path: string): Promise<Uint8Array>;
|
|
3178
3488
|
writeFile(path: string, data: Uint8Array): Promise<unknown>;
|
|
3179
3489
|
appendFile(path: string, data: Uint8Array): Promise<unknown>;
|
|
3490
|
+
rename?(from: string, to: string): Promise<unknown>;
|
|
3180
3491
|
/** Explicit capability for pre-allocation-bounded positional reads. */
|
|
3181
3492
|
openBoundedRead?(
|
|
3182
3493
|
path: string,
|
|
@@ -3947,6 +4258,8 @@ const storageAppendResultFromRow = (
|
|
|
3947
4258
|
trimHashRows,
|
|
3948
4259
|
documentTrimmedHeadsProcessed,
|
|
3949
4260
|
documentPreviousContextRow,
|
|
4261
|
+
_trimGidExtension,
|
|
4262
|
+
trimGidRows,
|
|
3950
4263
|
] = row as [
|
|
3951
4264
|
unknown[],
|
|
3952
4265
|
unknown[] | undefined,
|
|
@@ -3957,6 +4270,8 @@ const storageAppendResultFromRow = (
|
|
|
3957
4270
|
unknown[] | undefined,
|
|
3958
4271
|
boolean | undefined,
|
|
3959
4272
|
unknown[] | undefined,
|
|
4273
|
+
undefined,
|
|
4274
|
+
string[] | undefined,
|
|
3960
4275
|
];
|
|
3961
4276
|
return {
|
|
3962
4277
|
entry: storageFactsEntryFromRow(entryRow),
|
|
@@ -3965,6 +4280,8 @@ const storageAppendResultFromRow = (
|
|
|
3965
4280
|
assignedToRangeBoundary,
|
|
3966
4281
|
coordinate: appendCoordinatePlanFromRow(resolution, coordinateRow),
|
|
3967
4282
|
...trimmedRowsAndHashesResult(trimRows, trimHashRows),
|
|
4283
|
+
trimmedGids:
|
|
4284
|
+
trimGidRows ?? (trimRows as unknown[][]).map((trim) => trim[1] as string),
|
|
3968
4285
|
documentTrimmedHeadsProcessed,
|
|
3969
4286
|
documentPreviousContext: documentContextFactsFromRow(
|
|
3970
4287
|
documentPreviousContextRow,
|
|
@@ -3986,6 +4303,8 @@ const committedStorageAppendResultFromRow = (
|
|
|
3986
4303
|
trimHashRows,
|
|
3987
4304
|
documentTrimmedHeadsProcessed,
|
|
3988
4305
|
documentPreviousContextRow,
|
|
4306
|
+
_trimGidExtension,
|
|
4307
|
+
trimGidRows,
|
|
3989
4308
|
] = row as [
|
|
3990
4309
|
unknown[],
|
|
3991
4310
|
unknown[] | undefined,
|
|
@@ -3996,6 +4315,8 @@ const committedStorageAppendResultFromRow = (
|
|
|
3996
4315
|
unknown[] | undefined,
|
|
3997
4316
|
boolean | undefined,
|
|
3998
4317
|
unknown[] | undefined,
|
|
4318
|
+
undefined,
|
|
4319
|
+
string[] | undefined,
|
|
3999
4320
|
];
|
|
4000
4321
|
return {
|
|
4001
4322
|
entry: committedStorageFactsEntryFromRow(entryRow),
|
|
@@ -4004,6 +4325,8 @@ const committedStorageAppendResultFromRow = (
|
|
|
4004
4325
|
assignedToRangeBoundary,
|
|
4005
4326
|
coordinate: appendCoordinatePlanFromRow(resolution, coordinateRow),
|
|
4006
4327
|
...trimmedRowsAndHashesResult(trimRows, trimHashRows),
|
|
4328
|
+
trimmedGids:
|
|
4329
|
+
trimGidRows ?? (trimRows as unknown[][]).map((trim) => trim[1] as string),
|
|
4007
4330
|
documentTrimmedHeadsProcessed,
|
|
4008
4331
|
documentPreviousContext: documentContextFactsFromRow(
|
|
4009
4332
|
documentPreviousContextRow,
|
|
@@ -4015,7 +4338,14 @@ const compactCommittedNoNextStorageAppendResultFromRow = (
|
|
|
4015
4338
|
resolution: RangeResolution,
|
|
4016
4339
|
row: unknown[],
|
|
4017
4340
|
): NativeBackboneAppendResult => {
|
|
4018
|
-
const
|
|
4341
|
+
const appendedTrimGids =
|
|
4342
|
+
row.length >= 12 &&
|
|
4343
|
+
row[row.length - 2] === undefined &&
|
|
4344
|
+
Array.isArray(row[row.length - 1])
|
|
4345
|
+
? (row[row.length - 1] as string[])
|
|
4346
|
+
: undefined;
|
|
4347
|
+
const baseRow = appendedTrimGids ? row.slice(0, -2) : row;
|
|
4348
|
+
const [hash, byteLength, metaBytes, fourth] = baseRow as [
|
|
4019
4349
|
string,
|
|
4020
4350
|
number,
|
|
4021
4351
|
Uint8Array | undefined,
|
|
@@ -4023,7 +4353,7 @@ const compactCommittedNoNextStorageAppendResultFromRow = (
|
|
|
4023
4353
|
];
|
|
4024
4354
|
const hasDigestRow = fourth instanceof Uint8Array;
|
|
4025
4355
|
const hashDigestBytes = hasDigestRow ? (fourth as Uint8Array) : undefined;
|
|
4026
|
-
const rest =
|
|
4356
|
+
const rest = baseRow.slice(hasDigestRow ? 4 : 3);
|
|
4027
4357
|
const usesNestedCoordinateRow = Array.isArray(rest[0]);
|
|
4028
4358
|
let coordinate: NativeBackboneCoordinatePlan;
|
|
4029
4359
|
let leaderRows: unknown[] | undefined;
|
|
@@ -4106,6 +4436,7 @@ const compactCommittedNoNextStorageAppendResultFromRow = (
|
|
|
4106
4436
|
coordinate,
|
|
4107
4437
|
trimmed: [],
|
|
4108
4438
|
trimmedHashes: trimHashRows ?? [],
|
|
4439
|
+
trimmedGids: appendedTrimGids,
|
|
4109
4440
|
documentTrimmedHeadsProcessed,
|
|
4110
4441
|
};
|
|
4111
4442
|
};
|
|
@@ -4114,6 +4445,13 @@ const compactCommittedLatestStorageAppendResultFromRow = (
|
|
|
4114
4445
|
resolution: RangeResolution,
|
|
4115
4446
|
row: unknown[],
|
|
4116
4447
|
): NativeBackboneAppendResult => {
|
|
4448
|
+
const appendedTrimGids =
|
|
4449
|
+
row.length >= 13 &&
|
|
4450
|
+
row[row.length - 2] === undefined &&
|
|
4451
|
+
Array.isArray(row[row.length - 1])
|
|
4452
|
+
? (row[row.length - 1] as string[])
|
|
4453
|
+
: undefined;
|
|
4454
|
+
const baseRow = appendedTrimGids ? row.slice(0, -2) : row;
|
|
4117
4455
|
const [
|
|
4118
4456
|
hash,
|
|
4119
4457
|
byteLength,
|
|
@@ -4127,7 +4465,7 @@ const compactCommittedLatestStorageAppendResultFromRow = (
|
|
|
4127
4465
|
tenth,
|
|
4128
4466
|
eleventh,
|
|
4129
4467
|
twelfth,
|
|
4130
|
-
] =
|
|
4468
|
+
] = baseRow as [
|
|
4131
4469
|
string,
|
|
4132
4470
|
number,
|
|
4133
4471
|
Uint8Array | undefined,
|
|
@@ -4172,6 +4510,7 @@ const compactCommittedLatestStorageAppendResultFromRow = (
|
|
|
4172
4510
|
coordinate,
|
|
4173
4511
|
trimmed: [],
|
|
4174
4512
|
trimmedHashes: trimHashRows ?? [],
|
|
4513
|
+
trimmedGids: appendedTrimGids,
|
|
4175
4514
|
documentTrimmedHeadsProcessed,
|
|
4176
4515
|
documentPreviousContext: documentContextFactsFromRow(
|
|
4177
4516
|
documentPreviousContextRow,
|
|
@@ -4200,6 +4539,20 @@ const preparedCommitFactsFromRow = (
|
|
|
4200
4539
|
return prepared;
|
|
4201
4540
|
};
|
|
4202
4541
|
|
|
4542
|
+
const preparedCommitFactsWithTrimRefsFromRow = (
|
|
4543
|
+
row: unknown[],
|
|
4544
|
+
): NativeBackboneCommittedEntry & {
|
|
4545
|
+
trimmedEntryHashes: string[];
|
|
4546
|
+
trimmedEntryGids: string[];
|
|
4547
|
+
} => {
|
|
4548
|
+
const prepared = committedStorageFactsEntryFromRow(row[0] as unknown[]);
|
|
4549
|
+
return {
|
|
4550
|
+
...prepared,
|
|
4551
|
+
trimmedEntryHashes: row[1] as string[],
|
|
4552
|
+
trimmedEntryGids: row[2] as string[],
|
|
4553
|
+
};
|
|
4554
|
+
};
|
|
4555
|
+
|
|
4203
4556
|
const preparedCommitFactsWithLatestDocumentContextFromRow = (
|
|
4204
4557
|
row: unknown[],
|
|
4205
4558
|
): NativeBackboneCommittedEntry & {
|
|
@@ -4222,12 +4575,39 @@ const preparedCommitFactsWithLatestDocumentContextFromRow = (
|
|
|
4222
4575
|
};
|
|
4223
4576
|
};
|
|
4224
4577
|
|
|
4225
|
-
const
|
|
4578
|
+
const preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow = (
|
|
4226
4579
|
row: unknown[],
|
|
4227
4580
|
): NativeBackboneCommittedEntry & {
|
|
4228
|
-
trimmedEntryHashes
|
|
4581
|
+
trimmedEntryHashes: string[];
|
|
4582
|
+
trimmedEntryGids: string[];
|
|
4229
4583
|
documentTrimmedHeadsProcessed?: boolean;
|
|
4584
|
+
documentPreviousContext?: NativeBackboneDocumentContextFacts;
|
|
4230
4585
|
} => {
|
|
4586
|
+
const [
|
|
4587
|
+
entryRow,
|
|
4588
|
+
trimHashRows,
|
|
4589
|
+
trimGidRows,
|
|
4590
|
+
documentTrimmedHeadsProcessed,
|
|
4591
|
+
contextRow,
|
|
4592
|
+
] = row as [
|
|
4593
|
+
unknown[],
|
|
4594
|
+
string[],
|
|
4595
|
+
string[],
|
|
4596
|
+
boolean | undefined,
|
|
4597
|
+
unknown[] | undefined,
|
|
4598
|
+
];
|
|
4599
|
+
return {
|
|
4600
|
+
...committedStorageFactsEntryFromRow(entryRow),
|
|
4601
|
+
trimmedEntryHashes: trimHashRows,
|
|
4602
|
+
trimmedEntryGids: trimGidRows,
|
|
4603
|
+
documentTrimmedHeadsProcessed,
|
|
4604
|
+
documentPreviousContext: documentContextFactsFromRow(contextRow),
|
|
4605
|
+
};
|
|
4606
|
+
};
|
|
4607
|
+
|
|
4608
|
+
const compactPreparedCommitFactsBaseFromRow = (
|
|
4609
|
+
row: unknown[],
|
|
4610
|
+
): { entry: NativeBackboneCommittedEntry; trimRowOffset: number } => {
|
|
4231
4611
|
const [hash, byteLength, metaBytes, fourth] = row as [
|
|
4232
4612
|
string,
|
|
4233
4613
|
number,
|
|
@@ -4235,24 +4615,58 @@ const compactPreparedCommitFactsWithTrimHashesFromRow = (
|
|
|
4235
4615
|
Uint8Array | string[] | undefined,
|
|
4236
4616
|
];
|
|
4237
4617
|
const hasDigestRow = fourth instanceof Uint8Array;
|
|
4238
|
-
|
|
4239
|
-
|
|
4240
|
-
|
|
4241
|
-
|
|
4618
|
+
return {
|
|
4619
|
+
entry: {
|
|
4620
|
+
cid: hash,
|
|
4621
|
+
hash,
|
|
4622
|
+
next: [],
|
|
4623
|
+
metaBytes,
|
|
4624
|
+
byteLength,
|
|
4625
|
+
hashDigestBytes: hasDigestRow ? fourth : undefined,
|
|
4626
|
+
},
|
|
4627
|
+
trimRowOffset: hasDigestRow ? 4 : 3,
|
|
4628
|
+
};
|
|
4629
|
+
};
|
|
4630
|
+
|
|
4631
|
+
const compactPreparedCommitFactsWithTrimHashesFromRow = (
|
|
4632
|
+
row: unknown[],
|
|
4633
|
+
): NativeBackboneCommittedEntry & {
|
|
4634
|
+
trimmedEntryHashes?: string[];
|
|
4635
|
+
documentTrimmedHeadsProcessed?: boolean;
|
|
4636
|
+
} => {
|
|
4637
|
+
const { entry, trimRowOffset } = compactPreparedCommitFactsBaseFromRow(row);
|
|
4638
|
+
const trimHashRows = row[trimRowOffset] as string[] | undefined;
|
|
4639
|
+
const documentTrimmedHeadsProcessed = row[trimRowOffset + 1] as
|
|
4242
4640
|
| boolean
|
|
4243
4641
|
| undefined;
|
|
4244
4642
|
return {
|
|
4245
|
-
|
|
4246
|
-
hash,
|
|
4247
|
-
next: [],
|
|
4248
|
-
metaBytes,
|
|
4249
|
-
byteLength,
|
|
4250
|
-
hashDigestBytes,
|
|
4643
|
+
...entry,
|
|
4251
4644
|
trimmedEntryHashes: trimHashRows ?? [],
|
|
4252
4645
|
documentTrimmedHeadsProcessed,
|
|
4253
4646
|
};
|
|
4254
4647
|
};
|
|
4255
4648
|
|
|
4649
|
+
const compactPreparedCommitFactsWithTrimRefsFromRow = (
|
|
4650
|
+
row: unknown[],
|
|
4651
|
+
): NativeBackboneCommittedEntry & {
|
|
4652
|
+
trimmedEntryHashes: string[];
|
|
4653
|
+
trimmedEntryGids: string[];
|
|
4654
|
+
documentTrimmedHeadsProcessed?: boolean;
|
|
4655
|
+
} => {
|
|
4656
|
+
const { entry, trimRowOffset } = compactPreparedCommitFactsBaseFromRow(row);
|
|
4657
|
+
const trimHashRows = row[trimRowOffset] as string[];
|
|
4658
|
+
const trimGidRows = row[trimRowOffset + 1] as string[];
|
|
4659
|
+
const documentTrimmedHeadsProcessed = row[trimRowOffset + 2] as
|
|
4660
|
+
| boolean
|
|
4661
|
+
| undefined;
|
|
4662
|
+
return {
|
|
4663
|
+
...entry,
|
|
4664
|
+
trimmedEntryHashes: trimHashRows,
|
|
4665
|
+
trimmedEntryGids: trimGidRows,
|
|
4666
|
+
documentTrimmedHeadsProcessed,
|
|
4667
|
+
};
|
|
4668
|
+
};
|
|
4669
|
+
|
|
4256
4670
|
const nativeLogCommitEntryColumns = (
|
|
4257
4671
|
entries: NativeBackboneLogCommitEntry[],
|
|
4258
4672
|
) => {
|
|
@@ -5080,7 +5494,9 @@ class NativeBackboneLogGraph {
|
|
|
5080
5494
|
| (NativeBackboneCommittedEntry & {
|
|
5081
5495
|
trimmedEntries?: NativeBackboneLogEntry[];
|
|
5082
5496
|
trimmedEntryHashes?: string[];
|
|
5497
|
+
trimmedEntryGids?: string[];
|
|
5083
5498
|
documentTrimmedHeadsProcessed?: boolean;
|
|
5499
|
+
documentPreviousContext?: NativeBackboneDocumentContextFacts;
|
|
5084
5500
|
})
|
|
5085
5501
|
| undefined {
|
|
5086
5502
|
if (this.options?.commitBlocks === false) {
|
|
@@ -5100,31 +5516,31 @@ class NativeBackboneLogGraph {
|
|
|
5100
5516
|
const documentIndexArgs = nativeDocumentIndexArgs(documentIndex);
|
|
5101
5517
|
const projection = documentIndex?.projection;
|
|
5102
5518
|
if (
|
|
5103
|
-
|
|
5104
|
-
|
|
5105
|
-
input.
|
|
5519
|
+
!documentIndexArgs &&
|
|
5520
|
+
input.resolveTrimmedEntries === false &&
|
|
5521
|
+
input.trimLengthTo != null &&
|
|
5522
|
+
this.native.prepare_plain_entry_commit_facts_trim_refs
|
|
5106
5523
|
) {
|
|
5107
|
-
|
|
5108
|
-
|
|
5109
|
-
|
|
5110
|
-
|
|
5111
|
-
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
5116
|
-
|
|
5117
|
-
|
|
5118
|
-
|
|
5119
|
-
|
|
5120
|
-
|
|
5121
|
-
|
|
5122
|
-
|
|
5123
|
-
|
|
5124
|
-
|
|
5125
|
-
|
|
5126
|
-
|
|
5127
|
-
this.native.prepare_plain_entry_commit_latest_facts_document_index_trim_hashes(
|
|
5524
|
+
return preparedCommitFactsWithTrimRefsFromRow(
|
|
5525
|
+
this.native.prepare_plain_entry_commit_facts_trim_refs(
|
|
5526
|
+
wallTime,
|
|
5527
|
+
logical,
|
|
5528
|
+
input.gid,
|
|
5529
|
+
input.next ?? [],
|
|
5530
|
+
entryType,
|
|
5531
|
+
input.metaData,
|
|
5532
|
+
input.payloadData,
|
|
5533
|
+
input.trimLengthTo,
|
|
5534
|
+
),
|
|
5535
|
+
);
|
|
5536
|
+
}
|
|
5537
|
+
if (
|
|
5538
|
+
documentIndex?.useLatestContext &&
|
|
5539
|
+
documentIndexArgs &&
|
|
5540
|
+
input.resolveTrimmedEntries === false
|
|
5541
|
+
) {
|
|
5542
|
+
if (projection && this.options?.documentProjectionPlanId) {
|
|
5543
|
+
const args = [
|
|
5128
5544
|
wallTime,
|
|
5129
5545
|
logical,
|
|
5130
5546
|
input.gid,
|
|
@@ -5133,12 +5549,53 @@ class NativeBackboneLogGraph {
|
|
|
5133
5549
|
input.payloadData,
|
|
5134
5550
|
input.trimLengthTo,
|
|
5135
5551
|
documentIndex.key,
|
|
5136
|
-
documentIndex.valuePrefixBytes ?? EMPTY_UINT8_ARRAY,
|
|
5137
5552
|
documentIndex.byteElementIndexLimit ?? 0,
|
|
5138
5553
|
documentIndex.deleteTrimmedHeads === true,
|
|
5139
|
-
projection
|
|
5140
|
-
projection
|
|
5141
|
-
projection
|
|
5554
|
+
this.options.documentProjectionPlanId(projection.plan),
|
|
5555
|
+
projection.encodedDocument,
|
|
5556
|
+
projection.signer,
|
|
5557
|
+
] as const;
|
|
5558
|
+
const prepareTrimRefs =
|
|
5559
|
+
this.native
|
|
5560
|
+
.prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_refs;
|
|
5561
|
+
if (prepareTrimRefs) {
|
|
5562
|
+
return preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow(
|
|
5563
|
+
prepareTrimRefs.call(this.native, ...args),
|
|
5564
|
+
);
|
|
5565
|
+
}
|
|
5566
|
+
return preparedCommitFactsWithLatestDocumentContextFromRow(
|
|
5567
|
+
this.native.prepare_plain_entry_commit_latest_facts_document_index_cached_plan_trim_hashes(
|
|
5568
|
+
...args,
|
|
5569
|
+
),
|
|
5570
|
+
);
|
|
5571
|
+
}
|
|
5572
|
+
const args = [
|
|
5573
|
+
wallTime,
|
|
5574
|
+
logical,
|
|
5575
|
+
input.gid,
|
|
5576
|
+
entryType,
|
|
5577
|
+
input.metaData,
|
|
5578
|
+
input.payloadData,
|
|
5579
|
+
input.trimLengthTo,
|
|
5580
|
+
documentIndex.key,
|
|
5581
|
+
documentIndex.valuePrefixBytes ?? EMPTY_UINT8_ARRAY,
|
|
5582
|
+
documentIndex.byteElementIndexLimit ?? 0,
|
|
5583
|
+
documentIndex.deleteTrimmedHeads === true,
|
|
5584
|
+
projection?.plan,
|
|
5585
|
+
projection?.encodedDocument,
|
|
5586
|
+
projection?.signer,
|
|
5587
|
+
] as const;
|
|
5588
|
+
const prepareTrimRefs =
|
|
5589
|
+
this.native
|
|
5590
|
+
.prepare_plain_entry_commit_latest_facts_document_index_trim_refs;
|
|
5591
|
+
if (prepareTrimRefs) {
|
|
5592
|
+
return preparedCommitFactsWithLatestDocumentContextAndTrimRefsFromRow(
|
|
5593
|
+
prepareTrimRefs.call(this.native, ...args),
|
|
5594
|
+
);
|
|
5595
|
+
}
|
|
5596
|
+
return preparedCommitFactsWithLatestDocumentContextFromRow(
|
|
5597
|
+
this.native.prepare_plain_entry_commit_latest_facts_document_index_trim_hashes(
|
|
5598
|
+
...args,
|
|
5142
5599
|
),
|
|
5143
5600
|
);
|
|
5144
5601
|
}
|
|
@@ -5153,49 +5610,71 @@ class NativeBackboneLogGraph {
|
|
|
5153
5610
|
const projectionPlanId = this.options.documentProjectionPlanId(
|
|
5154
5611
|
projection.plan,
|
|
5155
5612
|
);
|
|
5613
|
+
const plainPutPayloadArgs = [
|
|
5614
|
+
wallTime,
|
|
5615
|
+
logical,
|
|
5616
|
+
input.gid,
|
|
5617
|
+
entryType,
|
|
5618
|
+
input.metaData,
|
|
5619
|
+
input.payloadData,
|
|
5620
|
+
input.trimLengthTo,
|
|
5621
|
+
documentIndex.key,
|
|
5622
|
+
documentIndex.existingCreated == null
|
|
5623
|
+
? ""
|
|
5624
|
+
: integerString(documentIndex.existingCreated),
|
|
5625
|
+
documentIndex.byteElementIndexLimit ?? 0,
|
|
5626
|
+
documentIndex.deleteTrimmedHeads === true,
|
|
5627
|
+
projectionPlanId,
|
|
5628
|
+
projection.signer,
|
|
5629
|
+
] as const;
|
|
5630
|
+
const plainPutPayloadCommitWithTrimRefs =
|
|
5631
|
+
this.native
|
|
5632
|
+
.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs_plain_put_payload;
|
|
5633
|
+
if (plainPutPayloadCommitWithTrimRefs) {
|
|
5634
|
+
return compactPreparedCommitFactsWithTrimRefsFromRow(
|
|
5635
|
+
plainPutPayloadCommitWithTrimRefs.call(
|
|
5636
|
+
this.native,
|
|
5637
|
+
...plainPutPayloadArgs,
|
|
5638
|
+
),
|
|
5639
|
+
);
|
|
5640
|
+
}
|
|
5156
5641
|
const plainPutPayloadCommit =
|
|
5157
5642
|
this.native
|
|
5158
5643
|
.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes_plain_put_payload;
|
|
5159
5644
|
if (plainPutPayloadCommit) {
|
|
5160
5645
|
return compactPreparedCommitFactsWithTrimHashesFromRow(
|
|
5161
|
-
plainPutPayloadCommit.call(
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
5168
|
-
|
|
5169
|
-
|
|
5170
|
-
|
|
5171
|
-
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5646
|
+
plainPutPayloadCommit.call(this.native, ...plainPutPayloadArgs),
|
|
5647
|
+
);
|
|
5648
|
+
}
|
|
5649
|
+
const args = [
|
|
5650
|
+
wallTime,
|
|
5651
|
+
logical,
|
|
5652
|
+
input.gid,
|
|
5653
|
+
entryType,
|
|
5654
|
+
input.metaData,
|
|
5655
|
+
input.payloadData,
|
|
5656
|
+
input.trimLengthTo,
|
|
5657
|
+
documentIndex.key,
|
|
5658
|
+
documentIndex.existingCreated == null
|
|
5659
|
+
? ""
|
|
5660
|
+
: integerString(documentIndex.existingCreated),
|
|
5661
|
+
documentIndex.byteElementIndexLimit ?? 0,
|
|
5662
|
+
documentIndex.deleteTrimmedHeads === true,
|
|
5663
|
+
projectionPlanId,
|
|
5664
|
+
projection.encodedDocument,
|
|
5665
|
+
projection.signer,
|
|
5666
|
+
] as const;
|
|
5667
|
+
const prepareTrimRefs =
|
|
5668
|
+
this.native
|
|
5669
|
+
.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_refs;
|
|
5670
|
+
if (prepareTrimRefs) {
|
|
5671
|
+
return compactPreparedCommitFactsWithTrimRefsFromRow(
|
|
5672
|
+
prepareTrimRefs.call(this.native, ...args),
|
|
5179
5673
|
);
|
|
5180
5674
|
}
|
|
5181
5675
|
return compactPreparedCommitFactsWithTrimHashesFromRow(
|
|
5182
5676
|
this.native.prepare_plain_entry_commit_no_next_facts_document_index_cached_plan_compact_trim_hashes(
|
|
5183
|
-
|
|
5184
|
-
logical,
|
|
5185
|
-
input.gid,
|
|
5186
|
-
entryType,
|
|
5187
|
-
input.metaData,
|
|
5188
|
-
input.payloadData,
|
|
5189
|
-
input.trimLengthTo,
|
|
5190
|
-
documentIndex.key,
|
|
5191
|
-
documentIndex.existingCreated == null
|
|
5192
|
-
? ""
|
|
5193
|
-
: integerString(documentIndex.existingCreated),
|
|
5194
|
-
documentIndex.byteElementIndexLimit ?? 0,
|
|
5195
|
-
documentIndex.deleteTrimmedHeads === true,
|
|
5196
|
-
projectionPlanId,
|
|
5197
|
-
projection.encodedDocument,
|
|
5198
|
-
projection.signer,
|
|
5677
|
+
...args,
|
|
5199
5678
|
),
|
|
5200
5679
|
);
|
|
5201
5680
|
}
|
|
@@ -5302,6 +5781,24 @@ class NativeBackboneLogGraph {
|
|
|
5302
5781
|
input.trimLengthTo != null &&
|
|
5303
5782
|
hasNoNext
|
|
5304
5783
|
) {
|
|
5784
|
+
const prepareTrimRefs =
|
|
5785
|
+
this.native
|
|
5786
|
+
.prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_refs;
|
|
5787
|
+
if (prepareTrimRefs) {
|
|
5788
|
+
return compactPreparedCommitFactsWithTrimRefsFromRow(
|
|
5789
|
+
prepareTrimRefs.call(
|
|
5790
|
+
this.native,
|
|
5791
|
+
wallTime,
|
|
5792
|
+
logical,
|
|
5793
|
+
input.gid,
|
|
5794
|
+
entryType,
|
|
5795
|
+
input.metaData,
|
|
5796
|
+
input.payloadData,
|
|
5797
|
+
input.trimLengthTo,
|
|
5798
|
+
...documentIndexArgs,
|
|
5799
|
+
),
|
|
5800
|
+
);
|
|
5801
|
+
}
|
|
5305
5802
|
return compactPreparedCommitFactsWithTrimHashesFromRow(
|
|
5306
5803
|
this.native.prepare_plain_entry_commit_no_next_facts_document_index_compact_trim_hashes(
|
|
5307
5804
|
wallTime,
|
|
@@ -6455,9 +6952,7 @@ export class NativePeerbitBackbone {
|
|
|
6455
6952
|
end2: bigint | number | string;
|
|
6456
6953
|
limit: number;
|
|
6457
6954
|
}): bigint[] | undefined {
|
|
6458
|
-
if (
|
|
6459
|
-
typeof this.native.entry_hash_numbers_in_range_limited !== "function"
|
|
6460
|
-
) {
|
|
6955
|
+
if (typeof this.native.entry_hash_numbers_in_range_limited !== "function") {
|
|
6461
6956
|
return undefined;
|
|
6462
6957
|
}
|
|
6463
6958
|
return rowsToNumbers(
|
|
@@ -6967,6 +7462,21 @@ export class NativePeerbitBackbone {
|
|
|
6967
7462
|
return this.native.full_replica_candidates_for(minReplicas, selfHash);
|
|
6968
7463
|
}
|
|
6969
7464
|
|
|
7465
|
+
getRoutingFullReplicaLeaders(
|
|
7466
|
+
replicas: number,
|
|
7467
|
+
options?: NativeBackboneFindLeaderOptions,
|
|
7468
|
+
): Map<string, NativeBackboneLeaderSample> | undefined {
|
|
7469
|
+
if (!this.native.get_routing_full_replica_leaders) {
|
|
7470
|
+
return undefined;
|
|
7471
|
+
}
|
|
7472
|
+
return rowsToSamples(
|
|
7473
|
+
this.native.get_routing_full_replica_leaders(
|
|
7474
|
+
replicas,
|
|
7475
|
+
...findLeaderArguments(options),
|
|
7476
|
+
),
|
|
7477
|
+
);
|
|
7478
|
+
}
|
|
7479
|
+
|
|
6970
7480
|
clearEntryCoordinates(): void {
|
|
6971
7481
|
this.native.clear_entry_coordinates();
|
|
6972
7482
|
}
|
|
@@ -8768,6 +9278,7 @@ export class NativeBackboneNodeCoordinatePersistenceStore
|
|
|
8768
9278
|
maxBytes: number,
|
|
8769
9279
|
) => Promise<Uint8Array | undefined>;
|
|
8770
9280
|
readonly durableBarrier?: (name?: string) => Promise<void>;
|
|
9281
|
+
readonly atomicReplace?: (name: string, bytes: Uint8Array) => Promise<void>;
|
|
8771
9282
|
|
|
8772
9283
|
constructor(
|
|
8773
9284
|
private readonly directory: string,
|
|
@@ -8783,6 +9294,12 @@ export class NativeBackboneNodeCoordinatePersistenceStore
|
|
|
8783
9294
|
this.readLimited = (name, maxBytes) =>
|
|
8784
9295
|
this.readWithinLimit(name, maxBytes);
|
|
8785
9296
|
}
|
|
9297
|
+
if (
|
|
9298
|
+
!fs ||
|
|
9299
|
+
(typeof fs.open === "function" && typeof fs.rename === "function")
|
|
9300
|
+
) {
|
|
9301
|
+
this.atomicReplace = (name, bytes) => this.replaceAtomically(name, bytes);
|
|
9302
|
+
}
|
|
8786
9303
|
}
|
|
8787
9304
|
|
|
8788
9305
|
private async nodeFs(): Promise<NativeBackboneNodeFs> {
|
|
@@ -8965,6 +9482,51 @@ export class NativeBackboneNodeCoordinatePersistenceStore
|
|
|
8965
9482
|
await fs.writeFile(path, bytes);
|
|
8966
9483
|
}
|
|
8967
9484
|
|
|
9485
|
+
private async replaceAtomically(
|
|
9486
|
+
name: string,
|
|
9487
|
+
bytes: Uint8Array,
|
|
9488
|
+
): Promise<void> {
|
|
9489
|
+
const fs = await this.ensureDirectory();
|
|
9490
|
+
if (!fs.open || !fs.rename) {
|
|
9491
|
+
throw new Error(
|
|
9492
|
+
"Node coordinate persistence does not expose atomic file replacement",
|
|
9493
|
+
);
|
|
9494
|
+
}
|
|
9495
|
+
const validName = validateCoordinatePersistenceName(name);
|
|
9496
|
+
const temporaryName = validateCoordinatePersistenceName(`${validName}.tmp`);
|
|
9497
|
+
const path = await this.filePath(validName);
|
|
9498
|
+
const temporaryPath = await this.filePath(temporaryName);
|
|
9499
|
+
await this.closeAppendHandle(path);
|
|
9500
|
+
await this.closeAppendHandle(temporaryPath);
|
|
9501
|
+
await fs.rm(temporaryPath, { force: true });
|
|
9502
|
+
await fs.writeFile(temporaryPath, bytes);
|
|
9503
|
+
let temporaryHandle: NativeBackboneNodeAppendFileHandle | undefined;
|
|
9504
|
+
try {
|
|
9505
|
+
temporaryHandle = await fs.open(temporaryPath, "r");
|
|
9506
|
+
if (typeof temporaryHandle.sync !== "function") {
|
|
9507
|
+
throw new Error(
|
|
9508
|
+
"Node coordinate persistence atomic replacement does not expose FileHandle.sync",
|
|
9509
|
+
);
|
|
9510
|
+
}
|
|
9511
|
+
await temporaryHandle.sync();
|
|
9512
|
+
} finally {
|
|
9513
|
+
await temporaryHandle?.close();
|
|
9514
|
+
}
|
|
9515
|
+
await fs.rename(temporaryPath, path);
|
|
9516
|
+
let directoryHandle: NativeBackboneNodeAppendFileHandle | undefined;
|
|
9517
|
+
try {
|
|
9518
|
+
directoryHandle = await fs.open(this.directory, "r");
|
|
9519
|
+
if (typeof directoryHandle.sync !== "function") {
|
|
9520
|
+
throw new Error(
|
|
9521
|
+
"Node coordinate persistence atomic replacement does not expose directory sync",
|
|
9522
|
+
);
|
|
9523
|
+
}
|
|
9524
|
+
await directoryHandle.sync();
|
|
9525
|
+
} finally {
|
|
9526
|
+
await directoryHandle?.close();
|
|
9527
|
+
}
|
|
9528
|
+
}
|
|
9529
|
+
|
|
8968
9530
|
async append(name: string, bytes: Uint8Array): Promise<void> {
|
|
8969
9531
|
if (this.appendFailure !== undefined) {
|
|
8970
9532
|
throw this.appendFailure;
|
|
@@ -9317,6 +9879,7 @@ export class NativeBackboneBufferedCoordinatePersistenceStore
|
|
|
9317
9879
|
) => Promise<Uint8Array | undefined>;
|
|
9318
9880
|
readonly supportsRemoval: boolean;
|
|
9319
9881
|
readonly durableBarrier?: (name?: string) => Promise<void>;
|
|
9882
|
+
readonly atomicReplace?: (name: string, bytes: Uint8Array) => Promise<void>;
|
|
9320
9883
|
|
|
9321
9884
|
constructor(
|
|
9322
9885
|
private readonly inner: NativeBackboneCoordinatePersistenceStore,
|
|
@@ -9335,6 +9898,12 @@ export class NativeBackboneBufferedCoordinatePersistenceStore
|
|
|
9335
9898
|
await inner.durableBarrier!(name);
|
|
9336
9899
|
};
|
|
9337
9900
|
}
|
|
9901
|
+
if (typeof inner.atomicReplace === "function") {
|
|
9902
|
+
this.atomicReplace = async (name, bytes) => {
|
|
9903
|
+
await this.flush(name);
|
|
9904
|
+
await inner.atomicReplace!(name, bytes);
|
|
9905
|
+
};
|
|
9906
|
+
}
|
|
9338
9907
|
}
|
|
9339
9908
|
|
|
9340
9909
|
private buffer(name: string): Uint8Array[] {
|
|
@@ -9480,7 +10049,7 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9480
10049
|
readonly flushIntervalMs?: number;
|
|
9481
10050
|
readonly compactMaxJournalBytes?: number;
|
|
9482
10051
|
readonly compactMaxJournalRecords?: number;
|
|
9483
|
-
readonly crashSafeCompaction
|
|
10052
|
+
readonly crashSafeCompaction: boolean;
|
|
9484
10053
|
readonly durableBarrier: boolean;
|
|
9485
10054
|
readonly supportsDrop: boolean;
|
|
9486
10055
|
readonly dropIsTerminal = true;
|
|
@@ -9490,9 +10059,32 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9490
10059
|
private readonly documentJournalFile: string;
|
|
9491
10060
|
private readonly documentSignerSnapshotFile: string;
|
|
9492
10061
|
private readonly documentSignerJournalFile: string;
|
|
10062
|
+
private readonly checkpointConfigurationChecksum: number;
|
|
10063
|
+
private readonly checkpointStateFile: string;
|
|
10064
|
+
private readonly checkpointFiles: Record<
|
|
10065
|
+
NativeBackboneCoordinateCheckpointSlot,
|
|
10066
|
+
string
|
|
10067
|
+
>;
|
|
10068
|
+
private readonly checkpointJournalFiles: Record<
|
|
10069
|
+
NativeBackboneCoordinateCheckpointSlot,
|
|
10070
|
+
{
|
|
10071
|
+
coordinate: string;
|
|
10072
|
+
document: string;
|
|
10073
|
+
signer: string;
|
|
10074
|
+
}
|
|
10075
|
+
>;
|
|
9493
10076
|
private journalInitialized: boolean | undefined;
|
|
10077
|
+
private journalByteLength = 0;
|
|
10078
|
+
private journalRecordCount = 0;
|
|
9494
10079
|
private documentJournalInitialized: boolean | undefined;
|
|
10080
|
+
private documentJournalByteLength = 0;
|
|
10081
|
+
private documentJournalRecordCount = 0;
|
|
9495
10082
|
private documentSignerJournalInitialized: boolean | undefined;
|
|
10083
|
+
private documentSignerJournalByteLength = 0;
|
|
10084
|
+
private documentSignerJournalRecordCount = 0;
|
|
10085
|
+
private checkpointHighwater = 0n;
|
|
10086
|
+
private checkpointCompleted?: NativeBackboneCoordinateCheckpointAuthority;
|
|
10087
|
+
private checkpointPending?: NativeBackboneCoordinateCheckpointAuthority;
|
|
9496
10088
|
private lastFlushMs = Date.now();
|
|
9497
10089
|
private persistenceQueue: Promise<unknown> | undefined;
|
|
9498
10090
|
private persistenceLifecycle:
|
|
@@ -9539,24 +10131,87 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9539
10131
|
options.documentSignerJournal ??
|
|
9540
10132
|
nativeBackboneCoordinatePersistenceFiles.documentSignerJournal,
|
|
9541
10133
|
);
|
|
10134
|
+
this.checkpointConfigurationChecksum = coordinateJournalChecksum(
|
|
10135
|
+
new TextEncoder().encode(
|
|
10136
|
+
JSON.stringify([
|
|
10137
|
+
this.snapshotFile,
|
|
10138
|
+
this.journalFile,
|
|
10139
|
+
this.documentSnapshotFile,
|
|
10140
|
+
this.documentJournalFile,
|
|
10141
|
+
this.documentSignerSnapshotFile,
|
|
10142
|
+
this.documentSignerJournalFile,
|
|
10143
|
+
]),
|
|
10144
|
+
),
|
|
10145
|
+
);
|
|
10146
|
+
this.checkpointStateFile = validateCoordinatePersistenceName(
|
|
10147
|
+
`${this.snapshotFile}.checkpoint-state`,
|
|
10148
|
+
);
|
|
10149
|
+
this.checkpointFiles = {
|
|
10150
|
+
a: validateCoordinatePersistenceName(`${this.snapshotFile}.checkpoint-a`),
|
|
10151
|
+
b: validateCoordinatePersistenceName(`${this.snapshotFile}.checkpoint-b`),
|
|
10152
|
+
};
|
|
10153
|
+
this.checkpointJournalFiles = {
|
|
10154
|
+
a: {
|
|
10155
|
+
coordinate: validateCoordinatePersistenceName(
|
|
10156
|
+
`${this.journalFile}.checkpoint-a`,
|
|
10157
|
+
),
|
|
10158
|
+
document: validateCoordinatePersistenceName(
|
|
10159
|
+
`${this.documentJournalFile}.checkpoint-a`,
|
|
10160
|
+
),
|
|
10161
|
+
signer: validateCoordinatePersistenceName(
|
|
10162
|
+
`${this.documentSignerJournalFile}.checkpoint-a`,
|
|
10163
|
+
),
|
|
10164
|
+
},
|
|
10165
|
+
b: {
|
|
10166
|
+
coordinate: validateCoordinatePersistenceName(
|
|
10167
|
+
`${this.journalFile}.checkpoint-b`,
|
|
10168
|
+
),
|
|
10169
|
+
document: validateCoordinatePersistenceName(
|
|
10170
|
+
`${this.documentJournalFile}.checkpoint-b`,
|
|
10171
|
+
),
|
|
10172
|
+
signer: validateCoordinatePersistenceName(
|
|
10173
|
+
`${this.documentSignerJournalFile}.checkpoint-b`,
|
|
10174
|
+
),
|
|
10175
|
+
},
|
|
10176
|
+
};
|
|
9542
10177
|
if (
|
|
9543
|
-
this.configuredFiles().includes(
|
|
10178
|
+
this.configuredFiles().includes(
|
|
10179
|
+
nativeBackboneCoordinateDropTombstoneFile,
|
|
10180
|
+
) ||
|
|
10181
|
+
new Set(this.configuredFiles()).size !== this.configuredFiles().length
|
|
9544
10182
|
) {
|
|
9545
10183
|
throw new Error(
|
|
9546
|
-
"Native backbone coordinate persistence
|
|
10184
|
+
"Native backbone coordinate persistence files conflict with one another or with its drop tombstone",
|
|
9547
10185
|
);
|
|
9548
10186
|
}
|
|
10187
|
+
this.crashSafeCompaction =
|
|
10188
|
+
typeof store.atomicReplace === "function" &&
|
|
10189
|
+
this.supportsDrop &&
|
|
10190
|
+
this.durableBarrier;
|
|
9549
10191
|
this.flushOnAppend = options.flushOnAppend ?? true;
|
|
9550
10192
|
this.flushMaxPendingBytes = resolveCoordinateFlushMaxPendingBytes(options);
|
|
9551
10193
|
if (options.flushIntervalMs != null) {
|
|
9552
10194
|
this.flushIntervalMs = Math.max(0, options.flushIntervalMs);
|
|
9553
10195
|
}
|
|
9554
10196
|
if (
|
|
9555
|
-
options.compactMaxJournalBytes != null ||
|
|
9556
|
-
|
|
10197
|
+
(options.compactMaxJournalBytes != null ||
|
|
10198
|
+
options.compactMaxJournalRecords != null) &&
|
|
10199
|
+
!this.crashSafeCompaction
|
|
9557
10200
|
) {
|
|
9558
10201
|
throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
|
|
9559
10202
|
}
|
|
10203
|
+
if (options.compactMaxJournalBytes != null) {
|
|
10204
|
+
this.compactMaxJournalBytes = validateCoordinateCompactionThreshold(
|
|
10205
|
+
options.compactMaxJournalBytes,
|
|
10206
|
+
"compactMaxJournalBytes",
|
|
10207
|
+
);
|
|
10208
|
+
}
|
|
10209
|
+
if (options.compactMaxJournalRecords != null) {
|
|
10210
|
+
this.compactMaxJournalRecords = validateCoordinateCompactionThreshold(
|
|
10211
|
+
options.compactMaxJournalRecords,
|
|
10212
|
+
"compactMaxJournalRecords",
|
|
10213
|
+
);
|
|
10214
|
+
}
|
|
9560
10215
|
}
|
|
9561
10216
|
|
|
9562
10217
|
/** Durable operation-intent capability for strict shared-log transactions. */
|
|
@@ -9564,7 +10219,7 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9564
10219
|
return this.store;
|
|
9565
10220
|
}
|
|
9566
10221
|
|
|
9567
|
-
private
|
|
10222
|
+
private legacyConfiguredFiles(): string[] {
|
|
9568
10223
|
return [
|
|
9569
10224
|
this.snapshotFile,
|
|
9570
10225
|
this.journalFile,
|
|
@@ -9575,6 +10230,47 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9575
10230
|
];
|
|
9576
10231
|
}
|
|
9577
10232
|
|
|
10233
|
+
private configuredFiles(): string[] {
|
|
10234
|
+
return [
|
|
10235
|
+
...this.legacyConfiguredFiles(),
|
|
10236
|
+
this.checkpointStateFile,
|
|
10237
|
+
this.checkpointFiles.a,
|
|
10238
|
+
this.checkpointFiles.b,
|
|
10239
|
+
this.checkpointJournalFiles.a.coordinate,
|
|
10240
|
+
this.checkpointJournalFiles.a.document,
|
|
10241
|
+
this.checkpointJournalFiles.a.signer,
|
|
10242
|
+
this.checkpointJournalFiles.b.coordinate,
|
|
10243
|
+
this.checkpointJournalFiles.b.document,
|
|
10244
|
+
this.checkpointJournalFiles.b.signer,
|
|
10245
|
+
`${this.checkpointStateFile}.tmp`,
|
|
10246
|
+
`${this.checkpointFiles.a}.tmp`,
|
|
10247
|
+
`${this.checkpointFiles.b}.tmp`,
|
|
10248
|
+
`${this.journalFile}.tmp`,
|
|
10249
|
+
`${nativeBackboneCoordinateDropTombstoneFile}.tmp`,
|
|
10250
|
+
];
|
|
10251
|
+
}
|
|
10252
|
+
|
|
10253
|
+
private checkpointSlot(
|
|
10254
|
+
generation: bigint,
|
|
10255
|
+
): NativeBackboneCoordinateCheckpointSlot {
|
|
10256
|
+
return generation % 2n === 1n ? "a" : "b";
|
|
10257
|
+
}
|
|
10258
|
+
|
|
10259
|
+
private activeJournalFiles(): {
|
|
10260
|
+
coordinate: string;
|
|
10261
|
+
document: string;
|
|
10262
|
+
signer: string;
|
|
10263
|
+
} {
|
|
10264
|
+
const completed = this.checkpointCompleted;
|
|
10265
|
+
return completed
|
|
10266
|
+
? this.checkpointJournalFiles[this.checkpointSlot(completed.generation)]
|
|
10267
|
+
: {
|
|
10268
|
+
coordinate: this.journalFile,
|
|
10269
|
+
document: this.documentJournalFile,
|
|
10270
|
+
signer: this.documentSignerJournalFile,
|
|
10271
|
+
};
|
|
10272
|
+
}
|
|
10273
|
+
|
|
9578
10274
|
private assertLifecycleActive(operation: string): void {
|
|
9579
10275
|
if (this.persistenceLifecycle !== "active") {
|
|
9580
10276
|
throw new Error(
|
|
@@ -9584,9 +10280,7 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9584
10280
|
}
|
|
9585
10281
|
|
|
9586
10282
|
private assertActive(operation: string): void {
|
|
9587
|
-
|
|
9588
|
-
throw this.persistenceFailure;
|
|
9589
|
-
}
|
|
10283
|
+
this.assertPersistenceHealthy();
|
|
9590
10284
|
this.assertLifecycleActive(operation);
|
|
9591
10285
|
if (this.dropInitiatedOnGeneration) {
|
|
9592
10286
|
throw new Error(
|
|
@@ -9595,13 +10289,44 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9595
10289
|
}
|
|
9596
10290
|
}
|
|
9597
10291
|
|
|
10292
|
+
private assertPersistenceHealthy(): void {
|
|
10293
|
+
if (this.persistenceFailure !== undefined) {
|
|
10294
|
+
throw this.persistenceFailure;
|
|
10295
|
+
}
|
|
10296
|
+
}
|
|
10297
|
+
|
|
9598
10298
|
private resetJournalTracking(): void {
|
|
9599
10299
|
this.journalInitialized = undefined;
|
|
10300
|
+
this.journalByteLength = 0;
|
|
10301
|
+
this.journalRecordCount = 0;
|
|
9600
10302
|
this.documentJournalInitialized = undefined;
|
|
10303
|
+
this.documentJournalByteLength = 0;
|
|
10304
|
+
this.documentJournalRecordCount = 0;
|
|
9601
10305
|
this.documentSignerJournalInitialized = undefined;
|
|
10306
|
+
this.documentSignerJournalByteLength = 0;
|
|
10307
|
+
this.documentSignerJournalRecordCount = 0;
|
|
10308
|
+
this.checkpointHighwater = 0n;
|
|
10309
|
+
this.checkpointCompleted = undefined;
|
|
10310
|
+
this.checkpointPending = undefined;
|
|
9602
10311
|
this.lastFlushMs = Date.now();
|
|
9603
10312
|
}
|
|
9604
10313
|
|
|
10314
|
+
private async persistDropTombstone(
|
|
10315
|
+
files: readonly string[],
|
|
10316
|
+
preferAtomicReplace = false,
|
|
10317
|
+
): Promise<void> {
|
|
10318
|
+
const bytes = nativeBackboneCoordinateDropTombstoneBytes(files);
|
|
10319
|
+
if (preferAtomicReplace && this.store.atomicReplace) {
|
|
10320
|
+
await this.store.atomicReplace(
|
|
10321
|
+
nativeBackboneCoordinateDropTombstoneFile,
|
|
10322
|
+
bytes,
|
|
10323
|
+
);
|
|
10324
|
+
return;
|
|
10325
|
+
}
|
|
10326
|
+
await this.store.write(nativeBackboneCoordinateDropTombstoneFile, bytes);
|
|
10327
|
+
await this.barrierFile(nativeBackboneCoordinateDropTombstoneFile);
|
|
10328
|
+
}
|
|
10329
|
+
|
|
9605
10330
|
private async eraseDropFiles(files: readonly string[]): Promise<void> {
|
|
9606
10331
|
if (!this.supportsDrop || !this.store.remove) {
|
|
9607
10332
|
throw new Error(
|
|
@@ -9651,17 +10376,7 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9651
10376
|
this.dropInitiatedOnGeneration = true;
|
|
9652
10377
|
this.persistenceLifecycle = "dropping";
|
|
9653
10378
|
await this.enqueuePersistence(async () => {
|
|
9654
|
-
await this.
|
|
9655
|
-
nativeBackboneCoordinateDropTombstoneFile,
|
|
9656
|
-
nativeBackboneCoordinateDropTombstoneBytes(files),
|
|
9657
|
-
);
|
|
9658
|
-
if (this.store.durableBarrier) {
|
|
9659
|
-
await this.store.durableBarrier(
|
|
9660
|
-
nativeBackboneCoordinateDropTombstoneFile,
|
|
9661
|
-
);
|
|
9662
|
-
} else {
|
|
9663
|
-
await this.store.flush?.(nativeBackboneCoordinateDropTombstoneFile);
|
|
9664
|
-
}
|
|
10379
|
+
await this.persistDropTombstone(files);
|
|
9665
10380
|
await this.eraseDropFiles(files);
|
|
9666
10381
|
this.resetJournalTracking();
|
|
9667
10382
|
this.persistenceLifecycle = "dropped";
|
|
@@ -9699,6 +10414,57 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9699
10414
|
});
|
|
9700
10415
|
}
|
|
9701
10416
|
|
|
10417
|
+
private async barrierFile(name: string): Promise<void> {
|
|
10418
|
+
if (this.store.durableBarrier) {
|
|
10419
|
+
await this.store.durableBarrier(name);
|
|
10420
|
+
} else {
|
|
10421
|
+
await this.store.flush?.(name);
|
|
10422
|
+
}
|
|
10423
|
+
}
|
|
10424
|
+
|
|
10425
|
+
private async hasDurableLegacyMigrationSentinel(): Promise<boolean> {
|
|
10426
|
+
const existing = await this.store.read(this.journalFile);
|
|
10427
|
+
if (
|
|
10428
|
+
existing &&
|
|
10429
|
+
existing.byteLength ===
|
|
10430
|
+
nativeBackboneCoordinateLegacyMigrationSentinel.byteLength &&
|
|
10431
|
+
nativeBackboneCoordinateLegacyMigrationSentinel.every(
|
|
10432
|
+
(byte, index) => existing[index] === byte,
|
|
10433
|
+
)
|
|
10434
|
+
) {
|
|
10435
|
+
// A previous process may have observed the bytes from page cache before
|
|
10436
|
+
// its durability barrier rejected. Cross a fresh barrier on every reopen
|
|
10437
|
+
// before any post-checkpoint WAL append can be admitted.
|
|
10438
|
+
await this.barrierFile(this.journalFile);
|
|
10439
|
+
return true;
|
|
10440
|
+
}
|
|
10441
|
+
return false;
|
|
10442
|
+
}
|
|
10443
|
+
|
|
10444
|
+
private async installLegacyMigrationSentinel(): Promise<void> {
|
|
10445
|
+
if (await this.hasDurableLegacyMigrationSentinel()) {
|
|
10446
|
+
return;
|
|
10447
|
+
}
|
|
10448
|
+
if (!this.store.atomicReplace) {
|
|
10449
|
+
throw new Error(
|
|
10450
|
+
"Native backbone completed checkpoint can not install its downgrade sentinel atomically",
|
|
10451
|
+
);
|
|
10452
|
+
}
|
|
10453
|
+
await this.store.atomicReplace(
|
|
10454
|
+
this.journalFile,
|
|
10455
|
+
nativeBackboneCoordinateLegacyMigrationSentinel,
|
|
10456
|
+
);
|
|
10457
|
+
}
|
|
10458
|
+
|
|
10459
|
+
private async requireLegacyMigrationSentinel(): Promise<void> {
|
|
10460
|
+
if (await this.hasDurableLegacyMigrationSentinel()) {
|
|
10461
|
+
return;
|
|
10462
|
+
}
|
|
10463
|
+
throw new Error(
|
|
10464
|
+
"Native backbone completed checkpoint downgrade sentinel is missing or has been replaced; refusing to overwrite possible legacy writes",
|
|
10465
|
+
);
|
|
10466
|
+
}
|
|
10467
|
+
|
|
9702
10468
|
async hydrate(backbone: NativePeerbitBackbone): Promise<number> {
|
|
9703
10469
|
this.assertActive("hydrate");
|
|
9704
10470
|
// Claim the lifecycle synchronously. close() queues after this complete
|
|
@@ -9709,33 +10475,175 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9709
10475
|
this.assertHydrating();
|
|
9710
10476
|
await this.resumeDropInternal("hydrating");
|
|
9711
10477
|
this.assertHydrating();
|
|
9712
|
-
const
|
|
9713
|
-
|
|
9714
|
-
|
|
9715
|
-
documentSnapshot,
|
|
9716
|
-
documentJournal,
|
|
9717
|
-
documentSignerSnapshot,
|
|
9718
|
-
documentSignerJournal,
|
|
9719
|
-
] = await Promise.all([
|
|
9720
|
-
this.store.read(this.snapshotFile),
|
|
9721
|
-
this.store.read(this.journalFile),
|
|
9722
|
-
this.store.read(this.documentSnapshotFile),
|
|
9723
|
-
this.store.read(this.documentJournalFile),
|
|
9724
|
-
this.store.read(this.documentSignerSnapshotFile),
|
|
9725
|
-
this.store.read(this.documentSignerJournalFile),
|
|
9726
|
-
]);
|
|
10478
|
+
const checkpointStateBytes = await this.store.read(
|
|
10479
|
+
this.checkpointStateFile,
|
|
10480
|
+
);
|
|
9727
10481
|
this.assertHydrating();
|
|
9728
|
-
|
|
9729
|
-
|
|
9730
|
-
|
|
9731
|
-
|
|
10482
|
+
let checkpointState:
|
|
10483
|
+
| NativeBackboneCoordinateCheckpointState
|
|
10484
|
+
| undefined;
|
|
10485
|
+
if (checkpointStateBytes) {
|
|
10486
|
+
try {
|
|
10487
|
+
checkpointState = parseNativeBackboneCoordinateCheckpointState(
|
|
10488
|
+
checkpointStateBytes,
|
|
10489
|
+
this.checkpointStateFile,
|
|
10490
|
+
);
|
|
10491
|
+
if (
|
|
10492
|
+
checkpointState.configurationChecksum !==
|
|
10493
|
+
this.checkpointConfigurationChecksum
|
|
10494
|
+
) {
|
|
10495
|
+
throw new Error(
|
|
10496
|
+
"Native backbone checkpoint authority does not match the configured persistence files",
|
|
10497
|
+
);
|
|
10498
|
+
}
|
|
10499
|
+
} catch (error) {
|
|
10500
|
+
this.persistenceFailure ??= error;
|
|
10501
|
+
throw this.persistenceFailure;
|
|
10502
|
+
}
|
|
10503
|
+
}
|
|
10504
|
+
this.checkpointHighwater = checkpointState?.highwater ?? 0n;
|
|
10505
|
+
this.checkpointPending = checkpointState?.pending;
|
|
10506
|
+
this.checkpointCompleted = checkpointState?.completed;
|
|
10507
|
+
let checkpointAuthority = this.checkpointCompleted;
|
|
10508
|
+
let promotePendingCheckpoint = false;
|
|
10509
|
+
if (!checkpointAuthority && this.checkpointPending) {
|
|
10510
|
+
try {
|
|
10511
|
+
// On the first generation only, the sentinel is the durable stage
|
|
10512
|
+
// boundary: publication writes it after the pending checkpoint and its
|
|
10513
|
+
// WAL headers, but before completed authority. A pre-checkpoint reader
|
|
10514
|
+
// can no longer consume legacy state, so recovery may validate and
|
|
10515
|
+
// promote the staged generation. Once a completed generation exists the
|
|
10516
|
+
// sentinel predates later pending work and is not a promotion signal.
|
|
10517
|
+
if (await this.hasDurableLegacyMigrationSentinel()) {
|
|
10518
|
+
checkpointAuthority = this.checkpointPending;
|
|
10519
|
+
promotePendingCheckpoint = true;
|
|
10520
|
+
}
|
|
10521
|
+
} catch (error) {
|
|
10522
|
+
this.persistenceFailure ??= error;
|
|
10523
|
+
throw this.persistenceFailure;
|
|
10524
|
+
}
|
|
10525
|
+
}
|
|
10526
|
+
|
|
10527
|
+
let snapshot: Uint8Array | undefined;
|
|
10528
|
+
let journal: Uint8Array | undefined;
|
|
10529
|
+
let documentSnapshot: Uint8Array | undefined;
|
|
10530
|
+
let documentJournal: Uint8Array | undefined;
|
|
10531
|
+
let documentSignerSnapshot: Uint8Array | undefined;
|
|
10532
|
+
let documentSignerJournal: Uint8Array | undefined;
|
|
10533
|
+
let journalNames = {
|
|
10534
|
+
coordinate: this.journalFile,
|
|
10535
|
+
document: this.documentJournalFile,
|
|
10536
|
+
signer: this.documentSignerJournalFile,
|
|
10537
|
+
};
|
|
10538
|
+
if (checkpointAuthority) {
|
|
10539
|
+
const authority = checkpointAuthority;
|
|
10540
|
+
const authorityKind = promotePendingCheckpoint
|
|
10541
|
+
? "promotable pending"
|
|
10542
|
+
: "completed";
|
|
10543
|
+
const slot = this.checkpointSlot(authority.generation);
|
|
10544
|
+
const checkpointFile = this.checkpointFiles[slot];
|
|
10545
|
+
journalNames = this.checkpointJournalFiles[slot];
|
|
10546
|
+
const [
|
|
10547
|
+
checkpointBytes,
|
|
10548
|
+
coordinateJournal,
|
|
10549
|
+
valueJournal,
|
|
10550
|
+
signerJournal,
|
|
10551
|
+
] = await Promise.all([
|
|
10552
|
+
this.store.read(checkpointFile),
|
|
10553
|
+
this.store.read(journalNames.coordinate),
|
|
10554
|
+
this.store.read(journalNames.document),
|
|
10555
|
+
this.store.read(journalNames.signer),
|
|
10556
|
+
]);
|
|
10557
|
+
this.assertHydrating();
|
|
10558
|
+
try {
|
|
10559
|
+
if (
|
|
10560
|
+
!checkpointBytes ||
|
|
10561
|
+
checkpointBytes.byteLength !== authority.byteLength ||
|
|
10562
|
+
coordinateJournalChecksum(checkpointBytes) !== authority.checksum
|
|
10563
|
+
) {
|
|
10564
|
+
throw new Error(
|
|
10565
|
+
`Native backbone ${authorityKind} checkpoint authority is missing or corrupt`,
|
|
10566
|
+
);
|
|
10567
|
+
}
|
|
10568
|
+
if (!coordinateJournal || !valueJournal || !signerJournal) {
|
|
10569
|
+
throw new Error(
|
|
10570
|
+
`Native backbone ${authorityKind} checkpoint WAL generation is incomplete`,
|
|
10571
|
+
);
|
|
10572
|
+
}
|
|
10573
|
+
const checkpoint = parseNativeBackboneCoordinateCheckpoint(
|
|
10574
|
+
checkpointBytes,
|
|
10575
|
+
checkpointFile,
|
|
10576
|
+
);
|
|
10577
|
+
if (
|
|
10578
|
+
checkpoint.generation !== authority.generation ||
|
|
10579
|
+
checkpoint.configurationChecksum !==
|
|
10580
|
+
this.checkpointConfigurationChecksum
|
|
10581
|
+
) {
|
|
10582
|
+
throw new Error(
|
|
10583
|
+
`Native backbone checkpoint generation or configuration does not match its ${authorityKind} authority`,
|
|
10584
|
+
);
|
|
10585
|
+
}
|
|
10586
|
+
snapshot = checkpoint.coordinateSnapshot;
|
|
10587
|
+
documentSnapshot = checkpoint.documentSnapshot;
|
|
10588
|
+
documentSignerSnapshot = checkpoint.documentSignerSnapshot;
|
|
10589
|
+
journal = coordinateJournal;
|
|
10590
|
+
documentJournal = valueJournal;
|
|
10591
|
+
documentSignerJournal = signerJournal;
|
|
10592
|
+
} catch (error) {
|
|
10593
|
+
this.persistenceFailure ??= error;
|
|
10594
|
+
throw this.persistenceFailure;
|
|
10595
|
+
}
|
|
10596
|
+
} else {
|
|
10597
|
+
[
|
|
10598
|
+
snapshot,
|
|
10599
|
+
journal,
|
|
10600
|
+
documentSnapshot,
|
|
10601
|
+
documentJournal,
|
|
10602
|
+
documentSignerSnapshot,
|
|
9732
10603
|
documentSignerJournal,
|
|
9733
|
-
|
|
9734
|
-
|
|
10604
|
+
] = await Promise.all([
|
|
10605
|
+
this.store.read(this.snapshotFile),
|
|
10606
|
+
this.store.read(this.journalFile),
|
|
10607
|
+
this.store.read(this.documentSnapshotFile),
|
|
10608
|
+
this.store.read(this.documentJournalFile),
|
|
10609
|
+
this.store.read(this.documentSignerSnapshotFile),
|
|
10610
|
+
this.store.read(this.documentSignerJournalFile),
|
|
10611
|
+
]);
|
|
10612
|
+
this.assertHydrating();
|
|
10613
|
+
}
|
|
10614
|
+
try {
|
|
10615
|
+
if (
|
|
10616
|
+
checkpointAuthority &&
|
|
10617
|
+
(!journal ||
|
|
10618
|
+
!hasCoordinateJournalMagic(journal) ||
|
|
10619
|
+
!documentJournal ||
|
|
10620
|
+
!hasCoordinateJournalMagic(documentJournal) ||
|
|
10621
|
+
!documentSignerJournal ||
|
|
10622
|
+
!hasCoordinateJournalMagic(documentSignerJournal))
|
|
10623
|
+
) {
|
|
10624
|
+
throw new Error(
|
|
10625
|
+
`Native backbone ${promotePendingCheckpoint ? "promotable pending" : "completed"} checkpoint WAL generation has a missing or invalid header`,
|
|
10626
|
+
);
|
|
10627
|
+
}
|
|
10628
|
+
validateCoordinateJournal(journal, journalNames.coordinate);
|
|
10629
|
+
validateCoordinateJournal(documentJournal, journalNames.document);
|
|
10630
|
+
validateCoordinateJournal(documentSignerJournal, journalNames.signer);
|
|
9735
10631
|
} catch (error) {
|
|
9736
10632
|
this.persistenceFailure ??= error;
|
|
9737
10633
|
throw this.persistenceFailure;
|
|
9738
10634
|
}
|
|
10635
|
+
if (checkpointAuthority && !promotePendingCheckpoint) {
|
|
10636
|
+
try {
|
|
10637
|
+
// Completed authority is never permission to overwrite an ordinary
|
|
10638
|
+
// legacy WAL: it may contain writes made by a downgraded process in the
|
|
10639
|
+
// publication cut. Require the exact durable sentinel before loading.
|
|
10640
|
+
await this.requireLegacyMigrationSentinel();
|
|
10641
|
+
this.assertHydrating();
|
|
10642
|
+
} catch (error) {
|
|
10643
|
+
this.persistenceFailure ??= error;
|
|
10644
|
+
throw this.persistenceFailure;
|
|
10645
|
+
}
|
|
10646
|
+
}
|
|
9739
10647
|
let operations: number;
|
|
9740
10648
|
let documentOperations: number;
|
|
9741
10649
|
let documentSignerOperations: number;
|
|
@@ -9767,12 +10675,52 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9767
10675
|
}
|
|
9768
10676
|
throw error;
|
|
9769
10677
|
}
|
|
10678
|
+
if (promotePendingCheckpoint) {
|
|
10679
|
+
try {
|
|
10680
|
+
const authority = checkpointAuthority!;
|
|
10681
|
+
if (!this.store.atomicReplace) {
|
|
10682
|
+
throw new Error(
|
|
10683
|
+
"Native backbone pending checkpoint can not publish completed authority atomically",
|
|
10684
|
+
);
|
|
10685
|
+
}
|
|
10686
|
+
await this.store.atomicReplace(
|
|
10687
|
+
this.checkpointStateFile,
|
|
10688
|
+
encodeNativeBackboneCoordinateCheckpointState({
|
|
10689
|
+
configurationChecksum: this.checkpointConfigurationChecksum,
|
|
10690
|
+
highwater: authority.generation,
|
|
10691
|
+
completed: authority,
|
|
10692
|
+
}),
|
|
10693
|
+
);
|
|
10694
|
+
this.checkpointCompleted = authority;
|
|
10695
|
+
this.checkpointPending = undefined;
|
|
10696
|
+
this.assertHydrating();
|
|
10697
|
+
} catch (error) {
|
|
10698
|
+
this.persistenceFailure ??= error;
|
|
10699
|
+
throw this.persistenceFailure;
|
|
10700
|
+
}
|
|
10701
|
+
}
|
|
10702
|
+
if (this.checkpointCompleted || this.checkpointPending) {
|
|
10703
|
+
try {
|
|
10704
|
+
await this.cleanupRetiredCheckpointFiles();
|
|
10705
|
+
this.assertHydrating();
|
|
10706
|
+
} catch (error) {
|
|
10707
|
+
this.persistenceFailure ??= error;
|
|
10708
|
+
throw this.persistenceFailure;
|
|
10709
|
+
}
|
|
10710
|
+
}
|
|
9770
10711
|
this.assertHydrating();
|
|
9771
10712
|
this.journalInitialized = !!journal && journal.byteLength > 0;
|
|
10713
|
+
this.journalByteLength = journal?.byteLength ?? 0;
|
|
10714
|
+
this.journalRecordCount = operations;
|
|
9772
10715
|
this.documentJournalInitialized =
|
|
9773
10716
|
!!documentJournal && documentJournal.byteLength > 0;
|
|
10717
|
+
this.documentJournalByteLength = documentJournal?.byteLength ?? 0;
|
|
10718
|
+
this.documentJournalRecordCount = documentOperations;
|
|
9774
10719
|
this.documentSignerJournalInitialized =
|
|
9775
10720
|
!!documentSignerJournal && documentSignerJournal.byteLength > 0;
|
|
10721
|
+
this.documentSignerJournalByteLength =
|
|
10722
|
+
documentSignerJournal?.byteLength ?? 0;
|
|
10723
|
+
this.documentSignerJournalRecordCount = documentSignerOperations;
|
|
9776
10724
|
backbone.setCoordinateJournalEnabled(true);
|
|
9777
10725
|
backbone.setDocumentJournalEnabled(true);
|
|
9778
10726
|
backbone.setDocumentSignerJournalEnabled(true);
|
|
@@ -9815,15 +10763,19 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9815
10763
|
return false;
|
|
9816
10764
|
}
|
|
9817
10765
|
let tombstone: NativeBackboneCoordinateDropTombstoneBody;
|
|
10766
|
+
let expandedFiles: string[];
|
|
9818
10767
|
try {
|
|
9819
10768
|
tombstone = parseNativeBackboneCoordinateDropTombstone(bytes);
|
|
9820
|
-
for (const file of this.
|
|
10769
|
+
for (const file of this.legacyConfiguredFiles()) {
|
|
9821
10770
|
if (!tombstone.files.includes(file)) {
|
|
9822
10771
|
throw new Error(
|
|
9823
10772
|
"Native backbone drop tombstone does not cover the configured namespace",
|
|
9824
10773
|
);
|
|
9825
10774
|
}
|
|
9826
10775
|
}
|
|
10776
|
+
expandedFiles = [
|
|
10777
|
+
...new Set([...tombstone.files, ...this.configuredFiles()]),
|
|
10778
|
+
];
|
|
9827
10779
|
} catch (error) {
|
|
9828
10780
|
// Corruption must never hydrate stale files, but an explicit drop must
|
|
9829
10781
|
// remain available to overwrite the bad marker and erase the namespace.
|
|
@@ -9831,6 +10783,17 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9831
10783
|
this.persistenceLifecycle = this.closePromise ? "closing" : "active";
|
|
9832
10784
|
throw this.persistenceFailure;
|
|
9833
10785
|
}
|
|
10786
|
+
if (expandedFiles.length !== tombstone.files.length) {
|
|
10787
|
+
// Older valid markers only knew the six snapshot/WAL files. Widen the
|
|
10788
|
+
// destructive intent durably before erasing any checkpoint generation,
|
|
10789
|
+
// while preserving additional namespace files recorded by that version.
|
|
10790
|
+
await this.persistDropTombstone(expandedFiles, true);
|
|
10791
|
+
tombstone = { ...tombstone, files: expandedFiles };
|
|
10792
|
+
}
|
|
10793
|
+
// A prior replacement attempt may have made the expanded bytes visible before
|
|
10794
|
+
// its durability barrier rejected. Cross a fresh barrier on every recovery
|
|
10795
|
+
// before allowing any target removal.
|
|
10796
|
+
await this.barrierFile(nativeBackboneCoordinateDropTombstoneFile);
|
|
9834
10797
|
await this.eraseDropFiles(tombstone.files);
|
|
9835
10798
|
this.resetJournalTracking();
|
|
9836
10799
|
this.persistenceLifecycle =
|
|
@@ -9887,7 +10850,13 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9887
10850
|
this.assertActive("flush");
|
|
9888
10851
|
// Serialized with compact() so a flush never clears records appended to
|
|
9889
10852
|
// the wasm journal while a previous flush was awaiting its disk write.
|
|
9890
|
-
return this.enqueuePersistence(() =>
|
|
10853
|
+
return this.enqueuePersistence(() => {
|
|
10854
|
+
// A call can be admitted while an earlier queued write is still in flight.
|
|
10855
|
+
// Re-check the sticky failure at execution time so no suffix can ACK after
|
|
10856
|
+
// that earlier write or authority switch failed.
|
|
10857
|
+
this.assertPersistenceHealthy();
|
|
10858
|
+
return this.flushJournalInternal(backbone);
|
|
10859
|
+
});
|
|
9891
10860
|
}
|
|
9892
10861
|
|
|
9893
10862
|
private enqueuePersistence<T>(fn: () => Promise<T>): Promise<T> {
|
|
@@ -9908,9 +10877,185 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9908
10877
|
return next;
|
|
9909
10878
|
}
|
|
9910
10879
|
|
|
10880
|
+
private shouldCompactJournal(
|
|
10881
|
+
additionalBytes: number,
|
|
10882
|
+
additionalRecords: number,
|
|
10883
|
+
): boolean {
|
|
10884
|
+
return (
|
|
10885
|
+
(this.compactMaxJournalBytes != null &&
|
|
10886
|
+
this.journalByteLength +
|
|
10887
|
+
this.documentJournalByteLength +
|
|
10888
|
+
this.documentSignerJournalByteLength +
|
|
10889
|
+
additionalBytes >=
|
|
10890
|
+
this.compactMaxJournalBytes) ||
|
|
10891
|
+
(this.compactMaxJournalRecords != null &&
|
|
10892
|
+
this.journalRecordCount +
|
|
10893
|
+
this.documentJournalRecordCount +
|
|
10894
|
+
this.documentSignerJournalRecordCount +
|
|
10895
|
+
additionalRecords >=
|
|
10896
|
+
this.compactMaxJournalRecords)
|
|
10897
|
+
);
|
|
10898
|
+
}
|
|
10899
|
+
|
|
10900
|
+
private nextCheckpointGeneration(): bigint {
|
|
10901
|
+
let generation = this.checkpointHighwater + 1n;
|
|
10902
|
+
const active = this.checkpointCompleted;
|
|
10903
|
+
if (
|
|
10904
|
+
active &&
|
|
10905
|
+
this.checkpointSlot(generation) === this.checkpointSlot(active.generation)
|
|
10906
|
+
) {
|
|
10907
|
+
generation += 1n;
|
|
10908
|
+
}
|
|
10909
|
+
if (generation > nativeBackboneCoordinateMaxU64) {
|
|
10910
|
+
throw new RangeError(
|
|
10911
|
+
"Native backbone checkpoint generation highwater is exhausted",
|
|
10912
|
+
);
|
|
10913
|
+
}
|
|
10914
|
+
return generation;
|
|
10915
|
+
}
|
|
10916
|
+
|
|
10917
|
+
private async publishCheckpoint(checkpoint: {
|
|
10918
|
+
coordinateSnapshot: Uint8Array;
|
|
10919
|
+
documentSnapshot: Uint8Array;
|
|
10920
|
+
documentSignerSnapshot: Uint8Array;
|
|
10921
|
+
}): Promise<void> {
|
|
10922
|
+
if (
|
|
10923
|
+
!this.crashSafeCompaction ||
|
|
10924
|
+
!this.store.atomicReplace ||
|
|
10925
|
+
!this.store.remove
|
|
10926
|
+
) {
|
|
10927
|
+
throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
|
|
10928
|
+
}
|
|
10929
|
+
const generation = this.nextCheckpointGeneration();
|
|
10930
|
+
const slot = this.checkpointSlot(generation);
|
|
10931
|
+
const checkpointBytes = encodeNativeBackboneCoordinateCheckpoint({
|
|
10932
|
+
configurationChecksum: this.checkpointConfigurationChecksum,
|
|
10933
|
+
generation,
|
|
10934
|
+
...checkpoint,
|
|
10935
|
+
});
|
|
10936
|
+
const authority: NativeBackboneCoordinateCheckpointAuthority = {
|
|
10937
|
+
generation,
|
|
10938
|
+
byteLength: checkpointBytes.byteLength,
|
|
10939
|
+
checksum: coordinateJournalChecksum(checkpointBytes),
|
|
10940
|
+
};
|
|
10941
|
+
const pendingState: NativeBackboneCoordinateCheckpointState = {
|
|
10942
|
+
configurationChecksum: this.checkpointConfigurationChecksum,
|
|
10943
|
+
highwater: generation,
|
|
10944
|
+
pending: authority,
|
|
10945
|
+
completed: this.checkpointCompleted,
|
|
10946
|
+
};
|
|
10947
|
+
await this.store.atomicReplace(
|
|
10948
|
+
this.checkpointStateFile,
|
|
10949
|
+
encodeNativeBackboneCoordinateCheckpointState(pendingState),
|
|
10950
|
+
);
|
|
10951
|
+
this.checkpointHighwater = generation;
|
|
10952
|
+
this.checkpointPending = authority;
|
|
10953
|
+
|
|
10954
|
+
const targetJournals = this.checkpointJournalFiles[slot];
|
|
10955
|
+
for (const file of [
|
|
10956
|
+
targetJournals.coordinate,
|
|
10957
|
+
targetJournals.document,
|
|
10958
|
+
targetJournals.signer,
|
|
10959
|
+
]) {
|
|
10960
|
+
await this.store.remove(file);
|
|
10961
|
+
}
|
|
10962
|
+
const journalHeaders = [
|
|
10963
|
+
[targetJournals.coordinate, nativeBackboneCoordinateJournalMagic],
|
|
10964
|
+
[targetJournals.document, nativeBackboneCoordinateJournalMagic],
|
|
10965
|
+
[targetJournals.signer, nativeBackboneCoordinateJournalMagic],
|
|
10966
|
+
] as const;
|
|
10967
|
+
for (const [file, header] of journalHeaders) {
|
|
10968
|
+
await this.store.write(file, header);
|
|
10969
|
+
await this.barrierFile(file);
|
|
10970
|
+
}
|
|
10971
|
+
await this.store.atomicReplace(this.checkpointFiles[slot], checkpointBytes);
|
|
10972
|
+
|
|
10973
|
+
// A prior release does not understand checkpoint authority. The sentinel is
|
|
10974
|
+
// the first-generation publication boundary: install and durably revalidate
|
|
10975
|
+
// it before completed authority becomes visible. On later generations it
|
|
10976
|
+
// must already be exact; never overwrite possible writes from a downgrade.
|
|
10977
|
+
if (this.checkpointCompleted) {
|
|
10978
|
+
await this.requireLegacyMigrationSentinel();
|
|
10979
|
+
} else {
|
|
10980
|
+
await this.installLegacyMigrationSentinel();
|
|
10981
|
+
}
|
|
10982
|
+
|
|
10983
|
+
const completedState: NativeBackboneCoordinateCheckpointState = {
|
|
10984
|
+
configurationChecksum: this.checkpointConfigurationChecksum,
|
|
10985
|
+
highwater: generation,
|
|
10986
|
+
completed: authority,
|
|
10987
|
+
};
|
|
10988
|
+
await this.store.atomicReplace(
|
|
10989
|
+
this.checkpointStateFile,
|
|
10990
|
+
encodeNativeBackboneCoordinateCheckpointState(completedState),
|
|
10991
|
+
);
|
|
10992
|
+
|
|
10993
|
+
// The completed authority is now the only writable generation. Update all
|
|
10994
|
+
// in-memory routing synchronously before another queued flush can start.
|
|
10995
|
+
this.checkpointCompleted = authority;
|
|
10996
|
+
this.checkpointPending = undefined;
|
|
10997
|
+
this.journalInitialized = true;
|
|
10998
|
+
this.documentJournalInitialized = true;
|
|
10999
|
+
this.documentSignerJournalInitialized = true;
|
|
11000
|
+
this.journalByteLength = nativeBackboneCoordinateJournalMagic.byteLength;
|
|
11001
|
+
this.documentJournalByteLength =
|
|
11002
|
+
nativeBackboneCoordinateJournalMagic.byteLength;
|
|
11003
|
+
this.documentSignerJournalByteLength =
|
|
11004
|
+
nativeBackboneCoordinateJournalMagic.byteLength;
|
|
11005
|
+
this.journalRecordCount = 0;
|
|
11006
|
+
this.documentJournalRecordCount = 0;
|
|
11007
|
+
this.documentSignerJournalRecordCount = 0;
|
|
11008
|
+
}
|
|
11009
|
+
|
|
11010
|
+
private async cleanupRetiredCheckpointFiles(): Promise<void> {
|
|
11011
|
+
if (
|
|
11012
|
+
!this.store.remove ||
|
|
11013
|
+
(!this.checkpointCompleted && !this.checkpointPending)
|
|
11014
|
+
) {
|
|
11015
|
+
return;
|
|
11016
|
+
}
|
|
11017
|
+
const retiredSlot: NativeBackboneCoordinateCheckpointSlot = this
|
|
11018
|
+
.checkpointCompleted
|
|
11019
|
+
? this.checkpointSlot(this.checkpointCompleted.generation) === "a"
|
|
11020
|
+
? "b"
|
|
11021
|
+
: "a"
|
|
11022
|
+
: this.checkpointSlot(this.checkpointPending!.generation);
|
|
11023
|
+
const retiredJournals = this.checkpointJournalFiles[retiredSlot];
|
|
11024
|
+
const files = [
|
|
11025
|
+
...(this.checkpointCompleted
|
|
11026
|
+
? [
|
|
11027
|
+
this.snapshotFile,
|
|
11028
|
+
this.documentSnapshotFile,
|
|
11029
|
+
this.documentJournalFile,
|
|
11030
|
+
this.documentSignerSnapshotFile,
|
|
11031
|
+
this.documentSignerJournalFile,
|
|
11032
|
+
]
|
|
11033
|
+
: []),
|
|
11034
|
+
this.checkpointFiles[retiredSlot],
|
|
11035
|
+
`${this.checkpointFiles[retiredSlot]}.tmp`,
|
|
11036
|
+
retiredJournals.coordinate,
|
|
11037
|
+
retiredJournals.document,
|
|
11038
|
+
retiredJournals.signer,
|
|
11039
|
+
];
|
|
11040
|
+
const removals = await Promise.allSettled(
|
|
11041
|
+
files.map((file) => this.store.remove!(file)),
|
|
11042
|
+
);
|
|
11043
|
+
if (removals.every((result) => result.status === "fulfilled")) {
|
|
11044
|
+
try {
|
|
11045
|
+
await this.store.durableBarrier?.();
|
|
11046
|
+
} catch {
|
|
11047
|
+
// Cleanup is retryable debt. Publication and the caller's ACK already
|
|
11048
|
+
// belong to the new generation, so never turn this into an ambiguous
|
|
11049
|
+
// append failure after the authority switch.
|
|
11050
|
+
}
|
|
11051
|
+
}
|
|
11052
|
+
}
|
|
11053
|
+
|
|
9911
11054
|
private async flushJournalInternal(
|
|
9912
11055
|
backbone: NativePeerbitBackbone,
|
|
11056
|
+
forceCheckpoint = false,
|
|
9913
11057
|
): Promise<number> {
|
|
11058
|
+
this.assertPersistenceHealthy();
|
|
9914
11059
|
let written = 0;
|
|
9915
11060
|
let persistenceMutationStarted = false;
|
|
9916
11061
|
let coordinateBytes: Uint8Array | undefined;
|
|
@@ -9922,10 +11067,42 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9922
11067
|
const documentRecordCount = backbone.documentPendingJournalLength;
|
|
9923
11068
|
const signerRecords = backbone.documentSignerJournal();
|
|
9924
11069
|
const signerRecordCount = backbone.documentSignerPendingJournalLength;
|
|
11070
|
+
const additionalBytes =
|
|
11071
|
+
coordinateRecords.byteLength +
|
|
11072
|
+
documentRecords.byteLength +
|
|
11073
|
+
signerRecords.byteLength +
|
|
11074
|
+
(coordinateRecords.byteLength > 0 && this.journalInitialized !== true
|
|
11075
|
+
? backbone.coordinateJournalHeader().byteLength
|
|
11076
|
+
: 0) +
|
|
11077
|
+
(documentRecords.byteLength > 0 &&
|
|
11078
|
+
this.documentJournalInitialized !== true
|
|
11079
|
+
? backbone.documentJournalHeader().byteLength
|
|
11080
|
+
: 0) +
|
|
11081
|
+
(signerRecords.byteLength > 0 &&
|
|
11082
|
+
this.documentSignerJournalInitialized !== true
|
|
11083
|
+
? backbone.documentSignerJournalHeader().byteLength
|
|
11084
|
+
: 0);
|
|
11085
|
+
const additionalRecords =
|
|
11086
|
+
coordinateRecordCount + documentRecordCount + signerRecordCount;
|
|
11087
|
+
const checkpointRequested =
|
|
11088
|
+
forceCheckpoint ||
|
|
11089
|
+
(this.crashSafeCompaction &&
|
|
11090
|
+
this.shouldCompactJournal(additionalBytes, additionalRecords));
|
|
11091
|
+
// These three synchronous copies and the journal prefixes above are one
|
|
11092
|
+
// logical cut. No mutation can slip into the checkpoint without also being
|
|
11093
|
+
// represented by one of the captured prefixes.
|
|
11094
|
+
const checkpoint = checkpointRequested
|
|
11095
|
+
? {
|
|
11096
|
+
coordinateSnapshot: backbone.coordinateSnapshot(),
|
|
11097
|
+
documentSnapshot: backbone.documentSnapshot(),
|
|
11098
|
+
documentSignerSnapshot: backbone.documentSignerSnapshot(),
|
|
11099
|
+
}
|
|
11100
|
+
: undefined;
|
|
9925
11101
|
try {
|
|
11102
|
+
const activeJournals = this.activeJournalFiles();
|
|
9926
11103
|
if (coordinateRecords.byteLength > 0) {
|
|
9927
11104
|
if (this.journalInitialized === undefined) {
|
|
9928
|
-
const existing = await this.store.read(
|
|
11105
|
+
const existing = await this.store.read(activeJournals.coordinate);
|
|
9929
11106
|
this.journalInitialized = !!existing && existing.byteLength > 0;
|
|
9930
11107
|
}
|
|
9931
11108
|
coordinateBytes = this.journalInitialized
|
|
@@ -9934,28 +11111,26 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9934
11111
|
backbone.coordinateJournalHeader(),
|
|
9935
11112
|
coordinateRecords,
|
|
9936
11113
|
]);
|
|
9937
|
-
await this.store.append(
|
|
11114
|
+
await this.store.append(activeJournals.coordinate, coordinateBytes);
|
|
9938
11115
|
persistenceMutationStarted = true;
|
|
9939
11116
|
written += coordinateRecords.byteLength;
|
|
9940
11117
|
}
|
|
9941
11118
|
if (documentRecords.byteLength > 0) {
|
|
9942
11119
|
if (this.documentJournalInitialized === undefined) {
|
|
9943
|
-
const existing = await this.store.read(
|
|
11120
|
+
const existing = await this.store.read(activeJournals.document);
|
|
9944
11121
|
this.documentJournalInitialized =
|
|
9945
11122
|
!!existing && existing.byteLength > 0;
|
|
9946
11123
|
}
|
|
9947
11124
|
documentBytes = this.documentJournalInitialized
|
|
9948
11125
|
? documentRecords
|
|
9949
11126
|
: concatBytes([backbone.documentJournalHeader(), documentRecords]);
|
|
9950
|
-
await this.store.append(
|
|
11127
|
+
await this.store.append(activeJournals.document, documentBytes);
|
|
9951
11128
|
persistenceMutationStarted = true;
|
|
9952
11129
|
written += documentRecords.byteLength;
|
|
9953
11130
|
}
|
|
9954
11131
|
if (signerRecords.byteLength > 0) {
|
|
9955
11132
|
if (this.documentSignerJournalInitialized === undefined) {
|
|
9956
|
-
const existing = await this.store.read(
|
|
9957
|
-
this.documentSignerJournalFile,
|
|
9958
|
-
);
|
|
11133
|
+
const existing = await this.store.read(activeJournals.signer);
|
|
9959
11134
|
this.documentSignerJournalInitialized =
|
|
9960
11135
|
!!existing && existing.byteLength > 0;
|
|
9961
11136
|
}
|
|
@@ -9965,11 +11140,11 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9965
11140
|
backbone.documentSignerJournalHeader(),
|
|
9966
11141
|
signerRecords,
|
|
9967
11142
|
]);
|
|
9968
|
-
await this.store.append(
|
|
11143
|
+
await this.store.append(activeJournals.signer, signerBytes);
|
|
9969
11144
|
persistenceMutationStarted = true;
|
|
9970
11145
|
written += signerRecords.byteLength;
|
|
9971
11146
|
}
|
|
9972
|
-
if (written === 0) {
|
|
11147
|
+
if (written === 0 && !checkpoint) {
|
|
9973
11148
|
this.lastFlushMs = Date.now();
|
|
9974
11149
|
return 0;
|
|
9975
11150
|
}
|
|
@@ -9977,27 +11152,34 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
9977
11152
|
// `append` may only enqueue bytes in a buffered store. Drain and fsync
|
|
9978
11153
|
// every affected WAL before clearing its wasm prefix or returning an ACK.
|
|
9979
11154
|
for (const file of [
|
|
9980
|
-
coordinateBytes ?
|
|
9981
|
-
documentBytes ?
|
|
9982
|
-
signerBytes ?
|
|
11155
|
+
coordinateBytes ? activeJournals.coordinate : undefined,
|
|
11156
|
+
documentBytes ? activeJournals.document : undefined,
|
|
11157
|
+
signerBytes ? activeJournals.signer : undefined,
|
|
9983
11158
|
]) {
|
|
9984
11159
|
if (file) {
|
|
9985
|
-
|
|
9986
|
-
await this.store.durableBarrier(file);
|
|
9987
|
-
} else {
|
|
9988
|
-
await this.store.flush?.(file);
|
|
9989
|
-
}
|
|
11160
|
+
await this.barrierFile(file);
|
|
9990
11161
|
}
|
|
9991
11162
|
}
|
|
9992
11163
|
|
|
9993
11164
|
if (coordinateBytes) {
|
|
9994
11165
|
this.journalInitialized = true;
|
|
11166
|
+
this.journalByteLength += coordinateBytes.byteLength;
|
|
11167
|
+
this.journalRecordCount += coordinateRecordCount;
|
|
9995
11168
|
}
|
|
9996
11169
|
if (documentBytes) {
|
|
9997
11170
|
this.documentJournalInitialized = true;
|
|
11171
|
+
this.documentJournalByteLength += documentBytes.byteLength;
|
|
11172
|
+
this.documentJournalRecordCount += documentRecordCount;
|
|
9998
11173
|
}
|
|
9999
11174
|
if (signerBytes) {
|
|
10000
11175
|
this.documentSignerJournalInitialized = true;
|
|
11176
|
+
this.documentSignerJournalByteLength += signerBytes.byteLength;
|
|
11177
|
+
this.documentSignerJournalRecordCount += signerRecordCount;
|
|
11178
|
+
}
|
|
11179
|
+
|
|
11180
|
+
if (checkpoint) {
|
|
11181
|
+
persistenceMutationStarted = true;
|
|
11182
|
+
await this.publishCheckpoint(checkpoint);
|
|
10001
11183
|
}
|
|
10002
11184
|
|
|
10003
11185
|
backbone.clearCoordinateJournalPrefix(
|
|
@@ -10012,6 +11194,9 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
10012
11194
|
signerRecords.byteLength,
|
|
10013
11195
|
signerRecordCount,
|
|
10014
11196
|
);
|
|
11197
|
+
if (checkpoint) {
|
|
11198
|
+
await this.cleanupRetiredCheckpointFiles();
|
|
11199
|
+
}
|
|
10015
11200
|
this.lastFlushMs = Date.now();
|
|
10016
11201
|
return written;
|
|
10017
11202
|
} catch (error) {
|
|
@@ -10023,9 +11208,15 @@ export class NativeBackboneCoordinatePersistence {
|
|
|
10023
11208
|
}
|
|
10024
11209
|
}
|
|
10025
11210
|
|
|
10026
|
-
async compact(
|
|
11211
|
+
async compact(backbone: NativePeerbitBackbone): Promise<void> {
|
|
10027
11212
|
this.assertActive("compact");
|
|
10028
|
-
|
|
11213
|
+
if (!this.crashSafeCompaction) {
|
|
11214
|
+
throw new Error(nativeBackboneCoordinateCompactionDisabledMessage);
|
|
11215
|
+
}
|
|
11216
|
+
await this.enqueuePersistence(async () => {
|
|
11217
|
+
this.assertPersistenceHealthy();
|
|
11218
|
+
await this.flushJournalInternal(backbone, true);
|
|
11219
|
+
});
|
|
10029
11220
|
}
|
|
10030
11221
|
|
|
10031
11222
|
close(): Promise<void> {
|