@peerbit/native-backbone 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/src/durability/codec.d.ts +86 -0
- package/dist/src/durability/codec.d.ts.map +1 -0
- package/dist/src/durability/codec.js +365 -0
- package/dist/src/durability/codec.js.map +1 -0
- package/dist/src/durability/lease.d.ts +59 -0
- package/dist/src/durability/lease.d.ts.map +1 -0
- package/dist/src/durability/lease.js +48 -0
- package/dist/src/durability/lease.js.map +1 -0
- package/dist/src/durability/memory-storage.d.ts +37 -0
- package/dist/src/durability/memory-storage.d.ts.map +1 -0
- package/dist/src/durability/memory-storage.js +436 -0
- package/dist/src/durability/memory-storage.js.map +1 -0
- package/dist/src/durability/node-lease.d.ts +14 -0
- package/dist/src/durability/node-lease.d.ts.map +1 -0
- package/dist/src/durability/node-lease.js +214 -0
- package/dist/src/durability/node-lease.js.map +1 -0
- package/dist/src/durability/node-storage.d.ts +76 -0
- package/dist/src/durability/node-storage.d.ts.map +1 -0
- package/dist/src/durability/node-storage.js +1813 -0
- package/dist/src/durability/node-storage.js.map +1 -0
- package/dist/src/durability/storage.d.ts +224 -0
- package/dist/src/durability/storage.d.ts.map +1 -0
- package/dist/src/durability/storage.js +343 -0
- package/dist/src/durability/storage.js.map +1 -0
- package/dist/src/index.d.ts +93 -15
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +717 -199
- package/dist/src/index.js.map +1 -1
- package/dist/wasm/README.md +31 -0
- package/dist/wasm/native_backbone.d.ts +93 -77
- package/dist/wasm/native_backbone.js +96 -0
- package/dist/wasm/native_backbone_bg.wasm +0 -0
- package/dist/wasm/native_backbone_bg.wasm.d.ts +81 -77
- package/package.json +4 -3
- package/src/append_tx/committed_latest.rs +24 -43
- package/src/documents.rs +7 -9
- package/src/durability/codec.ts +683 -0
- package/src/durability/lease.ts +87 -0
- package/src/durability/memory-storage.ts +593 -0
- package/src/durability/node-lease.ts +293 -0
- package/src/durability/node-storage.ts +2798 -0
- package/src/durability/storage.ts +682 -0
- package/src/durability.rs +1872 -0
- package/src/error.rs +8 -0
- package/src/index.ts +1392 -735
- package/src/lib.rs +1 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
export type NativeDurabilityFence = Readonly<{
|
|
2
|
+
epoch: bigint;
|
|
3
|
+
ownerId: string;
|
|
4
|
+
domainId: string;
|
|
5
|
+
}>;
|
|
6
|
+
|
|
7
|
+
export const NATIVE_DURABILITY_MAX_U64 = (1n << 64n) - 1n;
|
|
8
|
+
export const NATIVE_DURABILITY_MAX_WRITER_ID_BYTES = 1024;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Exclusive ownership capability for one native durability directory.
|
|
12
|
+
*
|
|
13
|
+
* Implementations keep the underlying ownership primitive alive until
|
|
14
|
+
* `close()` completes. Storage writers must wrap the complete asynchronous
|
|
15
|
+
* mutation and its barrier in `runWhileHeld()` and persist the accompanying
|
|
16
|
+
* fence with their records. `assertHeld()` is diagnostic only; it is not a
|
|
17
|
+
* lifecycle guard for an operation that later awaits I/O.
|
|
18
|
+
*/
|
|
19
|
+
export interface NativeDurabilityLease {
|
|
20
|
+
readonly fence: NativeDurabilityFence;
|
|
21
|
+
assertHeld(): Promise<void>;
|
|
22
|
+
/**
|
|
23
|
+
* Keep the lease alive for the complete asynchronous operation. `close()`
|
|
24
|
+
* starts rejecting new operations immediately and waits for operations that
|
|
25
|
+
* already entered this guard before releasing the underlying OS lock.
|
|
26
|
+
*/
|
|
27
|
+
runWhileHeld<T>(operation: () => Promise<T>): Promise<T>;
|
|
28
|
+
close(): Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export class NativeDurabilityLeaseUnavailableError extends Error {
|
|
32
|
+
readonly code = "NATIVE_DURABILITY_LEASE_UNAVAILABLE";
|
|
33
|
+
|
|
34
|
+
constructor(
|
|
35
|
+
readonly directory: string,
|
|
36
|
+
options?: { cause?: unknown },
|
|
37
|
+
) {
|
|
38
|
+
super(`Native durability directory is already open: ${directory}`, options);
|
|
39
|
+
this.name = "NativeDurabilityLeaseUnavailableError";
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export class NativeDurabilityLeaseClosedError extends Error {
|
|
44
|
+
readonly code = "NATIVE_DURABILITY_LEASE_CLOSED";
|
|
45
|
+
|
|
46
|
+
constructor(readonly fence: NativeDurabilityFence) {
|
|
47
|
+
super(`Native durability lease is no longer held: ${fence.domainId}`);
|
|
48
|
+
this.name = "NativeDurabilityLeaseClosedError";
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export class NativeDurabilityLeaseStateError extends Error {
|
|
53
|
+
readonly code = "NATIVE_DURABILITY_LEASE_STATE_INVALID";
|
|
54
|
+
|
|
55
|
+
constructor(
|
|
56
|
+
readonly directory: string,
|
|
57
|
+
message: string,
|
|
58
|
+
options?: { cause?: unknown },
|
|
59
|
+
) {
|
|
60
|
+
super(message, options);
|
|
61
|
+
this.name = "NativeDurabilityLeaseStateError";
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export class NativeDurabilityLeaseDirectorySyncError extends Error {
|
|
66
|
+
readonly code = "NATIVE_DURABILITY_LEASE_DIRECTORY_SYNC_FAILED";
|
|
67
|
+
|
|
68
|
+
constructor(
|
|
69
|
+
readonly directory: string,
|
|
70
|
+
options?: { cause?: unknown },
|
|
71
|
+
) {
|
|
72
|
+
super(
|
|
73
|
+
`Native durability requires directory fsync support: ${directory}`,
|
|
74
|
+
options,
|
|
75
|
+
);
|
|
76
|
+
this.name = "NativeDurabilityLeaseDirectorySyncError";
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export class NativeDurabilityFenceExhaustedError extends Error {
|
|
81
|
+
readonly code = "NATIVE_DURABILITY_FENCE_EXHAUSTED";
|
|
82
|
+
|
|
83
|
+
constructor(readonly directory: string) {
|
|
84
|
+
super(`Native durability fence epoch is exhausted: ${directory}`);
|
|
85
|
+
this.name = "NativeDurabilityFenceExhaustedError";
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,593 @@
|
|
|
1
|
+
import {
|
|
2
|
+
NATIVE_DURABILITY_MAX_U64,
|
|
3
|
+
type NativeDurabilityLease,
|
|
4
|
+
} from "./lease.js";
|
|
5
|
+
import {
|
|
6
|
+
NATIVE_DURABILITY_STORAGE_VERSION,
|
|
7
|
+
type NativeDurabilityCheckpoint,
|
|
8
|
+
type NativeDurabilityCheckpointReceipt,
|
|
9
|
+
type NativeDurabilityCheckpointRequest,
|
|
10
|
+
type NativeDurabilityDeleteReceipt,
|
|
11
|
+
type NativeDurabilityDeleteRequest,
|
|
12
|
+
NativeDurabilityDigestMismatchError,
|
|
13
|
+
NativeDurabilityIncompleteTailMismatchError,
|
|
14
|
+
type NativeDurabilityIncompleteTailReconciliationRequest,
|
|
15
|
+
type NativeDurabilityJournalAppendRequest,
|
|
16
|
+
type NativeDurabilityJournalClassifier,
|
|
17
|
+
NativeDurabilityJournalOffsetConflictError,
|
|
18
|
+
type NativeDurabilityJournalReceipt,
|
|
19
|
+
type NativeDurabilityJournalReconciliationReceipt,
|
|
20
|
+
type NativeDurabilityStageReceipt,
|
|
21
|
+
type NativeDurabilityStageRequest,
|
|
22
|
+
type NativeDurabilityStagedBlockReference,
|
|
23
|
+
type NativeDurabilityStagingManifest,
|
|
24
|
+
type NativeDurabilityStorage,
|
|
25
|
+
NativeDurabilityStorageClosedError,
|
|
26
|
+
type NativeDurabilityStorageStats,
|
|
27
|
+
assertNativeDurabilityCheckpointRequest,
|
|
28
|
+
assertNativeDurabilityDeleteRequest,
|
|
29
|
+
assertNativeDurabilityFence,
|
|
30
|
+
assertNativeDurabilityJournalAppendRequest,
|
|
31
|
+
assertNativeDurabilityOperationScope,
|
|
32
|
+
assertNativeDurabilityStageRequest,
|
|
33
|
+
copyNativeDurabilityBytes,
|
|
34
|
+
nativeDurabilityBytesEqual,
|
|
35
|
+
nativeDurabilityScopeDigest,
|
|
36
|
+
sha256NativeDurability,
|
|
37
|
+
} from "./storage.js";
|
|
38
|
+
|
|
39
|
+
const cloneFence = (
|
|
40
|
+
fence: NativeDurabilityLease["fence"],
|
|
41
|
+
): NativeDurabilityLease["fence"] => ({ ...fence });
|
|
42
|
+
|
|
43
|
+
const cloneScope = <T extends { transactionId: string; txSequence: bigint }>(
|
|
44
|
+
scope: T,
|
|
45
|
+
): T => ({ ...scope });
|
|
46
|
+
|
|
47
|
+
const cloneRetainedTransactions = (
|
|
48
|
+
transactions: NativeDurabilityCheckpointRequest["retainedTransactions"],
|
|
49
|
+
): NativeDurabilityCheckpoint["retainedTransactions"] =>
|
|
50
|
+
transactions.map((transaction) => ({
|
|
51
|
+
...transaction,
|
|
52
|
+
planDigest: copyNativeDurabilityBytes(transaction.planDigest),
|
|
53
|
+
}));
|
|
54
|
+
|
|
55
|
+
const snapshotStageRequest = (
|
|
56
|
+
request: NativeDurabilityStageRequest,
|
|
57
|
+
): NativeDurabilityStageRequest => {
|
|
58
|
+
assertNativeDurabilityStageRequest(request);
|
|
59
|
+
return {
|
|
60
|
+
scope: { ...request.scope },
|
|
61
|
+
blocks: [...request.blocks]
|
|
62
|
+
.sort((left, right) => left.ordinal - right.ordinal)
|
|
63
|
+
.map((block) => ({
|
|
64
|
+
...block,
|
|
65
|
+
bytes: copyNativeDurabilityBytes(block.bytes),
|
|
66
|
+
digest: copyNativeDurabilityBytes(block.digest),
|
|
67
|
+
})),
|
|
68
|
+
};
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
const snapshotJournalRequest = (
|
|
72
|
+
request: NativeDurabilityJournalAppendRequest,
|
|
73
|
+
): NativeDurabilityJournalAppendRequest => {
|
|
74
|
+
assertNativeDurabilityJournalAppendRequest(request);
|
|
75
|
+
return {
|
|
76
|
+
...request,
|
|
77
|
+
frames: copyNativeDurabilityBytes(request.frames),
|
|
78
|
+
framesDigest: copyNativeDurabilityBytes(request.framesDigest),
|
|
79
|
+
};
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
const snapshotCheckpointRequest = (
|
|
83
|
+
request: NativeDurabilityCheckpointRequest,
|
|
84
|
+
): NativeDurabilityCheckpointRequest => {
|
|
85
|
+
assertNativeDurabilityCheckpointRequest(request);
|
|
86
|
+
return {
|
|
87
|
+
...request,
|
|
88
|
+
scope: { ...request.scope },
|
|
89
|
+
bytes: copyNativeDurabilityBytes(request.bytes),
|
|
90
|
+
digest: copyNativeDurabilityBytes(request.digest),
|
|
91
|
+
stagingCoverage: request.stagingCoverage.map((coverage) => ({
|
|
92
|
+
...coverage,
|
|
93
|
+
stagingManifestDigest: copyNativeDurabilityBytes(
|
|
94
|
+
coverage.stagingManifestDigest,
|
|
95
|
+
),
|
|
96
|
+
})),
|
|
97
|
+
retainedTransactions: cloneRetainedTransactions(
|
|
98
|
+
request.retainedTransactions,
|
|
99
|
+
),
|
|
100
|
+
};
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
const snapshotDeleteRequest = (
|
|
104
|
+
request: NativeDurabilityDeleteRequest,
|
|
105
|
+
): NativeDurabilityDeleteRequest => {
|
|
106
|
+
assertNativeDurabilityDeleteRequest(request);
|
|
107
|
+
return {
|
|
108
|
+
scope: { ...request.scope },
|
|
109
|
+
targets: request.targets.map((target) => ({ ...target })),
|
|
110
|
+
};
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
export class MemoryNativeDurabilityStorage implements NativeDurabilityStorage {
|
|
114
|
+
readonly version = NATIVE_DURABILITY_STORAGE_VERSION;
|
|
115
|
+
readonly kind = "memory" as const;
|
|
116
|
+
readonly crashSafe = false;
|
|
117
|
+
readonly domainId: string;
|
|
118
|
+
readonly fence: NativeDurabilityLease["fence"];
|
|
119
|
+
|
|
120
|
+
private barrierOrdinal = 0n;
|
|
121
|
+
private strictDeleteCount = 0n;
|
|
122
|
+
private journal = new Uint8Array();
|
|
123
|
+
private readonly staging = new Map<
|
|
124
|
+
string,
|
|
125
|
+
{
|
|
126
|
+
manifest: NativeDurabilityStagingManifest;
|
|
127
|
+
blocks: Map<number, Uint8Array>;
|
|
128
|
+
}
|
|
129
|
+
>();
|
|
130
|
+
private readonly checkpoints = new Map<bigint, NativeDurabilityCheckpoint>();
|
|
131
|
+
private generationHighwater = 0n;
|
|
132
|
+
private operationTail: Promise<void> = Promise.resolve();
|
|
133
|
+
private closing = false;
|
|
134
|
+
private closed = false;
|
|
135
|
+
|
|
136
|
+
constructor(
|
|
137
|
+
private readonly lease: NativeDurabilityLease,
|
|
138
|
+
private readonly journalClassifier: NativeDurabilityJournalClassifier,
|
|
139
|
+
) {
|
|
140
|
+
assertNativeDurabilityFence(lease.fence);
|
|
141
|
+
this.domainId = lease.fence.domainId;
|
|
142
|
+
this.fence = Object.freeze({ ...lease.fence });
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
private barrier(): bigint {
|
|
146
|
+
this.barrierOrdinal++;
|
|
147
|
+
return this.barrierOrdinal;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
private enqueue<T>(operation: () => Promise<T>): Promise<T> {
|
|
151
|
+
if (this.closing || this.closed) {
|
|
152
|
+
return Promise.reject(new NativeDurabilityStorageClosedError());
|
|
153
|
+
}
|
|
154
|
+
let resolveResult!: (value: T | PromiseLike<T>) => void;
|
|
155
|
+
let rejectResult!: (reason?: unknown) => void;
|
|
156
|
+
const result = new Promise<T>((resolve, reject) => {
|
|
157
|
+
resolveResult = resolve;
|
|
158
|
+
rejectResult = reject;
|
|
159
|
+
});
|
|
160
|
+
this.operationTail = this.operationTail.then(async () => {
|
|
161
|
+
try {
|
|
162
|
+
resolveResult(await this.lease.runWhileHeld(operation));
|
|
163
|
+
} catch (error) {
|
|
164
|
+
rejectResult(error);
|
|
165
|
+
}
|
|
166
|
+
});
|
|
167
|
+
return result;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async stageAndSync(
|
|
171
|
+
unsafeRequest: NativeDurabilityStageRequest,
|
|
172
|
+
): Promise<NativeDurabilityStageReceipt> {
|
|
173
|
+
const request = snapshotStageRequest(unsafeRequest);
|
|
174
|
+
return this.enqueue(async () => {
|
|
175
|
+
const references: NativeDurabilityStagedBlockReference[] = [];
|
|
176
|
+
const blocks = new Map<number, Uint8Array>();
|
|
177
|
+
const seenOrdinals = new Set<number>();
|
|
178
|
+
for (const block of request.blocks) {
|
|
179
|
+
if (!Number.isSafeInteger(block.ordinal) || block.ordinal < 0) {
|
|
180
|
+
throw new RangeError(
|
|
181
|
+
"Staging ordinal must be a non-negative safe integer",
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
if (seenOrdinals.has(block.ordinal)) {
|
|
185
|
+
throw new Error(`Duplicate staging ordinal ${block.ordinal}`);
|
|
186
|
+
}
|
|
187
|
+
seenOrdinals.add(block.ordinal);
|
|
188
|
+
const actual = await sha256NativeDurability(block.bytes);
|
|
189
|
+
if (!nativeDurabilityBytesEqual(actual, block.digest)) {
|
|
190
|
+
throw new NativeDurabilityDigestMismatchError(
|
|
191
|
+
`staged block ${block.ordinal}`,
|
|
192
|
+
);
|
|
193
|
+
}
|
|
194
|
+
blocks.set(block.ordinal, copyNativeDurabilityBytes(block.bytes));
|
|
195
|
+
references.push({
|
|
196
|
+
ordinal: block.ordinal,
|
|
197
|
+
cid: block.cid,
|
|
198
|
+
byteLength: block.bytes.byteLength,
|
|
199
|
+
digest: copyNativeDurabilityBytes(block.digest),
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
references.sort((left, right) => left.ordinal - right.ordinal);
|
|
203
|
+
const manifestPayload = {
|
|
204
|
+
version: this.version,
|
|
205
|
+
scope: request.scope,
|
|
206
|
+
fence: this.lease.fence,
|
|
207
|
+
blocks: references,
|
|
208
|
+
};
|
|
209
|
+
const manifestDigest = await nativeDurabilityScopeDigest(manifestPayload);
|
|
210
|
+
const manifest: NativeDurabilityStagingManifest = {
|
|
211
|
+
...manifestPayload,
|
|
212
|
+
scope: cloneScope(request.scope),
|
|
213
|
+
fence: cloneFence(this.lease.fence),
|
|
214
|
+
blocks: references,
|
|
215
|
+
manifestDigest,
|
|
216
|
+
};
|
|
217
|
+
const existing = this.staging.get(request.scope.transactionId);
|
|
218
|
+
if (
|
|
219
|
+
existing &&
|
|
220
|
+
!nativeDurabilityBytesEqual(
|
|
221
|
+
existing.manifest.manifestDigest,
|
|
222
|
+
manifest.manifestDigest,
|
|
223
|
+
)
|
|
224
|
+
) {
|
|
225
|
+
throw new Error(
|
|
226
|
+
`Staging transaction ${request.scope.transactionId} already has a different manifest`,
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
this.staging.set(request.scope.transactionId, { manifest, blocks });
|
|
230
|
+
const receipt: NativeDurabilityStageReceipt = {
|
|
231
|
+
version: this.version,
|
|
232
|
+
kind: "stage",
|
|
233
|
+
domainId: this.domainId,
|
|
234
|
+
fence: cloneFence(this.lease.fence),
|
|
235
|
+
transactionId: request.scope.transactionId,
|
|
236
|
+
txSequence: request.scope.txSequence,
|
|
237
|
+
firstRecordLsn: request.scope.recordLsn,
|
|
238
|
+
lastRecordLsn: request.scope.recordLsn,
|
|
239
|
+
scopeDigest: await nativeDurabilityScopeDigest(request),
|
|
240
|
+
barrierOrdinal: this.barrier(),
|
|
241
|
+
blocks: references.map((block) => ({
|
|
242
|
+
...block,
|
|
243
|
+
digest: copyNativeDurabilityBytes(block.digest),
|
|
244
|
+
})),
|
|
245
|
+
manifestDigest: copyNativeDurabilityBytes(manifestDigest),
|
|
246
|
+
};
|
|
247
|
+
return receipt;
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
async readStagingManifest(
|
|
252
|
+
transactionId: string,
|
|
253
|
+
): Promise<NativeDurabilityStagingManifest | undefined> {
|
|
254
|
+
return this.enqueue(async () => {
|
|
255
|
+
const manifest = this.staging.get(transactionId)?.manifest;
|
|
256
|
+
if (!manifest) return undefined;
|
|
257
|
+
return {
|
|
258
|
+
...manifest,
|
|
259
|
+
scope: cloneScope(manifest.scope),
|
|
260
|
+
fence: cloneFence(manifest.fence),
|
|
261
|
+
blocks: manifest.blocks.map((block) => ({
|
|
262
|
+
...block,
|
|
263
|
+
digest: copyNativeDurabilityBytes(block.digest),
|
|
264
|
+
})),
|
|
265
|
+
manifestDigest: copyNativeDurabilityBytes(manifest.manifestDigest),
|
|
266
|
+
};
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
async readStagedBlock(
|
|
271
|
+
transactionId: string,
|
|
272
|
+
ordinal: number,
|
|
273
|
+
): Promise<Uint8Array | undefined> {
|
|
274
|
+
return this.enqueue(async () => {
|
|
275
|
+
const bytes = this.staging.get(transactionId)?.blocks.get(ordinal);
|
|
276
|
+
return bytes && copyNativeDurabilityBytes(bytes);
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
async listStagingTransactionIds(): Promise<string[]> {
|
|
281
|
+
return this.enqueue(async () => [...this.staging.keys()].sort());
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
async appendJournalAndSync(
|
|
285
|
+
unsafeRequest: NativeDurabilityJournalAppendRequest,
|
|
286
|
+
): Promise<NativeDurabilityJournalReceipt> {
|
|
287
|
+
const request = snapshotJournalRequest(unsafeRequest);
|
|
288
|
+
return this.enqueue(async () => {
|
|
289
|
+
if (request.expectedOffset !== this.journal.byteLength) {
|
|
290
|
+
throw new NativeDurabilityJournalOffsetConflictError(
|
|
291
|
+
request.expectedOffset,
|
|
292
|
+
this.journal.byteLength,
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
const actual = await sha256NativeDurability(request.frames);
|
|
296
|
+
if (!nativeDurabilityBytesEqual(actual, request.framesDigest)) {
|
|
297
|
+
throw new NativeDurabilityDigestMismatchError("journal frames");
|
|
298
|
+
}
|
|
299
|
+
const next = new Uint8Array(
|
|
300
|
+
this.journal.byteLength + request.frames.byteLength,
|
|
301
|
+
);
|
|
302
|
+
next.set(this.journal);
|
|
303
|
+
next.set(request.frames, this.journal.byteLength);
|
|
304
|
+
this.journal = next;
|
|
305
|
+
return {
|
|
306
|
+
version: this.version,
|
|
307
|
+
kind: "journal-append",
|
|
308
|
+
domainId: this.domainId,
|
|
309
|
+
fence: cloneFence(this.lease.fence),
|
|
310
|
+
transactionId: request.transactionId,
|
|
311
|
+
txSequence: request.txSequence,
|
|
312
|
+
firstRecordLsn: request.firstRecordLsn,
|
|
313
|
+
lastRecordLsn: request.lastRecordLsn,
|
|
314
|
+
scopeDigest: await nativeDurabilityScopeDigest(request),
|
|
315
|
+
barrierOrdinal: this.barrier(),
|
|
316
|
+
offset: request.expectedOffset,
|
|
317
|
+
endOffset: this.journal.byteLength,
|
|
318
|
+
framesDigest: copyNativeDurabilityBytes(request.framesDigest),
|
|
319
|
+
};
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
async readJournal(): Promise<Uint8Array> {
|
|
324
|
+
return this.enqueue(async () => copyNativeDurabilityBytes(this.journal));
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
async reconcileIncompleteJournalTailAndSync(
|
|
328
|
+
unsafeRequest: NativeDurabilityIncompleteTailReconciliationRequest,
|
|
329
|
+
): Promise<NativeDurabilityJournalReconciliationReceipt> {
|
|
330
|
+
const request = { ...unsafeRequest };
|
|
331
|
+
try {
|
|
332
|
+
assertNativeDurabilityOperationScope({
|
|
333
|
+
...request,
|
|
334
|
+
recordLsn: 0n,
|
|
335
|
+
});
|
|
336
|
+
} catch (error) {
|
|
337
|
+
return Promise.reject(error);
|
|
338
|
+
}
|
|
339
|
+
if (request.txSequence === 0n) {
|
|
340
|
+
return Promise.reject(
|
|
341
|
+
new TypeError("Invalid incomplete-tail reconciliation request"),
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
return this.enqueue(async () => {
|
|
345
|
+
const observed = copyNativeDurabilityBytes(this.journal);
|
|
346
|
+
const observedDigest = await sha256NativeDurability(observed);
|
|
347
|
+
const classified = await this.journalClassifier.classify(
|
|
348
|
+
copyNativeDurabilityBytes(observed),
|
|
349
|
+
);
|
|
350
|
+
const classification = { ...classified };
|
|
351
|
+
if (
|
|
352
|
+
classification.kind !== "incomplete-tail" ||
|
|
353
|
+
!Number.isSafeInteger(classification.validLength) ||
|
|
354
|
+
classification.validLength < 0 ||
|
|
355
|
+
classification.validLength >= observed.byteLength ||
|
|
356
|
+
typeof classification.lastRecordLsn !== "bigint" ||
|
|
357
|
+
classification.lastRecordLsn < 0n ||
|
|
358
|
+
classification.lastRecordLsn > NATIVE_DURABILITY_MAX_U64 ||
|
|
359
|
+
(classification.reason !== "short-header" &&
|
|
360
|
+
classification.reason !== "short-body" &&
|
|
361
|
+
classification.reason !== "short-trailer")
|
|
362
|
+
) {
|
|
363
|
+
throw new NativeDurabilityIncompleteTailMismatchError(
|
|
364
|
+
"The exact current journal does not end in a structurally incomplete frame",
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
this.journal = observed.slice(0, classification.validLength);
|
|
368
|
+
return {
|
|
369
|
+
version: this.version,
|
|
370
|
+
kind: "journal-tail-reconciliation",
|
|
371
|
+
domainId: this.domainId,
|
|
372
|
+
fence: cloneFence(this.lease.fence),
|
|
373
|
+
transactionId: request.transactionId,
|
|
374
|
+
txSequence: request.txSequence,
|
|
375
|
+
firstRecordLsn: classification.lastRecordLsn,
|
|
376
|
+
lastRecordLsn: classification.lastRecordLsn,
|
|
377
|
+
scopeDigest: await nativeDurabilityScopeDigest({
|
|
378
|
+
...request,
|
|
379
|
+
classification,
|
|
380
|
+
observedDigest,
|
|
381
|
+
}),
|
|
382
|
+
barrierOrdinal: this.barrier(),
|
|
383
|
+
previousLength: observed.byteLength,
|
|
384
|
+
validLength: classification.validLength,
|
|
385
|
+
observedDigest,
|
|
386
|
+
};
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
async writeCheckpointAndSync(
|
|
391
|
+
unsafeRequest: NativeDurabilityCheckpointRequest,
|
|
392
|
+
): Promise<NativeDurabilityCheckpointReceipt> {
|
|
393
|
+
const request = snapshotCheckpointRequest(unsafeRequest);
|
|
394
|
+
return this.enqueue(async () => {
|
|
395
|
+
const actual = await sha256NativeDurability(request.bytes);
|
|
396
|
+
if (!nativeDurabilityBytesEqual(actual, request.digest)) {
|
|
397
|
+
throw new NativeDurabilityDigestMismatchError("checkpoint");
|
|
398
|
+
}
|
|
399
|
+
for (const coverage of request.stagingCoverage) {
|
|
400
|
+
const staged = this.staging.get(coverage.transactionId)?.manifest;
|
|
401
|
+
if (
|
|
402
|
+
!staged ||
|
|
403
|
+
staged.scope.txSequence !== coverage.txSequence ||
|
|
404
|
+
coverage.coveredThroughLsn < staged.scope.recordLsn ||
|
|
405
|
+
!nativeDurabilityBytesEqual(
|
|
406
|
+
staged.manifestDigest,
|
|
407
|
+
coverage.stagingManifestDigest,
|
|
408
|
+
)
|
|
409
|
+
) {
|
|
410
|
+
throw new Error(
|
|
411
|
+
`Checkpoint coverage does not match staging transaction ${coverage.transactionId}`,
|
|
412
|
+
);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
const generation = ++this.generationHighwater;
|
|
416
|
+
const manifestSlot = generation % 2n === 1n ? "a" : "b";
|
|
417
|
+
this.checkpoints.set(generation, {
|
|
418
|
+
version: this.version,
|
|
419
|
+
generation,
|
|
420
|
+
checkpointLsn: request.checkpointLsn,
|
|
421
|
+
txSequenceHighwater: request.txSequenceHighwater,
|
|
422
|
+
bytes: copyNativeDurabilityBytes(request.bytes),
|
|
423
|
+
digest: copyNativeDurabilityBytes(request.digest),
|
|
424
|
+
originFence: cloneFence(this.lease.fence),
|
|
425
|
+
manifestSlot,
|
|
426
|
+
stagingCoverage: request.stagingCoverage.map((coverage) => ({
|
|
427
|
+
...coverage,
|
|
428
|
+
stagingManifestDigest: copyNativeDurabilityBytes(
|
|
429
|
+
coverage.stagingManifestDigest,
|
|
430
|
+
),
|
|
431
|
+
})),
|
|
432
|
+
retainedTransactions: cloneRetainedTransactions(
|
|
433
|
+
request.retainedTransactions,
|
|
434
|
+
),
|
|
435
|
+
});
|
|
436
|
+
return {
|
|
437
|
+
version: this.version,
|
|
438
|
+
kind: "checkpoint",
|
|
439
|
+
domainId: this.domainId,
|
|
440
|
+
fence: cloneFence(this.lease.fence),
|
|
441
|
+
transactionId: request.scope.transactionId,
|
|
442
|
+
txSequence: request.scope.txSequence,
|
|
443
|
+
firstRecordLsn: request.scope.recordLsn,
|
|
444
|
+
lastRecordLsn: request.scope.recordLsn,
|
|
445
|
+
scopeDigest: await nativeDurabilityScopeDigest(request),
|
|
446
|
+
barrierOrdinal: this.barrier(),
|
|
447
|
+
generation,
|
|
448
|
+
checkpointLsn: request.checkpointLsn,
|
|
449
|
+
checkpointDigest: copyNativeDurabilityBytes(request.digest),
|
|
450
|
+
manifestSlot,
|
|
451
|
+
stagingCoverageDigest: await nativeDurabilityScopeDigest(
|
|
452
|
+
request.stagingCoverage,
|
|
453
|
+
),
|
|
454
|
+
};
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
async readLatestCheckpoint(): Promise<
|
|
459
|
+
NativeDurabilityCheckpoint | undefined
|
|
460
|
+
> {
|
|
461
|
+
return this.enqueue(async () => {
|
|
462
|
+
const latest = [...this.checkpoints.keys()].sort((left, right) =>
|
|
463
|
+
left < right ? 1 : left > right ? -1 : 0,
|
|
464
|
+
)[0];
|
|
465
|
+
const checkpoint =
|
|
466
|
+
latest == null ? undefined : this.checkpoints.get(latest);
|
|
467
|
+
return (
|
|
468
|
+
checkpoint && {
|
|
469
|
+
...checkpoint,
|
|
470
|
+
bytes: copyNativeDurabilityBytes(checkpoint.bytes),
|
|
471
|
+
digest: copyNativeDurabilityBytes(checkpoint.digest),
|
|
472
|
+
originFence: cloneFence(checkpoint.originFence),
|
|
473
|
+
stagingCoverage: checkpoint.stagingCoverage.map((coverage) => ({
|
|
474
|
+
...coverage,
|
|
475
|
+
stagingManifestDigest: copyNativeDurabilityBytes(
|
|
476
|
+
coverage.stagingManifestDigest,
|
|
477
|
+
),
|
|
478
|
+
})),
|
|
479
|
+
retainedTransactions: cloneRetainedTransactions(
|
|
480
|
+
checkpoint.retainedTransactions,
|
|
481
|
+
),
|
|
482
|
+
}
|
|
483
|
+
);
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
async deleteAndSync(
|
|
488
|
+
unsafeRequest: NativeDurabilityDeleteRequest,
|
|
489
|
+
): Promise<NativeDurabilityDeleteReceipt> {
|
|
490
|
+
const request = snapshotDeleteRequest(unsafeRequest);
|
|
491
|
+
return this.enqueue(async () => {
|
|
492
|
+
const checkpointGenerations = [...this.checkpoints.keys()].sort(
|
|
493
|
+
(left, right) => (left < right ? 1 : left > right ? -1 : 0),
|
|
494
|
+
);
|
|
495
|
+
const active = checkpointGenerations[0];
|
|
496
|
+
const previous = checkpointGenerations[1];
|
|
497
|
+
const activeCheckpoint =
|
|
498
|
+
active == null ? undefined : this.checkpoints.get(active);
|
|
499
|
+
// Validate every target before applying any of them.
|
|
500
|
+
for (const target of request.targets) {
|
|
501
|
+
if (target.kind === "staging") {
|
|
502
|
+
const staged = this.staging.get(target.transactionId)?.manifest;
|
|
503
|
+
if (staged) {
|
|
504
|
+
const covered = activeCheckpoint?.stagingCoverage.some(
|
|
505
|
+
(coverage) =>
|
|
506
|
+
coverage.transactionId === staged.scope.transactionId &&
|
|
507
|
+
coverage.txSequence === staged.scope.txSequence &&
|
|
508
|
+
coverage.coveredThroughLsn >= staged.scope.recordLsn &&
|
|
509
|
+
nativeDurabilityBytesEqual(
|
|
510
|
+
coverage.stagingManifestDigest,
|
|
511
|
+
staged.manifestDigest,
|
|
512
|
+
),
|
|
513
|
+
);
|
|
514
|
+
if (!covered) {
|
|
515
|
+
throw new Error(
|
|
516
|
+
`Active checkpoint does not cover staging transaction ${target.transactionId}`,
|
|
517
|
+
);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
} else {
|
|
521
|
+
if (target.generation === active || target.generation === previous) {
|
|
522
|
+
throw new Error(
|
|
523
|
+
`Cannot delete active or previous checkpoint generation ${target.generation}`,
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
for (const target of request.targets) {
|
|
529
|
+
if (target.kind === "staging") {
|
|
530
|
+
this.staging.delete(target.transactionId);
|
|
531
|
+
} else {
|
|
532
|
+
this.checkpoints.delete(target.generation);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
this.strictDeleteCount++;
|
|
536
|
+
return {
|
|
537
|
+
version: this.version,
|
|
538
|
+
kind: "delete",
|
|
539
|
+
domainId: this.domainId,
|
|
540
|
+
fence: cloneFence(this.lease.fence),
|
|
541
|
+
transactionId: request.scope.transactionId,
|
|
542
|
+
txSequence: request.scope.txSequence,
|
|
543
|
+
firstRecordLsn: request.scope.recordLsn,
|
|
544
|
+
lastRecordLsn: request.scope.recordLsn,
|
|
545
|
+
scopeDigest: await nativeDurabilityScopeDigest(request),
|
|
546
|
+
barrierOrdinal: this.barrier(),
|
|
547
|
+
targets: request.targets.map((target) => ({ ...target })),
|
|
548
|
+
};
|
|
549
|
+
});
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
async stats(): Promise<NativeDurabilityStorageStats> {
|
|
553
|
+
return this.enqueue(async () => {
|
|
554
|
+
let stagedBlocks = 0;
|
|
555
|
+
let stagedBytes = 0;
|
|
556
|
+
for (const transaction of this.staging.values()) {
|
|
557
|
+
stagedBlocks += transaction.blocks.size;
|
|
558
|
+
for (const block of transaction.blocks.values()) {
|
|
559
|
+
stagedBytes += block.byteLength;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
let checkpointBytes = 0;
|
|
563
|
+
for (const checkpoint of this.checkpoints.values()) {
|
|
564
|
+
checkpointBytes += checkpoint.bytes.byteLength;
|
|
565
|
+
}
|
|
566
|
+
return {
|
|
567
|
+
kind: this.kind,
|
|
568
|
+
domainId: this.domainId,
|
|
569
|
+
strictBarrierCount: this.barrierOrdinal,
|
|
570
|
+
strictDeleteCount: this.strictDeleteCount,
|
|
571
|
+
journalBytes: this.journal.byteLength,
|
|
572
|
+
stagingTransactions: this.staging.size,
|
|
573
|
+
stagedBlocks,
|
|
574
|
+
stagedBytes,
|
|
575
|
+
checkpointGenerations: this.checkpoints.size,
|
|
576
|
+
checkpointBytes,
|
|
577
|
+
};
|
|
578
|
+
});
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
async close(): Promise<void> {
|
|
582
|
+
if (this.closed) return;
|
|
583
|
+
this.closing = true;
|
|
584
|
+
await this.operationTail;
|
|
585
|
+
this.closed = true;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
export const createMemoryNativeDurabilityStorage = (
|
|
590
|
+
lease: NativeDurabilityLease,
|
|
591
|
+
journalClassifier: NativeDurabilityJournalClassifier,
|
|
592
|
+
): MemoryNativeDurabilityStorage =>
|
|
593
|
+
new MemoryNativeDurabilityStorage(lease, journalClassifier);
|