@peerbit/shared-log 13.2.5 → 13.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/errors.d.ts +19 -0
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +40 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/index.d.ts +72 -6
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3251 -541
- package/dist/src/index.js.map +1 -1
- package/package.json +23 -23
- package/src/errors.ts +62 -0
- package/src/index.ts +4431 -879
package/dist/src/index.js
CHANGED
|
@@ -40,7 +40,7 @@ import { AccessError, Ed25519Keypair, Ed25519PublicKey, PublicSignKey, Secp256k1
|
|
|
40
40
|
import { And, ByteMatchQuery, NotStartedError as IndexNotStartedError, Or, Sort, StringMatch, toId, } from "@peerbit/indexer-interface";
|
|
41
41
|
import { Entry, EntryType, LamportClock, Log, Meta, ShallowEntry, ShallowMeta, Timestamp, verifyEd25519Batch, verifyEntryV0Ed25519BatchFromEntries, } from "@peerbit/log";
|
|
42
42
|
import { logger as loggerFn } from "@peerbit/logger";
|
|
43
|
-
import { ClosedError, Program } from "@peerbit/program";
|
|
43
|
+
import { ClosedError, Program, TerminalOperationNotStartedError, } from "@peerbit/program";
|
|
44
44
|
import { FanoutChannel, waitForSubscribers, } from "@peerbit/pubsub";
|
|
45
45
|
import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
|
|
46
46
|
import { RPC } from "@peerbit/rpc";
|
|
@@ -53,8 +53,8 @@ import { BlocksMessage } from "./blocks.js";
|
|
|
53
53
|
import { CheckedPruneCoordinator, } from "./checked-prune.js";
|
|
54
54
|
import { CPUUsageIntervalLag } from "./cpu.js";
|
|
55
55
|
import { debouncedAccumulatorMap, } from "./debounce.js";
|
|
56
|
-
import { NoPeersError } from "./errors.js";
|
|
57
|
-
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, ResponseIPrune, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas,
|
|
56
|
+
import { NativeDurableCommitError, NoPeersError } from "./errors.js";
|
|
57
|
+
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, ResponseIPrune, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
|
|
58
58
|
import { FanoutEnvelope } from "./fanout-envelope.js";
|
|
59
59
|
import { MAX_U32, MAX_U64, createNumbers, } from "./integers.js";
|
|
60
60
|
import { TransportMessage } from "./message.js";
|
|
@@ -105,9 +105,9 @@ const toLocalPublicSignKey = (key) => {
|
|
|
105
105
|
export { createReplicationDomainHash, createReplicationDomainTime, };
|
|
106
106
|
export { CPUUsageIntervalLag };
|
|
107
107
|
export * from "./replication.js";
|
|
108
|
-
export { ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, EntryReplicatedU32, EntryReplicatedU64, NoPeersError, };
|
|
108
|
+
export { ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, EntryReplicatedU32, EntryReplicatedU64, NativeDurableCommitError, NoPeersError, };
|
|
109
109
|
export { MAX_U32, MAX_U64 };
|
|
110
|
-
export { ExchangeHeadsMessage, RawExchangeHeadsMessage };
|
|
110
|
+
export { ExchangeHeadsMessage, RawExchangeHeadsMessage, StashBackedRawExchangeHeadsMessage, };
|
|
111
111
|
export const logger = loggerFn("peerbit:shared-log");
|
|
112
112
|
const warn = logger.newScope("warn");
|
|
113
113
|
const traceLogger = logger.trace;
|
|
@@ -129,6 +129,21 @@ const canUseOptionalNativeModuleImports = () => {
|
|
|
129
129
|
* to avoid doubled slashes.
|
|
130
130
|
*/
|
|
131
131
|
const joinNativeCoordinateDirectory = (nodeDirectory, fsSafeLogId) => `${nodeDirectory.replace(/[/\\]+$/, "")}/coordinates/${fsSafeLogId}`;
|
|
132
|
+
const createNativeDurableBlockStore = async (storage) => new AnyBlockStore(await storage.sublevel("blocks", {
|
|
133
|
+
// Strict mirrors remain WAL-backed across close. The available snapshot
|
|
134
|
+
// rewrite is not a crash-atomic generation protocol, so thresholds must
|
|
135
|
+
// not re-enable it behind this acknowledgement boundary.
|
|
136
|
+
compactOnClose: false,
|
|
137
|
+
// A native append is acknowledged only after this mirror resolves. The
|
|
138
|
+
// Rust store's normal immutable fast path may resolve before its WAL write;
|
|
139
|
+
// strict mode waits for the journal write and sync, closing the SIGKILL gap.
|
|
140
|
+
durability: "strict",
|
|
141
|
+
}));
|
|
142
|
+
const createDefaultDurableBlockStore = async (storage) => new AnyBlockStore(await storage.sublevel("blocks", {
|
|
143
|
+
// State this default explicitly so a cached child created by the native
|
|
144
|
+
// path cannot silently carry its deferred-close policy into this path.
|
|
145
|
+
compactOnClose: true,
|
|
146
|
+
}));
|
|
132
147
|
/**
|
|
133
148
|
* Write-through block store bridging the native backbone's in-wasm-memory block
|
|
134
149
|
* store to a durable per-program {@link AnyBlockStore}.
|
|
@@ -148,11 +163,12 @@ const joinNativeCoordinateDirectory = (nodeDirectory, fsSafeLogId) => `${nodeDir
|
|
|
148
163
|
* METHOD SURFACE (see #1006): `RemoteBlocks` and the log feature-detect the
|
|
149
164
|
* optional batch methods (`putMany`/`putKnown`/`putKnownMany`/
|
|
150
165
|
* `putKnownManyColumns`/`rmMany`). To keep the receive-fusion / columnar fast
|
|
151
|
-
* paths engaged this wrapper
|
|
152
|
-
*
|
|
153
|
-
* — and delegates each. It
|
|
154
|
-
*
|
|
155
|
-
*
|
|
166
|
+
* paths engaged this wrapper preserves the native store's optional write
|
|
167
|
+
* methods — including `putKnownManyColumns`, which `AnyBlockStore` does not have
|
|
168
|
+
* — and delegates each. It adds only local durability/trim coordination hooks
|
|
169
|
+
* consumed by `RemoteBlocks` and the log; protocol/native-handle capabilities
|
|
170
|
+
* such as `getBlockResponsePayload`/`getNativeLogBlockStoreHandle` stay absent,
|
|
171
|
+
* so their optional-chained probes keep the existing fallback behavior.
|
|
156
172
|
*/
|
|
157
173
|
class NativeBackboneWriteThroughBlockStore {
|
|
158
174
|
native;
|
|
@@ -170,8 +186,302 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
170
186
|
// vanishing and leaving the block out of durable while native/log report
|
|
171
187
|
// success. `.catch` also prevents unhandled-rejection noise.
|
|
172
188
|
pendingDurableWrites = new Set();
|
|
173
|
-
|
|
174
|
-
|
|
189
|
+
nativeDurableCommitFailure;
|
|
190
|
+
stopCompleted = false;
|
|
191
|
+
nativeDeleteTombstones = new Map();
|
|
192
|
+
nativeDeleteEpoch = 0;
|
|
193
|
+
nativeBlockWriteGenerations = new Map();
|
|
194
|
+
pendingNativeDeleteCleanup = new Map();
|
|
195
|
+
stagedNativeDeleteCleanups = new Map();
|
|
196
|
+
nextNativeDeleteCleanupToken = 0;
|
|
197
|
+
nativeDeleteCleanupRunning;
|
|
198
|
+
nextNativeCommitOwnershipToken = 0;
|
|
199
|
+
nativeCommitOwnerships = new Map();
|
|
200
|
+
nativeCommitOwnershipsByCid = new Map();
|
|
201
|
+
getNativeDurableCommitFailure() {
|
|
202
|
+
return this.nativeDurableCommitFailure;
|
|
203
|
+
}
|
|
204
|
+
recordNativeDurableCommitFailure(cause, options) {
|
|
205
|
+
if (this.nativeDurableCommitFailure) {
|
|
206
|
+
this.nativeDurableCommitFailure.addCommitContext(options);
|
|
207
|
+
return this.nativeDurableCommitFailure;
|
|
208
|
+
}
|
|
209
|
+
this.nativeDurableCommitFailure =
|
|
210
|
+
cause instanceof NativeDurableCommitError
|
|
211
|
+
? cause
|
|
212
|
+
: new NativeDurableCommitError(cause, options);
|
|
213
|
+
if (cause instanceof NativeDurableCommitError) {
|
|
214
|
+
cause.addCommitContext(options);
|
|
215
|
+
}
|
|
216
|
+
return this.nativeDurableCommitFailure;
|
|
217
|
+
}
|
|
218
|
+
throwIfNativeDurableCommitFailed() {
|
|
219
|
+
if (this.nativeDurableCommitFailure) {
|
|
220
|
+
throw this.nativeDurableCommitFailure;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
throwIfDurableWritesFailed() {
|
|
224
|
+
this.throwIfNativeDurableCommitFailed();
|
|
225
|
+
}
|
|
226
|
+
async commitDurableMutation(operation, committedCids, failedCids) {
|
|
227
|
+
this.throwIfNativeDurableCommitFailed();
|
|
228
|
+
const committedCidList = [...committedCids];
|
|
229
|
+
const failedCidList = failedCids ? [...failedCids] : committedCidList;
|
|
230
|
+
let result;
|
|
231
|
+
const operationResult = Promise.resolve().then(operation);
|
|
232
|
+
this.trackAwaitedDurable(operationResult);
|
|
233
|
+
try {
|
|
234
|
+
result = await operationResult;
|
|
235
|
+
}
|
|
236
|
+
catch (error) {
|
|
237
|
+
throw this.recordNativeDurableCommitFailure(error, {
|
|
238
|
+
committedCids: committedCidList,
|
|
239
|
+
failedCids: failedCidList,
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
// A different concurrent native mutation may have poisoned the wrapper
|
|
243
|
+
// while this durable call was in flight. Include this operation among the
|
|
244
|
+
// native-applied facts, but not among durable calls that actually failed.
|
|
245
|
+
if (this.nativeDurableCommitFailure) {
|
|
246
|
+
this.nativeDurableCommitFailure.addCommitContext({
|
|
247
|
+
committedCids: committedCidList,
|
|
248
|
+
failedCids: [],
|
|
249
|
+
});
|
|
250
|
+
throw this.nativeDurableCommitFailure;
|
|
251
|
+
}
|
|
252
|
+
return result;
|
|
253
|
+
}
|
|
254
|
+
beginNativeDelete(cids) {
|
|
255
|
+
this.nativeDeleteEpoch++;
|
|
256
|
+
for (const cid of cids) {
|
|
257
|
+
this.nativeDeleteTombstones.set(cid, (this.nativeDeleteTombstones.get(cid) ?? 0) + 1);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
endNativeDelete(cids) {
|
|
261
|
+
for (const cid of cids) {
|
|
262
|
+
const remaining = (this.nativeDeleteTombstones.get(cid) ?? 1) - 1;
|
|
263
|
+
if (remaining <= 0) {
|
|
264
|
+
this.nativeDeleteTombstones.delete(cid);
|
|
265
|
+
}
|
|
266
|
+
else {
|
|
267
|
+
this.nativeDeleteTombstones.set(cid, remaining);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
isNativeDeletePending(cid) {
|
|
272
|
+
return this.nativeDeleteTombstones.has(cid);
|
|
273
|
+
}
|
|
274
|
+
// A CID can be legitimately re-added after a native trim (content addressing
|
|
275
|
+
// makes the bytes identical, but its liveness is new). Cancel any queued trim
|
|
276
|
+
// for that CID and advance its generation before the write is exposed. An
|
|
277
|
+
// already-running cleanup uses the generation/pending map to avoid deleting
|
|
278
|
+
// the new native value; synchronous columnar writes also chain their durable
|
|
279
|
+
// mirror behind that cleanup below.
|
|
280
|
+
noteNativeBlockWrite(cids) {
|
|
281
|
+
const generations = new Map();
|
|
282
|
+
for (const cid of new Set(cids)) {
|
|
283
|
+
const generation = (this.nativeBlockWriteGenerations.get(cid) ?? 0) + 1;
|
|
284
|
+
this.nativeBlockWriteGenerations.set(cid, generation);
|
|
285
|
+
generations.set(cid, generation);
|
|
286
|
+
if (this.pendingNativeDeleteCleanup.delete(cid)) {
|
|
287
|
+
this.endNativeDelete([cid]);
|
|
288
|
+
}
|
|
289
|
+
for (const staged of this.stagedNativeDeleteCleanups.values()) {
|
|
290
|
+
if (staged.delete(cid)) {
|
|
291
|
+
this.endNativeDelete([cid]);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
return generations;
|
|
296
|
+
}
|
|
297
|
+
beginNativeCommitOwnership(generations) {
|
|
298
|
+
if (generations.size === 0) {
|
|
299
|
+
return undefined;
|
|
300
|
+
}
|
|
301
|
+
const token = {
|
|
302
|
+
id: ++this.nextNativeCommitOwnershipToken,
|
|
303
|
+
rows: new Map(),
|
|
304
|
+
};
|
|
305
|
+
for (const [cid, generation] of generations) {
|
|
306
|
+
const owners = this.nativeCommitOwnershipsByCid.get(cid) ?? new Set();
|
|
307
|
+
const shared = owners.size > 0;
|
|
308
|
+
for (const ownerId of owners) {
|
|
309
|
+
const owner = this.nativeCommitOwnerships.get(ownerId);
|
|
310
|
+
const row = owner?.rows.get(cid);
|
|
311
|
+
if (row)
|
|
312
|
+
row.shared = true;
|
|
313
|
+
}
|
|
314
|
+
owners.add(token.id);
|
|
315
|
+
this.nativeCommitOwnershipsByCid.set(cid, owners);
|
|
316
|
+
token.rows.set(cid, { generation, shared });
|
|
317
|
+
}
|
|
318
|
+
this.nativeCommitOwnerships.set(token.id, token);
|
|
319
|
+
return token;
|
|
320
|
+
}
|
|
321
|
+
releaseNativeCommitOwnership(token) {
|
|
322
|
+
if (!token ||
|
|
323
|
+
typeof token !== "object" ||
|
|
324
|
+
typeof token.id !== "number") {
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
const owned = this.nativeCommitOwnerships.get(token.id);
|
|
328
|
+
if (owned !== token) {
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
this.nativeCommitOwnerships.delete(owned.id);
|
|
332
|
+
for (const cid of owned.rows.keys()) {
|
|
333
|
+
const owners = this.nativeCommitOwnershipsByCid.get(cid);
|
|
334
|
+
owners?.delete(owned.id);
|
|
335
|
+
if (owners?.size === 0) {
|
|
336
|
+
this.nativeCommitOwnershipsByCid.delete(cid);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
acknowledgeNativeCommitOwnership(token) {
|
|
341
|
+
this.releaseNativeCommitOwnership(token);
|
|
342
|
+
}
|
|
343
|
+
enqueueNativeDeleteCleanup(cids) {
|
|
344
|
+
for (const cid of new Set(cids)) {
|
|
345
|
+
if (!this.pendingNativeDeleteCleanup.has(cid)) {
|
|
346
|
+
this.pendingNativeDeleteCleanup.set(cid, this.nativeBlockWriteGenerations.get(cid) ?? 0);
|
|
347
|
+
this.beginNativeDelete([cid]);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
releaseStagedNativeDeleteCleanup(token) {
|
|
352
|
+
const staged = this.stagedNativeDeleteCleanups.get(token);
|
|
353
|
+
if (!staged) {
|
|
354
|
+
return false;
|
|
355
|
+
}
|
|
356
|
+
this.stagedNativeDeleteCleanups.delete(token);
|
|
357
|
+
for (const [cid] of staged) {
|
|
358
|
+
this.endNativeDelete([cid]);
|
|
359
|
+
}
|
|
360
|
+
return true;
|
|
361
|
+
}
|
|
362
|
+
discardStagedNativeDeleteCleanups() {
|
|
363
|
+
for (const token of [...this.stagedNativeDeleteCleanups.keys()]) {
|
|
364
|
+
this.releaseStagedNativeDeleteCleanup(token);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
// Native commit callbacks call this immediately after the native transaction
|
|
368
|
+
// returns, before awaiting the new block's durable mirror. This stages read
|
|
369
|
+
// tombstones only. Durable deletion is promoted later by the exact EntryIndex
|
|
370
|
+
// consume token, so an unacknowledged/failed append cannot delete the old
|
|
371
|
+
// durable head that the lower log still publishes.
|
|
372
|
+
beginNativeDeleteCleanup(cids) {
|
|
373
|
+
this.throwIfNativeDurableCommitFailed();
|
|
374
|
+
const uniqueCids = [...new Set(cids)];
|
|
375
|
+
if (uniqueCids.length === 0) {
|
|
376
|
+
return undefined;
|
|
377
|
+
}
|
|
378
|
+
const token = ++this.nextNativeDeleteCleanupToken;
|
|
379
|
+
const staged = new Map(uniqueCids.map((cid) => [
|
|
380
|
+
cid,
|
|
381
|
+
this.nativeBlockWriteGenerations.get(cid) ?? 0,
|
|
382
|
+
]));
|
|
383
|
+
this.stagedNativeDeleteCleanups.set(token, staged);
|
|
384
|
+
this.beginNativeDelete(uniqueCids);
|
|
385
|
+
return token;
|
|
386
|
+
}
|
|
387
|
+
cancelNativeDeleteCleanup(cleanupToken) {
|
|
388
|
+
if (typeof cleanupToken === "number") {
|
|
389
|
+
this.releaseStagedNativeDeleteCleanup(cleanupToken);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
async waitForNativeDeleteCleanup() {
|
|
393
|
+
while (this.nativeDeleteCleanupRunning) {
|
|
394
|
+
await this.nativeDeleteCleanupRunning;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
async waitForTrackedDurableWrites() {
|
|
398
|
+
while (this.pendingDurableWrites.size > 0) {
|
|
399
|
+
await Promise.allSettled([...this.pendingDurableWrites]);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
async retryNativeDeleteCleanup(options) {
|
|
403
|
+
await this.waitForNativeDeleteCleanup();
|
|
404
|
+
if (this.nativeDurableCommitFailure && !options?.allowPoisoned) {
|
|
405
|
+
throw this.nativeDurableCommitFailure;
|
|
406
|
+
}
|
|
407
|
+
// A synchronous columnar write may already have scheduled its durable
|
|
408
|
+
// mirror. Let it settle before deleting queued CIDs, but leave any recorded
|
|
409
|
+
// error for drainDurable() to surface to its owning operation/stop.
|
|
410
|
+
await this.waitForTrackedDurableWrites();
|
|
411
|
+
await this.waitForNativeDeleteCleanup();
|
|
412
|
+
// A tracked mirror or the cleanup we just waited for may have poisoned the
|
|
413
|
+
// wrapper. Do not begin another durable mutation on the ordinary path after
|
|
414
|
+
// that asynchronous boundary. stop() alone opts into one cleanup retry so a
|
|
415
|
+
// transient delete failure can still release its tombstones and resources.
|
|
416
|
+
if (this.nativeDurableCommitFailure && !options?.allowPoisoned) {
|
|
417
|
+
throw this.nativeDurableCommitFailure;
|
|
418
|
+
}
|
|
419
|
+
if (this.pendingNativeDeleteCleanup.size === 0) {
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
const cleanupEntries = [...this.pendingNativeDeleteCleanup].filter(([cid, generation]) => (this.nativeBlockWriteGenerations.get(cid) ?? 0) === generation);
|
|
423
|
+
if (cleanupEntries.length === 0) {
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
const cids = cleanupEntries.map(([cid]) => cid);
|
|
427
|
+
let cleanupFailure;
|
|
428
|
+
const running = (async () => {
|
|
429
|
+
let durableRemoved = false;
|
|
430
|
+
let nativeRemoved = false;
|
|
431
|
+
try {
|
|
432
|
+
await this.durable.rmMany(cids);
|
|
433
|
+
durableRemoved = true;
|
|
434
|
+
}
|
|
435
|
+
catch (error) {
|
|
436
|
+
cleanupFailure = error;
|
|
437
|
+
// The new entry blocks are already durable at this point and the native
|
|
438
|
+
// transaction has selected the new graph/index state. Old content-addressed
|
|
439
|
+
// blocks are therefore harmless unreachable orphans. Keep this cleanup as
|
|
440
|
+
// retryable debt instead of poisoning/rolling back a fully durable append;
|
|
441
|
+
// a partial rmMany is safe for the same reason.
|
|
442
|
+
warn(`Failed durable native-trim cleanup; retaining retry debt: ${String(error)}`);
|
|
443
|
+
}
|
|
444
|
+
finally {
|
|
445
|
+
// A read that began in the native-transaction -> cleanup-hook gap may
|
|
446
|
+
// have repopulated native. Always repeat the native removal, even when
|
|
447
|
+
// durable rm failed. Exclude CIDs re-added while durable IO was pending;
|
|
448
|
+
// their generation change cancelled the queued delete.
|
|
449
|
+
const stillDeleted = cids.filter((cid) => this.pendingNativeDeleteCleanup.has(cid));
|
|
450
|
+
try {
|
|
451
|
+
if (stillDeleted.length > 0) {
|
|
452
|
+
await this.native.rmMany(stillDeleted);
|
|
453
|
+
}
|
|
454
|
+
nativeRemoved = true;
|
|
455
|
+
}
|
|
456
|
+
catch (error) {
|
|
457
|
+
cleanupFailure ??= error;
|
|
458
|
+
warn(`Failed to repeat native-trim hot block removal: ${String(error)}`);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
if (durableRemoved && nativeRemoved) {
|
|
462
|
+
for (const [cid, generation] of cleanupEntries) {
|
|
463
|
+
if (this.pendingNativeDeleteCleanup.get(cid) === generation) {
|
|
464
|
+
this.pendingNativeDeleteCleanup.delete(cid);
|
|
465
|
+
this.nativeBlockWriteGenerations.delete(cid);
|
|
466
|
+
this.endNativeDelete([cid]);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
})();
|
|
471
|
+
this.nativeDeleteCleanupRunning = running;
|
|
472
|
+
try {
|
|
473
|
+
await running;
|
|
474
|
+
}
|
|
475
|
+
finally {
|
|
476
|
+
if (this.nativeDeleteCleanupRunning === running) {
|
|
477
|
+
this.nativeDeleteCleanupRunning = undefined;
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
if (options?.throwOnFailure && cleanupFailure !== undefined) {
|
|
481
|
+
throw cleanupFailure;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
trackDurable(result, cids) {
|
|
175
485
|
// The durable store may answer synchronously (an in-memory or already
|
|
176
486
|
// resolved store); only a real pending promise needs tracking. A sync
|
|
177
487
|
// success is already durable; a sync throw would have propagated already.
|
|
@@ -182,9 +492,22 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
182
492
|
this.pendingDurableWrites.delete(tracked);
|
|
183
493
|
}, (error) => {
|
|
184
494
|
this.pendingDurableWrites.delete(tracked);
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
495
|
+
this.recordNativeDurableCommitFailure(error, {
|
|
496
|
+
committedCids: cids,
|
|
497
|
+
failedCids: cids,
|
|
498
|
+
});
|
|
499
|
+
});
|
|
500
|
+
this.pendingDurableWrites.add(tracked);
|
|
501
|
+
}
|
|
502
|
+
// Awaited mirror writes surface their own rejection to the append that
|
|
503
|
+
// created them. Track a non-rejecting settlement companion as well so stop()
|
|
504
|
+
// and later wrapper operations cannot close/use durable storage while that
|
|
505
|
+
// append barrier is still in flight.
|
|
506
|
+
trackAwaitedDurable(result) {
|
|
507
|
+
const tracked = result.then(() => {
|
|
508
|
+
this.pendingDurableWrites.delete(tracked);
|
|
509
|
+
}, () => {
|
|
510
|
+
this.pendingDurableWrites.delete(tracked);
|
|
188
511
|
});
|
|
189
512
|
this.pendingDurableWrites.add(tracked);
|
|
190
513
|
}
|
|
@@ -192,14 +515,8 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
192
515
|
// the first failure. Awaited methods call this so a prior columnar durable
|
|
193
516
|
// failure propagates as back-pressure to the next caller.
|
|
194
517
|
async drainDurable() {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
}
|
|
198
|
-
if (this.durableWriteError !== undefined) {
|
|
199
|
-
const error = this.durableWriteError;
|
|
200
|
-
this.durableWriteError = undefined;
|
|
201
|
-
throw error;
|
|
202
|
-
}
|
|
518
|
+
await this.waitForTrackedDurableWrites();
|
|
519
|
+
this.throwIfNativeDurableCommitFailed();
|
|
203
520
|
}
|
|
204
521
|
// --- lifecycle -------------------------------------------------------
|
|
205
522
|
// The native store's lifecycle hooks are no-ops; only the durable store
|
|
@@ -213,12 +530,55 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
213
530
|
// persisted coordinate + signer facts without resolving the entry block.
|
|
214
531
|
async start() {
|
|
215
532
|
await this.durable.start();
|
|
533
|
+
this.stopCompleted = false;
|
|
216
534
|
}
|
|
217
535
|
async stop() {
|
|
536
|
+
if (this.stopCompleted) {
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
218
539
|
// Surface any tracked (sync-path) durable write failure and ensure all
|
|
219
|
-
// mirror writes have settled before the durable store is torn down.
|
|
220
|
-
|
|
221
|
-
|
|
540
|
+
// mirror writes have settled before the durable store is torn down. Closing
|
|
541
|
+
// the durable store is unconditional: a prior columnar mirror failure must
|
|
542
|
+
// not leak the store lifecycle resource.
|
|
543
|
+
let firstError;
|
|
544
|
+
let shutdownError;
|
|
545
|
+
try {
|
|
546
|
+
await this.drainDurable();
|
|
547
|
+
}
|
|
548
|
+
catch (error) {
|
|
549
|
+
firstError = error;
|
|
550
|
+
}
|
|
551
|
+
// Tokens not consumed by EntryIndex belong to native prepares that never
|
|
552
|
+
// published their lower-log trim. Release their read tombstones, but never
|
|
553
|
+
// promote them to durable deletion during shutdown.
|
|
554
|
+
this.discardStagedNativeDeleteCleanups();
|
|
555
|
+
try {
|
|
556
|
+
await this.retryNativeDeleteCleanup({ allowPoisoned: true });
|
|
557
|
+
}
|
|
558
|
+
catch (error) {
|
|
559
|
+
firstError ??= error;
|
|
560
|
+
shutdownError ??= error;
|
|
561
|
+
}
|
|
562
|
+
try {
|
|
563
|
+
await this.durable.stop();
|
|
564
|
+
}
|
|
565
|
+
catch (error) {
|
|
566
|
+
firstError ??= error;
|
|
567
|
+
shutdownError ??= error;
|
|
568
|
+
}
|
|
569
|
+
if (shutdownError === undefined) {
|
|
570
|
+
// A durable poison belongs to the generation being closed. Report it once
|
|
571
|
+
// so the owning terminal call observes the failed append, but remember that
|
|
572
|
+
// every mandatory shutdown stage completed. Conservative content-addressed
|
|
573
|
+
// trim debt may remain in this retired wrapper after best-effort cleanup; a
|
|
574
|
+
// fresh generation gets a new wrapper and must not be wedged by that debt.
|
|
575
|
+
// The exact terminal retry may therefore finish parent bookkeeping without
|
|
576
|
+
// rethrowing the same latched poison forever.
|
|
577
|
+
this.stopCompleted = true;
|
|
578
|
+
}
|
|
579
|
+
if (firstError !== undefined) {
|
|
580
|
+
throw firstError;
|
|
581
|
+
}
|
|
222
582
|
}
|
|
223
583
|
status() {
|
|
224
584
|
return this.durable.status();
|
|
@@ -226,6 +586,12 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
226
586
|
waitFor() {
|
|
227
587
|
return Promise.resolve([]);
|
|
228
588
|
}
|
|
589
|
+
// Native commit APIs must keep their block-store callback synchronous. The
|
|
590
|
+
// lower log calls this barrier after that callback reports committed blocks
|
|
591
|
+
// and before publishing index/head facts.
|
|
592
|
+
waitForDurableWrites() {
|
|
593
|
+
return this.drainDurable();
|
|
594
|
+
}
|
|
229
595
|
// Mirror a single already-committed block (present in the wasm map) to the
|
|
230
596
|
// durable store ONLY. Used by the native commit-only append fast path: the
|
|
231
597
|
// native prepare commits the entry block into the wasm map and returns no raw
|
|
@@ -233,11 +599,157 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
233
599
|
// route the block through the log's finishBlocks/putKnown* (that would disturb
|
|
234
600
|
// the commit-only append path the RCS optimization depends on). Instead the
|
|
235
601
|
// caller reads the committed bytes back and calls this so the block lands in
|
|
236
|
-
// durable directly.
|
|
237
|
-
//
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
this.
|
|
602
|
+
// durable directly. The caller awaits this method before acknowledging its
|
|
603
|
+
// append, so a failed write rejects the append that produced the block.
|
|
604
|
+
async mirrorToDurable(cid, bytes, options) {
|
|
605
|
+
this.throwIfNativeDurableCommitFailed();
|
|
606
|
+
// The native commit happened before this call. Mark the CID live before
|
|
607
|
+
// awaiting anything so a concurrent retry of an older trim cannot remove
|
|
608
|
+
// the newly committed hot block.
|
|
609
|
+
let ownership;
|
|
610
|
+
if (options?.nativeTrimmed !== true) {
|
|
611
|
+
ownership = this.beginNativeCommitOwnership(this.noteNativeBlockWrite([cid]));
|
|
612
|
+
await this.waitForNativeDeleteCleanup();
|
|
613
|
+
}
|
|
614
|
+
try {
|
|
615
|
+
await this.drainDurable();
|
|
616
|
+
if (ownership) {
|
|
617
|
+
const existed = await this.durable.hasMany([...ownership.rows.keys()]);
|
|
618
|
+
let index = 0;
|
|
619
|
+
for (const row of ownership.rows.values()) {
|
|
620
|
+
row.durableExistedBefore = existed[index++] === true;
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
const result = this.commitDurableMutation(() => this.durable.putKnown(cid, bytes), [cid]);
|
|
624
|
+
await result;
|
|
625
|
+
await this.retryNativeDeleteCleanup();
|
|
626
|
+
this.throwIfNativeDurableCommitFailed();
|
|
627
|
+
return ownership;
|
|
628
|
+
}
|
|
629
|
+
catch (error) {
|
|
630
|
+
// The caller never receives an ownership token for an indeterminate write.
|
|
631
|
+
// Release the in-memory claim and preserve any bytes the backend may have
|
|
632
|
+
// applied before rejecting.
|
|
633
|
+
this.releaseNativeCommitOwnership(ownership);
|
|
634
|
+
throw error;
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
async mirrorManyToDurable(blocks, options) {
|
|
638
|
+
this.throwIfNativeDurableCommitFailed();
|
|
639
|
+
const cids = blocks.map(([cid]) => cid);
|
|
640
|
+
// Rows trimmed later in the same native batch deliberately remain owned by
|
|
641
|
+
// that batch's staged cleanup; only mark surviving rows live.
|
|
642
|
+
const liveCids = cids.filter((cid) => !options?.nativeTrimmedCids?.has(cid));
|
|
643
|
+
const ownership = this.beginNativeCommitOwnership(this.noteNativeBlockWrite(liveCids));
|
|
644
|
+
if (liveCids.length > 0) {
|
|
645
|
+
await this.waitForNativeDeleteCleanup();
|
|
646
|
+
}
|
|
647
|
+
try {
|
|
648
|
+
await this.drainDurable();
|
|
649
|
+
if (ownership) {
|
|
650
|
+
const existed = await this.durable.hasMany([...ownership.rows.keys()]);
|
|
651
|
+
let index = 0;
|
|
652
|
+
for (const row of ownership.rows.values()) {
|
|
653
|
+
row.durableExistedBefore = existed[index++] === true;
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
if (blocks.length > 0) {
|
|
657
|
+
await this.commitDurableMutation(() => this.durable.putKnownMany(blocks), cids);
|
|
658
|
+
}
|
|
659
|
+
await this.retryNativeDeleteCleanup();
|
|
660
|
+
this.throwIfNativeDurableCommitFailed();
|
|
661
|
+
return ownership;
|
|
662
|
+
}
|
|
663
|
+
catch (error) {
|
|
664
|
+
this.releaseNativeCommitOwnership(ownership);
|
|
665
|
+
throw error;
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
async rollbackFailedNativeCommits(cids, restoreNativeCids = [], ownershipToken) {
|
|
669
|
+
// This is the sole mutation allowed after poison: it removes a native
|
|
670
|
+
// transaction that the lower log never published. Do not route through the
|
|
671
|
+
// guarded rmMany path, and settle the failed mirror before compensating it.
|
|
672
|
+
await this.waitForTrackedDurableWrites();
|
|
673
|
+
// A failed replacement never published its trim. Reassert the restored CIDs
|
|
674
|
+
// as live before any compensation IO so staged/pending delete intents cannot
|
|
675
|
+
// remove the last acknowledged blocks during stop or reopen.
|
|
676
|
+
const ownership = ownershipToken && typeof ownershipToken === "object"
|
|
677
|
+
? this.nativeCommitOwnerships.get(ownershipToken.id)
|
|
678
|
+
: undefined;
|
|
679
|
+
const verifiedOwnership = ownership && ownership === ownershipToken ? ownership : undefined;
|
|
680
|
+
const restoreSet = new Set(restoreNativeCids);
|
|
681
|
+
const safeDurableDeletes = verifiedOwnership
|
|
682
|
+
? [...new Set(cids)].filter((cid) => {
|
|
683
|
+
const row = verifiedOwnership.rows.get(cid);
|
|
684
|
+
const owners = this.nativeCommitOwnershipsByCid.get(cid);
|
|
685
|
+
return (!restoreSet.has(cid) &&
|
|
686
|
+
row?.durableExistedBefore === false &&
|
|
687
|
+
row.shared === false &&
|
|
688
|
+
(this.nativeBlockWriteGenerations.get(cid) ?? 0) ===
|
|
689
|
+
row.generation &&
|
|
690
|
+
owners?.size === 1 &&
|
|
691
|
+
owners.has(verifiedOwnership.id));
|
|
692
|
+
})
|
|
693
|
+
: [];
|
|
694
|
+
this.noteNativeBlockWrite(restoreNativeCids);
|
|
695
|
+
let firstError;
|
|
696
|
+
try {
|
|
697
|
+
if (safeDurableDeletes.length > 0) {
|
|
698
|
+
await this.durable.rmMany(safeDurableDeletes);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
catch (error) {
|
|
702
|
+
firstError = error;
|
|
703
|
+
}
|
|
704
|
+
// A native prepare runs before ownership can observe the hot map, so it cannot
|
|
705
|
+
// prove that a CID was absent there before this operation. Keep native bytes as
|
|
706
|
+
// unreachable orphans rather than deleting acknowledged/shared/restored data.
|
|
707
|
+
if (restoreNativeCids.length > 0) {
|
|
708
|
+
try {
|
|
709
|
+
const values = await this.durable.getMany(restoreNativeCids);
|
|
710
|
+
const restore = [];
|
|
711
|
+
for (let index = 0; index < restoreNativeCids.length; index++) {
|
|
712
|
+
const value = values[index];
|
|
713
|
+
if (value)
|
|
714
|
+
restore.push([restoreNativeCids[index], value]);
|
|
715
|
+
}
|
|
716
|
+
if (restore.length > 0) {
|
|
717
|
+
this.native.putKnownMany(restore);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
catch (error) {
|
|
721
|
+
firstError ??= error;
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
this.releaseNativeCommitOwnership(ownershipToken);
|
|
725
|
+
if (firstError !== undefined)
|
|
726
|
+
throw firstError;
|
|
727
|
+
}
|
|
728
|
+
/**
|
|
729
|
+
* Compensate a native prepare that failed before its durable mirror began.
|
|
730
|
+
* Durable presence proves a same-CID acknowledged owner; an active ownership
|
|
731
|
+
* token proves a concurrent mirror. Only an unowned, non-durable hot block is
|
|
732
|
+
* exclusively attributable to the failed prepare and safe to remove.
|
|
733
|
+
*/
|
|
734
|
+
async rollbackUnmirroredNativeCommits(cids, restoreNativeCids = []) {
|
|
735
|
+
const unique = [...new Set(cids)];
|
|
736
|
+
// Native prepares bypass this wrapper. Any observed wrapper generation is
|
|
737
|
+
// therefore evidence of a generic/same-CID writer, not of the failed prepare.
|
|
738
|
+
// Snapshot before the first await and require both absence and stability so a
|
|
739
|
+
// write starting before or during durable.hasMany cannot lose its hot value to
|
|
740
|
+
// a stale `false` result.
|
|
741
|
+
const genericWriteGenerations = new Map(unique.map((cid) => [cid, this.nativeBlockWriteGenerations.get(cid)]));
|
|
742
|
+
await this.rollbackFailedNativeCommits(cids, restoreNativeCids);
|
|
743
|
+
const durablePresence = await this.durable.hasMany(unique);
|
|
744
|
+
const restore = new Set(restoreNativeCids);
|
|
745
|
+
const safeNativeDeletes = unique.filter((cid, index) => !restore.has(cid) &&
|
|
746
|
+
durablePresence[index] !== true &&
|
|
747
|
+
genericWriteGenerations.get(cid) === undefined &&
|
|
748
|
+
this.nativeBlockWriteGenerations.get(cid) === undefined &&
|
|
749
|
+
(this.nativeCommitOwnershipsByCid.get(cid)?.size ?? 0) === 0);
|
|
750
|
+
if (safeNativeDeletes.length > 0) {
|
|
751
|
+
await this.native.rmMany(safeNativeDeletes);
|
|
752
|
+
}
|
|
241
753
|
}
|
|
242
754
|
// --- writes (apply to BOTH: native first for the hot path, then durable) ---
|
|
243
755
|
async put(data) {
|
|
@@ -250,7 +762,13 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
250
762
|
const value = data instanceof Uint8Array
|
|
251
763
|
? data
|
|
252
764
|
: data.block.bytes;
|
|
253
|
-
|
|
765
|
+
this.noteNativeBlockWrite([cid]);
|
|
766
|
+
await this.waitForNativeDeleteCleanup();
|
|
767
|
+
// put() may have yielded while calculating the CID. Restore the hot value
|
|
768
|
+
// after any older cleanup that was already in flight.
|
|
769
|
+
this.throwIfNativeDurableCommitFailed();
|
|
770
|
+
this.native.putKnown(cid, value);
|
|
771
|
+
await this.commitDurableMutation(() => this.durable.putKnown(cid, value), [cid]);
|
|
254
772
|
return cid;
|
|
255
773
|
}
|
|
256
774
|
async putMany(blocks) {
|
|
@@ -263,7 +781,11 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
263
781
|
: block.block.bytes;
|
|
264
782
|
return [cid, value];
|
|
265
783
|
});
|
|
266
|
-
|
|
784
|
+
this.noteNativeBlockWrite(cids);
|
|
785
|
+
await this.waitForNativeDeleteCleanup();
|
|
786
|
+
this.throwIfNativeDurableCommitFailed();
|
|
787
|
+
this.native.putKnownMany(durableBlocks);
|
|
788
|
+
await this.commitDurableMutation(() => this.durable.putKnownMany(durableBlocks), cids);
|
|
267
789
|
return cids;
|
|
268
790
|
}
|
|
269
791
|
// Native put is synchronous (the authoritative hot store); the durable mirror
|
|
@@ -273,18 +795,30 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
273
795
|
// both await this method, so returning a promise is compatible.
|
|
274
796
|
async putKnown(cid, bytes) {
|
|
275
797
|
await this.drainDurable();
|
|
798
|
+
await this.waitForNativeDeleteCleanup();
|
|
799
|
+
this.throwIfNativeDurableCommitFailed();
|
|
276
800
|
const stored = this.native.putKnown(cid, bytes);
|
|
277
|
-
|
|
801
|
+
this.noteNativeBlockWrite([cid]);
|
|
802
|
+
await this.commitDurableMutation(() => this.durable.putKnown(cid, bytes), [cid]);
|
|
278
803
|
return stored;
|
|
279
804
|
}
|
|
280
805
|
async putKnownMany(blocks) {
|
|
281
806
|
await this.drainDurable();
|
|
807
|
+
await this.waitForNativeDeleteCleanup();
|
|
808
|
+
this.throwIfNativeDurableCommitFailed();
|
|
282
809
|
const cids = this.native.putKnownMany(blocks);
|
|
283
|
-
|
|
810
|
+
this.noteNativeBlockWrite(cids);
|
|
811
|
+
await this.commitDurableMutation(() => this.durable.putKnownMany(blocks), cids);
|
|
284
812
|
return cids;
|
|
285
813
|
}
|
|
286
814
|
putKnownManyColumns(cids, bytes) {
|
|
815
|
+
this.throwIfNativeDurableCommitFailed();
|
|
816
|
+
if (cids.length !== bytes.length) {
|
|
817
|
+
throw new Error("Expected equal block column lengths");
|
|
818
|
+
}
|
|
819
|
+
const cleanupBarrier = this.nativeDeleteCleanupRunning;
|
|
287
820
|
const stored = this.native.putKnownManyColumns(cids, bytes);
|
|
821
|
+
this.noteNativeBlockWrite(cids);
|
|
288
822
|
// AnyBlockStore has no columnar method; mirror via putKnownMany, which
|
|
289
823
|
// takes [cid, bytes] tuples and hits the same batched store path.
|
|
290
824
|
// This method must return a synchronous string[] (RemoteBlocks.putKnownManyColumns
|
|
@@ -292,7 +826,19 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
292
826
|
// inline. Track it instead of `void`ing it so a durable rejection surfaces
|
|
293
827
|
// on the next awaited wrapper method / stop() rather than being swallowed.
|
|
294
828
|
const durableBlocks = cids.map((cid, index) => [cid, bytes[index]]);
|
|
295
|
-
|
|
829
|
+
let durableResult;
|
|
830
|
+
try {
|
|
831
|
+
durableResult = cleanupBarrier
|
|
832
|
+
? cleanupBarrier.then(() => this.durable.putKnownMany(durableBlocks))
|
|
833
|
+
: this.durable.putKnownMany(durableBlocks);
|
|
834
|
+
}
|
|
835
|
+
catch (error) {
|
|
836
|
+
throw this.recordNativeDurableCommitFailure(error, {
|
|
837
|
+
committedCids: cids,
|
|
838
|
+
failedCids: cids,
|
|
839
|
+
});
|
|
840
|
+
}
|
|
841
|
+
this.trackDurable(durableResult, cids);
|
|
296
842
|
return stored;
|
|
297
843
|
}
|
|
298
844
|
// --- reads (native/wasm first; on miss, durable fallback + repopulate) ---
|
|
@@ -302,11 +848,28 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
302
848
|
// avoids a spurious miss (which would otherwise fall through to a remote
|
|
303
849
|
// read) for a block that is present on disk.
|
|
304
850
|
async get(cid, _options) {
|
|
851
|
+
if (this.nativeDurableCommitFailure) {
|
|
852
|
+
// After poison, durable storage is the last acknowledged authority. Do not
|
|
853
|
+
// repopulate or otherwise mutate the native store until reopen.
|
|
854
|
+
return this.isNativeDeletePending(cid)
|
|
855
|
+
? undefined
|
|
856
|
+
: this.durable.get(cid);
|
|
857
|
+
}
|
|
858
|
+
const deleteEpoch = this.nativeDeleteEpoch;
|
|
859
|
+
if (this.isNativeDeletePending(cid)) {
|
|
860
|
+
return undefined;
|
|
861
|
+
}
|
|
305
862
|
const local = this.native.get(cid);
|
|
306
863
|
if (local != null) {
|
|
307
|
-
return local;
|
|
864
|
+
return deleteEpoch === this.nativeDeleteEpoch ? local : this.get(cid);
|
|
308
865
|
}
|
|
309
866
|
const durableValue = await this.durable.get(cid);
|
|
867
|
+
if (this.nativeDurableCommitFailure) {
|
|
868
|
+
return this.isNativeDeletePending(cid) ? undefined : durableValue;
|
|
869
|
+
}
|
|
870
|
+
if (deleteEpoch !== this.nativeDeleteEpoch) {
|
|
871
|
+
return this.get(cid);
|
|
872
|
+
}
|
|
310
873
|
if (durableValue != null) {
|
|
311
874
|
// Repopulate the native map so the native graph reads it next time.
|
|
312
875
|
this.native.putKnownManyColumns([cid], [durableValue]);
|
|
@@ -315,25 +878,52 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
315
878
|
return undefined;
|
|
316
879
|
}
|
|
317
880
|
async getMany(cids) {
|
|
881
|
+
if (this.nativeDurableCommitFailure) {
|
|
882
|
+
const values = await this.durable.getMany(cids);
|
|
883
|
+
for (let index = 0; index < cids.length; index++) {
|
|
884
|
+
if (this.isNativeDeletePending(cids[index])) {
|
|
885
|
+
values[index] = undefined;
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
return values;
|
|
889
|
+
}
|
|
890
|
+
const deleteEpoch = this.nativeDeleteEpoch;
|
|
318
891
|
const results = await this.native.getMany(cids);
|
|
892
|
+
if (deleteEpoch !== this.nativeDeleteEpoch) {
|
|
893
|
+
return this.getMany(cids);
|
|
894
|
+
}
|
|
319
895
|
const missing = [];
|
|
896
|
+
const missingIndexes = [];
|
|
320
897
|
for (let i = 0; i < results.length; i++) {
|
|
321
|
-
if (
|
|
898
|
+
if (this.isNativeDeletePending(cids[i])) {
|
|
899
|
+
results[i] = undefined;
|
|
900
|
+
}
|
|
901
|
+
else if (results[i] == null) {
|
|
322
902
|
missing.push(cids[i]);
|
|
903
|
+
missingIndexes.push(i);
|
|
323
904
|
}
|
|
324
905
|
}
|
|
325
906
|
if (missing.length === 0) {
|
|
326
907
|
return results;
|
|
327
908
|
}
|
|
328
909
|
const durableValues = await this.durable.getMany(missing);
|
|
910
|
+
if (this.nativeDurableCommitFailure) {
|
|
911
|
+
const values = await this.durable.getMany(cids);
|
|
912
|
+
for (let index = 0; index < cids.length; index++) {
|
|
913
|
+
if (this.isNativeDeletePending(cids[index])) {
|
|
914
|
+
values[index] = undefined;
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
return values;
|
|
918
|
+
}
|
|
919
|
+
if (deleteEpoch !== this.nativeDeleteEpoch) {
|
|
920
|
+
return this.getMany(cids);
|
|
921
|
+
}
|
|
329
922
|
const repopulateCids = [];
|
|
330
923
|
const repopulateBytes = [];
|
|
331
|
-
let missingIndex = 0;
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
continue;
|
|
335
|
-
}
|
|
336
|
-
const value = durableValues[missingIndex++];
|
|
924
|
+
for (let missingIndex = 0; missingIndex < missingIndexes.length; missingIndex++) {
|
|
925
|
+
const i = missingIndexes[missingIndex];
|
|
926
|
+
const value = durableValues[missingIndex];
|
|
337
927
|
if (value != null) {
|
|
338
928
|
results[i] = value;
|
|
339
929
|
repopulateCids.push(cids[i]);
|
|
@@ -347,8 +937,15 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
347
937
|
return results;
|
|
348
938
|
}
|
|
349
939
|
async has(cid) {
|
|
940
|
+
if (this.nativeDurableCommitFailure) {
|
|
941
|
+
return this.isNativeDeletePending(cid) ? false : this.durable.has(cid);
|
|
942
|
+
}
|
|
943
|
+
const deleteEpoch = this.nativeDeleteEpoch;
|
|
944
|
+
if (this.isNativeDeletePending(cid)) {
|
|
945
|
+
return false;
|
|
946
|
+
}
|
|
350
947
|
if (this.native.has(cid)) {
|
|
351
|
-
return true;
|
|
948
|
+
return deleteEpoch === this.nativeDeleteEpoch ? true : this.has(cid);
|
|
352
949
|
}
|
|
353
950
|
// Mirror getMany/hasMany: a block absent from the native wasm map may still
|
|
354
951
|
// be present in the durable store (e.g. persisted on disk but not yet
|
|
@@ -356,25 +953,44 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
356
953
|
// checks agree with the resolves that getMany/hasMany already durable-fall
|
|
357
954
|
// back on. `Blocks.has` is declared `MaybePromise<boolean>`, so returning a
|
|
358
955
|
// promise here is contract-compatible.
|
|
359
|
-
|
|
956
|
+
const durableHas = await this.durable.has(cid);
|
|
957
|
+
return deleteEpoch === this.nativeDeleteEpoch ? durableHas : this.has(cid);
|
|
360
958
|
}
|
|
361
959
|
async hasMany(cids) {
|
|
960
|
+
if (this.nativeDurableCommitFailure) {
|
|
961
|
+
const values = await this.durable.hasMany(cids);
|
|
962
|
+
for (let index = 0; index < cids.length; index++) {
|
|
963
|
+
if (this.isNativeDeletePending(cids[index])) {
|
|
964
|
+
values[index] = false;
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
return values;
|
|
968
|
+
}
|
|
969
|
+
const deleteEpoch = this.nativeDeleteEpoch;
|
|
362
970
|
const nativeHas = await this.native.hasMany(cids);
|
|
971
|
+
if (deleteEpoch !== this.nativeDeleteEpoch) {
|
|
972
|
+
return this.hasMany(cids);
|
|
973
|
+
}
|
|
363
974
|
const missing = [];
|
|
975
|
+
const missingIndexes = [];
|
|
364
976
|
for (let i = 0; i < nativeHas.length; i++) {
|
|
365
|
-
if (
|
|
977
|
+
if (this.isNativeDeletePending(cids[i])) {
|
|
978
|
+
nativeHas[i] = false;
|
|
979
|
+
}
|
|
980
|
+
else if (!nativeHas[i]) {
|
|
366
981
|
missing.push(cids[i]);
|
|
982
|
+
missingIndexes.push(i);
|
|
367
983
|
}
|
|
368
984
|
}
|
|
369
985
|
if (missing.length === 0) {
|
|
370
986
|
return nativeHas;
|
|
371
987
|
}
|
|
372
988
|
const durableHas = await this.durable.hasMany(missing);
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
989
|
+
if (deleteEpoch !== this.nativeDeleteEpoch) {
|
|
990
|
+
return this.hasMany(cids);
|
|
991
|
+
}
|
|
992
|
+
for (let missingIndex = 0; missingIndex < missingIndexes.length; missingIndex++) {
|
|
993
|
+
nativeHas[missingIndexes[missingIndex]] = durableHas[missingIndex];
|
|
378
994
|
}
|
|
379
995
|
return nativeHas;
|
|
380
996
|
}
|
|
@@ -383,25 +999,128 @@ class NativeBackboneWriteThroughBlockStore {
|
|
|
383
999
|
// resolves only after both succeed and a durable failure rejects here rather
|
|
384
1000
|
// than being swallowed. All rm callers (RemoteBlocks.rm, the log) await it.
|
|
385
1001
|
async rm(cid) {
|
|
386
|
-
|
|
387
|
-
this.
|
|
388
|
-
|
|
1002
|
+
this.throwIfNativeDurableCommitFailed();
|
|
1003
|
+
const writeGeneration = this.nativeBlockWriteGenerations.get(cid);
|
|
1004
|
+
this.beginNativeDelete([cid]);
|
|
1005
|
+
try {
|
|
1006
|
+
await this.drainDurable();
|
|
1007
|
+
this.native.rm(cid);
|
|
1008
|
+
await this.commitDurableMutation(() => this.durable.rm(cid), [cid]);
|
|
1009
|
+
// A durable read that began before the tombstone may have repopulated
|
|
1010
|
+
// native while durable rm was pending. Remove it idempotently again.
|
|
1011
|
+
this.native.rm(cid);
|
|
1012
|
+
if (this.nativeBlockWriteGenerations.get(cid) === writeGeneration &&
|
|
1013
|
+
!this.pendingNativeDeleteCleanup.has(cid)) {
|
|
1014
|
+
this.nativeBlockWriteGenerations.delete(cid);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
finally {
|
|
1018
|
+
this.endNativeDelete([cid]);
|
|
1019
|
+
}
|
|
389
1020
|
}
|
|
390
1021
|
del(cid) {
|
|
391
|
-
|
|
1022
|
+
return this.rm(cid);
|
|
392
1023
|
}
|
|
393
1024
|
async rmMany(cids) {
|
|
394
|
-
|
|
395
|
-
const
|
|
396
|
-
|
|
397
|
-
|
|
1025
|
+
this.throwIfNativeDurableCommitFailed();
|
|
1026
|
+
const writeGenerations = new Map(cids.map((cid) => [cid, this.nativeBlockWriteGenerations.get(cid)]));
|
|
1027
|
+
this.beginNativeDelete(cids);
|
|
1028
|
+
try {
|
|
1029
|
+
await this.drainDurable();
|
|
1030
|
+
const removed = await this.native.rmMany(cids);
|
|
1031
|
+
await this.commitDurableMutation(() => this.durable.rmMany(cids), cids);
|
|
1032
|
+
await this.native.rmMany(cids);
|
|
1033
|
+
for (const cid of cids) {
|
|
1034
|
+
if (this.nativeBlockWriteGenerations.get(cid) ===
|
|
1035
|
+
writeGenerations.get(cid) &&
|
|
1036
|
+
!this.pendingNativeDeleteCleanup.has(cid)) {
|
|
1037
|
+
this.nativeBlockWriteGenerations.delete(cid);
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
return removed;
|
|
1041
|
+
}
|
|
1042
|
+
finally {
|
|
1043
|
+
this.endNativeDelete(cids);
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
// Native trim may already have removed the hot wasm blocks. Queue the durable
|
|
1047
|
+
// copy for cleanup, retaining read tombstones until removal succeeds; a
|
|
1048
|
+
// cleanup failure is retried and never fed into ordinary append rollback.
|
|
1049
|
+
// EntryIndex feature-detects this hook.
|
|
1050
|
+
async rmManyAfterNativeDelete(cids, cleanupToken) {
|
|
1051
|
+
if (this.nativeDurableCommitFailure) {
|
|
1052
|
+
this.cancelNativeDeleteCleanup(cleanupToken);
|
|
1053
|
+
throw this.nativeDurableCommitFailure;
|
|
1054
|
+
}
|
|
1055
|
+
let preannounced = false;
|
|
1056
|
+
if (typeof cleanupToken === "number") {
|
|
1057
|
+
const staged = this.stagedNativeDeleteCleanups.get(cleanupToken);
|
|
1058
|
+
if (staged) {
|
|
1059
|
+
preannounced = true;
|
|
1060
|
+
this.stagedNativeDeleteCleanups.delete(cleanupToken);
|
|
1061
|
+
for (const [cid, generation] of staged) {
|
|
1062
|
+
if ((this.nativeBlockWriteGenerations.get(cid) ?? 0) !== generation) {
|
|
1063
|
+
this.endNativeDelete([cid]);
|
|
1064
|
+
continue;
|
|
1065
|
+
}
|
|
1066
|
+
if (this.pendingNativeDeleteCleanup.has(cid)) {
|
|
1067
|
+
// Another delete already owns a tombstone for this CID.
|
|
1068
|
+
this.endNativeDelete([cid]);
|
|
1069
|
+
}
|
|
1070
|
+
else {
|
|
1071
|
+
// Transfer the staged tombstone to the now-published cleanup.
|
|
1072
|
+
this.pendingNativeDeleteCleanup.set(cid, generation);
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
if (!preannounced) {
|
|
1078
|
+
this.enqueueNativeDeleteCleanup(cids);
|
|
1079
|
+
}
|
|
1080
|
+
await this.retryNativeDeleteCleanup();
|
|
1081
|
+
}
|
|
1082
|
+
/** Finish committed trim GC before its durable recovery intent is retired. */
|
|
1083
|
+
async completeCommittedNativeDeleteCleanup(cids, options) {
|
|
1084
|
+
this.throwIfNativeDurableCommitFailed();
|
|
1085
|
+
const uniqueCids = [...new Set(cids.filter(Boolean))];
|
|
1086
|
+
if (uniqueCids.length === 0) {
|
|
1087
|
+
return;
|
|
1088
|
+
}
|
|
1089
|
+
// Only restart recovery reconstructs missing debt. On the live path, an
|
|
1090
|
+
// absent row may mean the CID was legitimately re-added after its original
|
|
1091
|
+
// generation-owned trim completed or was cancelled; re-enqueueing it here
|
|
1092
|
+
// would capture the new generation and delete live content.
|
|
1093
|
+
if (options?.reconstructMissing) {
|
|
1094
|
+
this.enqueueNativeDeleteCleanup(uniqueCids);
|
|
1095
|
+
}
|
|
1096
|
+
if (!uniqueCids.some((cid) => this.pendingNativeDeleteCleanup.has(cid))) {
|
|
1097
|
+
return;
|
|
1098
|
+
}
|
|
1099
|
+
await this.retryNativeDeleteCleanup({ throwOnFailure: true });
|
|
1100
|
+
const remaining = uniqueCids.filter((cid) => this.pendingNativeDeleteCleanup.has(cid));
|
|
1101
|
+
if (remaining.length > 0) {
|
|
1102
|
+
throw new Error(`Committed native trim cleanup remains incomplete: ${remaining.join(", ")}`);
|
|
1103
|
+
}
|
|
398
1104
|
}
|
|
399
1105
|
// --- misc ------------------------------------------------------------
|
|
400
1106
|
async *iterator() {
|
|
1107
|
+
if (this.nativeDurableCommitFailure) {
|
|
1108
|
+
for await (const block of this.durable.iterator()) {
|
|
1109
|
+
if (!this.isNativeDeletePending(block[0])) {
|
|
1110
|
+
yield block;
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
return;
|
|
1114
|
+
}
|
|
401
1115
|
yield* this.native.iterator();
|
|
402
1116
|
}
|
|
403
|
-
size() {
|
|
404
|
-
|
|
1117
|
+
async size() {
|
|
1118
|
+
// The hot wasm map is intentionally cold after reopen, so it cannot be the
|
|
1119
|
+
// storage-budget authority. Settle synchronous columnar mirrors first, then
|
|
1120
|
+
// report durable bytes; pending trim cleanup remains conservatively counted
|
|
1121
|
+
// until its durable deletion succeeds.
|
|
1122
|
+
await this.drainDurable();
|
|
1123
|
+
return this.durable.size();
|
|
405
1124
|
}
|
|
406
1125
|
persisted() {
|
|
407
1126
|
// The blocks are now mirrored to a durable store, so report persisted so
|
|
@@ -473,6 +1192,29 @@ const canConfirmNativeRequestPruneBatch = (hints, hashCount) => {
|
|
|
473
1192
|
}
|
|
474
1193
|
return true;
|
|
475
1194
|
};
|
|
1195
|
+
const NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILE = "strict-durable-transaction-intent.json";
|
|
1196
|
+
const NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_BACKUP_FILE = "strict-durable-transaction-intent.backup.json";
|
|
1197
|
+
const NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILES = [
|
|
1198
|
+
NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILE,
|
|
1199
|
+
NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_BACKUP_FILE,
|
|
1200
|
+
];
|
|
1201
|
+
const NATIVE_STRICT_DURABLE_TRANSACTION_JOURNAL_FORMAT = "peerbit-native-strict-durable-transaction";
|
|
1202
|
+
const nativeStrictDurableTransactionJournalBody = (sequence, intent) => ({
|
|
1203
|
+
format: NATIVE_STRICT_DURABLE_TRANSACTION_JOURNAL_FORMAT,
|
|
1204
|
+
version: 1,
|
|
1205
|
+
sequence,
|
|
1206
|
+
state: intent ? "intent" : "cleared",
|
|
1207
|
+
intent: intent ?? null,
|
|
1208
|
+
});
|
|
1209
|
+
const nativeStrictDurableTransactionJournalBodyBytes = (body) => new TextEncoder().encode(JSON.stringify(body));
|
|
1210
|
+
const nativeStrictDurableTransactionJournalRecordBytes = (sequence, intent) => {
|
|
1211
|
+
const body = nativeStrictDurableTransactionJournalBody(sequence, intent);
|
|
1212
|
+
const record = {
|
|
1213
|
+
...body,
|
|
1214
|
+
checksum: toHexString(sha256Sync(nativeStrictDurableTransactionJournalBodyBytes(body))),
|
|
1215
|
+
};
|
|
1216
|
+
return new TextEncoder().encode(JSON.stringify(record));
|
|
1217
|
+
};
|
|
476
1218
|
const getLatestEntry = (entries) => {
|
|
477
1219
|
let latest = undefined;
|
|
478
1220
|
for (const element of entries) {
|
|
@@ -602,6 +1344,43 @@ const isNotStartedError = (e) => {
|
|
|
602
1344
|
}
|
|
603
1345
|
return false;
|
|
604
1346
|
};
|
|
1347
|
+
/**
|
|
1348
|
+
* Replication announcements are best-effort convergence messages. A detached
|
|
1349
|
+
* fanout shard can time out even though the shared log itself remains open.
|
|
1350
|
+
* Keep retries deliberately limited to concrete TimeoutErrors: abort/close and
|
|
1351
|
+
* unexpected programming/data errors must retain their existing semantics.
|
|
1352
|
+
*
|
|
1353
|
+
* Exact constructor/name checks complement `instanceof` for errors crossing
|
|
1354
|
+
* worker or duplicate-package boundaries in browsers.
|
|
1355
|
+
*/
|
|
1356
|
+
const isTransientReplicationAnnouncementError = (error, seen = new Set()) => {
|
|
1357
|
+
if (error != null &&
|
|
1358
|
+
(typeof error === "object" || typeof error === "function")) {
|
|
1359
|
+
if (seen.has(error)) {
|
|
1360
|
+
return false;
|
|
1361
|
+
}
|
|
1362
|
+
seen.add(error);
|
|
1363
|
+
}
|
|
1364
|
+
if (error instanceof TimeoutError) {
|
|
1365
|
+
return true;
|
|
1366
|
+
}
|
|
1367
|
+
const nested = error?.errors;
|
|
1368
|
+
if (Array.isArray(nested) && nested.length > 0) {
|
|
1369
|
+
return nested.every((item) => isTransientReplicationAnnouncementError(item, new Set(seen)));
|
|
1370
|
+
}
|
|
1371
|
+
const cause = error?.cause;
|
|
1372
|
+
if (cause != null && isTransientReplicationAnnouncementError(cause, seen)) {
|
|
1373
|
+
return true;
|
|
1374
|
+
}
|
|
1375
|
+
const constructorName = typeof error?.constructor
|
|
1376
|
+
?.name === "string"
|
|
1377
|
+
? error.constructor.name
|
|
1378
|
+
: "";
|
|
1379
|
+
const name = typeof error?.name === "string"
|
|
1380
|
+
? error.name
|
|
1381
|
+
: "";
|
|
1382
|
+
return constructorName === "TimeoutError" || name === "TimeoutError";
|
|
1383
|
+
};
|
|
605
1384
|
const EMPTY_HASHES = [];
|
|
606
1385
|
const normalizedHashValues = (hashes) => {
|
|
607
1386
|
if (Array.isArray(hashes)) {
|
|
@@ -698,6 +1477,7 @@ const CHECKED_PRUNE_RETRY_MAX_ATTEMPTS = 3;
|
|
|
698
1477
|
const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
|
|
699
1478
|
// DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
|
|
700
1479
|
const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
|
|
1480
|
+
const REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL = 1000;
|
|
701
1481
|
const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
|
|
702
1482
|
const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
|
|
703
1483
|
const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
|
|
@@ -876,11 +1656,23 @@ let SharedLog = (() => {
|
|
|
876
1656
|
_nativeRangePlanner;
|
|
877
1657
|
_nativeSharedLogState;
|
|
878
1658
|
_nativeBackbone;
|
|
1659
|
+
_nativeDurableCommitFailure;
|
|
1660
|
+
_nativeDurableRecoveryReadyForReopen = false;
|
|
1661
|
+
_nativeDurableRecoveryCids = new Set();
|
|
879
1662
|
_wireSyncSession;
|
|
880
1663
|
_nativeBackboneCoordinatePersistence;
|
|
1664
|
+
_nativeBackboneCoordinatePersistenceStore;
|
|
1665
|
+
_nativeBackboneDropStarted = false;
|
|
881
1666
|
_nativeBackboneCoordinateJournalLastFlushMs = 0;
|
|
1667
|
+
_nativeStrictDurableTransactionTail;
|
|
1668
|
+
_nativeStrictDurableTransactions;
|
|
1669
|
+
_nativeStrictDurableTransactionJournalState;
|
|
1670
|
+
_nativeStrictDurableDocumentRecoveryDeferred = false;
|
|
1671
|
+
_nativeStrictDurableTransactionsClosing = false;
|
|
1672
|
+
_nativeStrictDurableTransactionFailure;
|
|
882
1673
|
_defaultAppendReplicaMetadataCache;
|
|
883
1674
|
_residentEntryCoordinatesByHash;
|
|
1675
|
+
_nativeCoordinateMutationGenerations;
|
|
884
1676
|
coordinateToHash;
|
|
885
1677
|
recentlyRebalanced;
|
|
886
1678
|
uniqueReplicators;
|
|
@@ -923,77 +1715,816 @@ let SharedLog = (() => {
|
|
|
923
1715
|
_replicatorLivenessFailures;
|
|
924
1716
|
_replicatorLastActivityAt;
|
|
925
1717
|
remoteBlocks;
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
1718
|
+
throwIfNativeDurableCommitFailed() {
|
|
1719
|
+
if (this._nativeStrictDurableTransactionFailure !== undefined) {
|
|
1720
|
+
throw new Error("Native durable transaction recovery is required before another mutation", { cause: this._nativeStrictDurableTransactionFailure });
|
|
1721
|
+
}
|
|
1722
|
+
const wrapperFailure = this.remoteBlocks?.localStore?.getNativeDurableCommitFailure?.();
|
|
1723
|
+
if (wrapperFailure) {
|
|
1724
|
+
this._nativeDurableCommitFailure ??= wrapperFailure;
|
|
1725
|
+
}
|
|
1726
|
+
if (this._nativeDurableCommitFailure) {
|
|
1727
|
+
throw this._nativeDurableCommitFailure;
|
|
1728
|
+
}
|
|
936
1729
|
}
|
|
937
|
-
|
|
938
|
-
|
|
1730
|
+
poisonNativeStrictDurableTransaction(cause) {
|
|
1731
|
+
this._nativeStrictDurableTransactionFailure ??= cause;
|
|
1732
|
+
this.log?.entryIndex?.poisonNativeDurableTransactionMutations(this._nativeStrictDurableTransactionFailure);
|
|
939
1733
|
}
|
|
940
|
-
|
|
941
|
-
|
|
1734
|
+
clearNativeStrictDurableTransactionFailure() {
|
|
1735
|
+
this._nativeStrictDurableTransactionFailure = undefined;
|
|
1736
|
+
this.log?.entryIndex?.clearNativeDurableTransactionMutationFailure();
|
|
942
1737
|
}
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
1738
|
+
failNativeDurableCommit(cause, options) {
|
|
1739
|
+
this.ensureNativeDurabilityRuntimeState();
|
|
1740
|
+
for (const cid of options?.committedCids ?? []) {
|
|
1741
|
+
this._nativeDurableRecoveryCids.add(cid);
|
|
1742
|
+
}
|
|
1743
|
+
if (cause instanceof NativeDurableCommitError) {
|
|
1744
|
+
cause.addCommitContext(options, { preferIncomingOrder: true });
|
|
1745
|
+
}
|
|
1746
|
+
this._nativeDurableCommitFailure ??=
|
|
1747
|
+
cause instanceof NativeDurableCommitError
|
|
1748
|
+
? cause
|
|
1749
|
+
: new NativeDurableCommitError(cause, options);
|
|
1750
|
+
this._nativeDurableCommitFailure.addCommitContext(options, {
|
|
1751
|
+
preferIncomingOrder: true,
|
|
1752
|
+
});
|
|
1753
|
+
throw this._nativeDurableCommitFailure;
|
|
1754
|
+
}
|
|
1755
|
+
snapshotNativeBackboneDocument(input) {
|
|
1756
|
+
const backbone = this._nativeBackbone;
|
|
1757
|
+
if (!backbone || !input)
|
|
1758
|
+
return undefined;
|
|
1759
|
+
const value = backbone.documentValueBytes(input.key);
|
|
1760
|
+
return {
|
|
1761
|
+
key: input.key,
|
|
1762
|
+
value: value ? new Uint8Array(value) : undefined,
|
|
1763
|
+
byteElementIndexLimit: input.byteElementIndexLimit ?? 0,
|
|
1764
|
+
};
|
|
1765
|
+
}
|
|
1766
|
+
restoreNativeBackboneDocument(rollback) {
|
|
1767
|
+
const backbone = this._nativeBackbone;
|
|
1768
|
+
if (!backbone)
|
|
1769
|
+
return;
|
|
1770
|
+
backbone.deleteDocument(rollback.key);
|
|
1771
|
+
if (rollback.value) {
|
|
1772
|
+
// documentValueBytes returns the complete stored encoding, so it can be
|
|
1773
|
+
// restored as one prefix with an empty suffix.
|
|
1774
|
+
backbone.putDocumentEncodedPartsStored(rollback.key, rollback.value, new Uint8Array(), rollback.byteElementIndexLimit);
|
|
1775
|
+
}
|
|
1776
|
+
}
|
|
1777
|
+
parseNativeStrictDurableTransactionJournalRecord(bytes, slot) {
|
|
1778
|
+
if (bytes === undefined) {
|
|
1779
|
+
return undefined;
|
|
1780
|
+
}
|
|
1781
|
+
// The pre-journal implementation represented a cleared intent as an empty
|
|
1782
|
+
// primary file. Treat it as generation zero so the first framed update is
|
|
1783
|
+
// written to the other slot and can never destroy the only valid state.
|
|
1784
|
+
if (bytes.byteLength === 0) {
|
|
1785
|
+
return { sequence: 0, slot };
|
|
1786
|
+
}
|
|
1787
|
+
let parsed;
|
|
1788
|
+
try {
|
|
1789
|
+
parsed = JSON.parse(new TextDecoder().decode(bytes));
|
|
1790
|
+
}
|
|
1791
|
+
catch (error) {
|
|
1792
|
+
throw new Error("Invalid native durable transaction journal JSON", {
|
|
1793
|
+
cause: error,
|
|
1794
|
+
});
|
|
1795
|
+
}
|
|
1796
|
+
if (!parsed || typeof parsed !== "object") {
|
|
1797
|
+
throw new Error("Invalid native durable transaction journal record");
|
|
1798
|
+
}
|
|
1799
|
+
const candidate = parsed;
|
|
1800
|
+
if (candidate.format === NATIVE_STRICT_DURABLE_TRANSACTION_JOURNAL_FORMAT) {
|
|
1801
|
+
if (candidate.version !== 1 ||
|
|
1802
|
+
!Number.isSafeInteger(candidate.sequence) ||
|
|
1803
|
+
(candidate.sequence ?? -1) < 1 ||
|
|
1804
|
+
(candidate.state !== "intent" && candidate.state !== "cleared") ||
|
|
1805
|
+
typeof candidate.checksum !== "string") {
|
|
1806
|
+
throw new Error("Invalid native durable transaction journal frame");
|
|
1807
|
+
}
|
|
1808
|
+
const intent = candidate.intent ?? null;
|
|
1809
|
+
if ((candidate.state === "intent" && intent?.version !== 1) ||
|
|
1810
|
+
(candidate.state === "cleared" && intent !== null)) {
|
|
1811
|
+
throw new Error("Invalid native durable transaction journal state");
|
|
1812
|
+
}
|
|
1813
|
+
const body = nativeStrictDurableTransactionJournalBody(candidate.sequence, intent ?? undefined);
|
|
1814
|
+
const checksum = toHexString(sha256Sync(nativeStrictDurableTransactionJournalBodyBytes(body)));
|
|
1815
|
+
if (checksum !== candidate.checksum) {
|
|
1816
|
+
throw new Error("Native durable transaction journal checksum mismatch");
|
|
1817
|
+
}
|
|
1818
|
+
return {
|
|
1819
|
+
sequence: candidate.sequence,
|
|
1820
|
+
slot,
|
|
1821
|
+
intent: intent ?? undefined,
|
|
1822
|
+
};
|
|
1823
|
+
}
|
|
1824
|
+
// Backward compatibility with the original single raw-JSON intent. A
|
|
1825
|
+
// framed generation always sorts after this synthetic generation zero.
|
|
1826
|
+
const legacy = parsed;
|
|
1827
|
+
if (legacy.version !== 1) {
|
|
1828
|
+
throw new Error("Unsupported native durable transaction recovery intent");
|
|
1829
|
+
}
|
|
1830
|
+
return {
|
|
1831
|
+
sequence: 0,
|
|
1832
|
+
slot,
|
|
1833
|
+
intent: legacy,
|
|
1834
|
+
};
|
|
1835
|
+
}
|
|
1836
|
+
async loadNativeStrictDurableTransactionJournalState() {
|
|
1837
|
+
if (this._nativeStrictDurableTransactionJournalState) {
|
|
1838
|
+
return this._nativeStrictDurableTransactionJournalState;
|
|
1839
|
+
}
|
|
1840
|
+
const store = this._nativeBackboneCoordinatePersistenceStore;
|
|
1841
|
+
if (!store) {
|
|
1842
|
+
return (this._nativeStrictDurableTransactionJournalState = {
|
|
1843
|
+
sequence: 0,
|
|
1844
|
+
slot: 0,
|
|
1845
|
+
});
|
|
1846
|
+
}
|
|
1847
|
+
const bytes = await Promise.all(NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILES.map((name) => store.read(name)));
|
|
1848
|
+
const valid = [];
|
|
1849
|
+
const errors = [];
|
|
1850
|
+
for (let index = 0; index < bytes.length; index++) {
|
|
1851
|
+
try {
|
|
1852
|
+
const state = this.parseNativeStrictDurableTransactionJournalRecord(bytes[index], index);
|
|
1853
|
+
if (state)
|
|
1854
|
+
valid.push(state);
|
|
1855
|
+
}
|
|
1856
|
+
catch (error) {
|
|
1857
|
+
errors.push(error);
|
|
1858
|
+
}
|
|
1859
|
+
}
|
|
1860
|
+
if (valid.length === 0) {
|
|
1861
|
+
if (errors.length > 0) {
|
|
1862
|
+
throw new AggregateError(errors, "No valid native durable transaction journal generation remains");
|
|
1863
|
+
}
|
|
1864
|
+
// A completely new store has an implicit cleared generation. Before the
|
|
1865
|
+
// first intent is written we materialize this baseline in one slot, so a
|
|
1866
|
+
// corrupt sole slot can never be confused with a safe first-write tear (or
|
|
1867
|
+
// with a torn legacy single-file intent).
|
|
1868
|
+
return (this._nativeStrictDurableTransactionJournalState = {
|
|
1869
|
+
sequence: 0,
|
|
1870
|
+
slot: 0,
|
|
1871
|
+
implicit: true,
|
|
1872
|
+
});
|
|
1873
|
+
}
|
|
1874
|
+
valid.sort((left, right) => left.sequence - right.sequence || left.slot - right.slot);
|
|
1875
|
+
return (this._nativeStrictDurableTransactionJournalState = valid.at(-1));
|
|
1876
|
+
}
|
|
1877
|
+
async writeNativeStrictDurableTransactionIntent(intent) {
|
|
1878
|
+
const store = this._nativeBackboneCoordinatePersistenceStore;
|
|
1879
|
+
if (!store) {
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1882
|
+
let previous = await this.loadNativeStrictDurableTransactionJournalState();
|
|
1883
|
+
if (previous.implicit) {
|
|
1884
|
+
const baselineSequence = previous.sequence + 1;
|
|
1885
|
+
const baselineSlot = (previous.slot === 0 ? 1 : 0);
|
|
1886
|
+
const baselineFile = NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILES[baselineSlot];
|
|
1887
|
+
await store.write(baselineFile, nativeStrictDurableTransactionJournalRecordBytes(baselineSequence, undefined));
|
|
1888
|
+
await this.barrierNativeStrictDurableStore(store, baselineFile);
|
|
1889
|
+
previous = {
|
|
1890
|
+
sequence: baselineSequence,
|
|
1891
|
+
slot: baselineSlot,
|
|
1892
|
+
};
|
|
1893
|
+
this._nativeStrictDurableTransactionJournalState = previous;
|
|
1894
|
+
}
|
|
1895
|
+
const sequence = previous.sequence + 1;
|
|
1896
|
+
const slot = (previous.slot === 0 ? 1 : 0);
|
|
1897
|
+
const bytes = nativeStrictDurableTransactionJournalRecordBytes(sequence, intent);
|
|
1898
|
+
const file = NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILES[slot];
|
|
1899
|
+
// Alternate slots. If this write is interrupted or torn, the previous
|
|
1900
|
+
// checksummed generation remains untouched and recovery ignores the invalid
|
|
1901
|
+
// newer slot. A durable shared-log generation requires an explicit physical
|
|
1902
|
+
// barrier before this generation can become recovery-authoritative.
|
|
1903
|
+
await store.write(file, bytes);
|
|
1904
|
+
await this.barrierNativeStrictDurableStore(store, file);
|
|
1905
|
+
this._nativeStrictDurableTransactionJournalState = {
|
|
1906
|
+
sequence,
|
|
1907
|
+
slot,
|
|
1908
|
+
intent,
|
|
1909
|
+
};
|
|
1910
|
+
}
|
|
1911
|
+
async barrierNativeStrictDurableStore(store, file) {
|
|
1912
|
+
if (this.node.directory != null) {
|
|
1913
|
+
if (typeof store.durableBarrier !== "function") {
|
|
1914
|
+
throw new Error("Durable native coordinate persistence does not expose a physical durability barrier");
|
|
1915
|
+
}
|
|
1916
|
+
await store.durableBarrier(file);
|
|
1917
|
+
return;
|
|
1918
|
+
}
|
|
1919
|
+
// Memory-only operation has no durable lower marker. Preserve compatibility
|
|
1920
|
+
// for transient adapters while still using a real barrier when they expose it.
|
|
1921
|
+
if (store.durableBarrier) {
|
|
1922
|
+
await store.durableBarrier(file);
|
|
1923
|
+
}
|
|
1924
|
+
else {
|
|
1925
|
+
await store.flush?.(file);
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
async beginNativeStrictDurableTransaction(documents) {
|
|
1929
|
+
this.throwIfNativeDurableCommitFailed();
|
|
1930
|
+
if (this._nativeStrictDurableTransactionsClosing) {
|
|
1931
|
+
throw new Error("Shared log is closing");
|
|
1932
|
+
}
|
|
1933
|
+
const previous = this._nativeStrictDurableTransactionTail ?? Promise.resolve();
|
|
1934
|
+
let release;
|
|
1935
|
+
const held = new Promise((resolve) => {
|
|
1936
|
+
release = resolve;
|
|
1937
|
+
});
|
|
1938
|
+
this._nativeStrictDurableTransactionTail = previous.then(() => held);
|
|
1939
|
+
await previous;
|
|
1940
|
+
try {
|
|
1941
|
+
this.throwIfNativeDurableCommitFailed();
|
|
1942
|
+
if (this._nativeStrictDurableTransactionsClosing) {
|
|
1943
|
+
throw new Error("Shared log is closing");
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
catch (error) {
|
|
1947
|
+
release();
|
|
1948
|
+
throw error;
|
|
1949
|
+
}
|
|
1950
|
+
const handle = {
|
|
1951
|
+
intent: {
|
|
1952
|
+
version: 1,
|
|
1953
|
+
lowerMarkerCommitted: false,
|
|
1954
|
+
appendHashes: [],
|
|
1955
|
+
trimHashes: [],
|
|
1956
|
+
coordinateDeleteHashes: [],
|
|
1957
|
+
lowerIndexRows: [],
|
|
1958
|
+
coordinates: [],
|
|
1959
|
+
documents: documents.map((document) => ({
|
|
1960
|
+
key: document.key,
|
|
1961
|
+
value: document.value ? [...document.value] : undefined,
|
|
1962
|
+
byteElementIndexLimit: document.byteElementIndexLimit,
|
|
1963
|
+
})),
|
|
1964
|
+
},
|
|
1965
|
+
release,
|
|
1966
|
+
released: false,
|
|
1967
|
+
};
|
|
1968
|
+
(this._nativeStrictDurableTransactions ??= new Set()).add(handle);
|
|
1969
|
+
try {
|
|
1970
|
+
await this.writeNativeStrictDurableTransactionIntent(handle.intent);
|
|
1971
|
+
return handle;
|
|
1972
|
+
}
|
|
1973
|
+
catch (error) {
|
|
1974
|
+
handle.released = true;
|
|
1975
|
+
this._nativeStrictDurableTransactions.delete(handle);
|
|
1976
|
+
handle.release();
|
|
1977
|
+
throw error;
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
async setNativeStrictDurableTransactionOperation(handle, appendHashes, trimHashes, coordinateRollback, coordinateDeleteHashes = []) {
|
|
1981
|
+
if (!handle) {
|
|
1982
|
+
return;
|
|
1983
|
+
}
|
|
1984
|
+
handle.intent.appendHashes = [...new Set(appendHashes.filter(Boolean))];
|
|
1985
|
+
handle.intent.trimHashes = [...new Set(trimHashes.filter(Boolean))];
|
|
1986
|
+
handle.intent.coordinateDeleteHashes = [
|
|
1987
|
+
...new Set(coordinateDeleteHashes.filter(Boolean)),
|
|
1988
|
+
];
|
|
1989
|
+
const lowerHashes = [
|
|
1990
|
+
...new Set([
|
|
1991
|
+
...handle.intent.appendHashes,
|
|
1992
|
+
...handle.intent.trimHashes,
|
|
1993
|
+
...(coordinateRollback?.hashes ?? []),
|
|
1994
|
+
...handle.intent.coordinateDeleteHashes,
|
|
1995
|
+
]),
|
|
1996
|
+
];
|
|
1997
|
+
if (handle.lowerHashMutationLockOwner) {
|
|
1998
|
+
throw new Error("Native durable transaction operation is already locked");
|
|
1999
|
+
}
|
|
2000
|
+
// This lease is shared with the lower native transaction and every ordinary
|
|
2001
|
+
// EntryIndex mutation. Acquire it before reading any before-image, then hold
|
|
2002
|
+
// it through marker acknowledgement or exact compensation.
|
|
2003
|
+
handle.lowerHashMutationLockOwner =
|
|
2004
|
+
await this.log.entryIndex.acquireHashMutationLocks(lowerHashes);
|
|
2005
|
+
handle.intent.lowerIndexRows = await Promise.all(lowerHashes.map(async (hash) => {
|
|
2006
|
+
// EntryIndex publication may still live only in its pending generation.
|
|
2007
|
+
// Snapshot the exact logical row that the marker phase will consume,
|
|
2008
|
+
// rather than only the durable index, so a pre-marker crash can restore a
|
|
2009
|
+
// pending external-next head instead of treating it as previously absent.
|
|
2010
|
+
const previous = (await this.log.entryIndex.getShallow(hash))?.value;
|
|
2011
|
+
return {
|
|
2012
|
+
hash,
|
|
2013
|
+
before: previous ? [...serialize(previous)] : undefined,
|
|
2014
|
+
};
|
|
2015
|
+
}));
|
|
2016
|
+
handle.intent.coordinates = coordinateRollback
|
|
2017
|
+
? [...coordinateRollback.hashes].map((hash) => {
|
|
2018
|
+
const previous = coordinateRollback.entries.get(hash);
|
|
2019
|
+
if (!previous) {
|
|
2020
|
+
return { hash };
|
|
2021
|
+
}
|
|
2022
|
+
const materialized = this.materializeResidentCoordinateEntry(previous);
|
|
2023
|
+
return {
|
|
2024
|
+
hash,
|
|
2025
|
+
value: {
|
|
2026
|
+
hashNumber: materialized.hashNumber.toString(),
|
|
2027
|
+
gid: materialized.gid,
|
|
2028
|
+
coordinates: materialized.coordinates.map((value) => value.toString()),
|
|
2029
|
+
wallTime: materialized.wallTime.toString(),
|
|
2030
|
+
assignedToRangeBoundary: materialized.assignedToRangeBoundary,
|
|
2031
|
+
metaBytes: [...materialized.getMetaBytes()],
|
|
2032
|
+
},
|
|
2033
|
+
};
|
|
2034
|
+
})
|
|
2035
|
+
: [];
|
|
2036
|
+
await this.writeNativeStrictDurableTransactionIntent(handle.intent);
|
|
2037
|
+
}
|
|
2038
|
+
async setNativeStrictDurableTransactionExpectedRows(handle, rows) {
|
|
2039
|
+
if (!handle) {
|
|
2040
|
+
return;
|
|
2041
|
+
}
|
|
2042
|
+
if (!handle.lowerHashMutationLockOwner) {
|
|
2043
|
+
throw new Error("Native durable transaction has no lower hash lock owner");
|
|
2044
|
+
}
|
|
2045
|
+
this.log.entryIndex.assertHashMutationLocks(handle.lowerHashMutationLockOwner, rows.flatMap((row) => [row.hash, ...row.meta.next]));
|
|
2046
|
+
const rowsByHash = new Map(handle.intent.lowerIndexRows.map((row) => [row.hash, row]));
|
|
2047
|
+
for (const row of rows) {
|
|
2048
|
+
const intentRow = rowsByHash.get(row.hash);
|
|
2049
|
+
if (intentRow) {
|
|
2050
|
+
intentRow.after = [...serialize(row)];
|
|
2051
|
+
}
|
|
2052
|
+
}
|
|
2053
|
+
for (const nextHash of new Set(rows.flatMap((row) => row.meta.next))) {
|
|
2054
|
+
const existingIntentRow = rowsByHash.get(nextHash);
|
|
2055
|
+
if (existingIntentRow) {
|
|
2056
|
+
if (!existingIntentRow.after && existingIntentRow.before) {
|
|
2057
|
+
const after = deserialize(Uint8Array.from(existingIntentRow.before), ShallowEntry);
|
|
2058
|
+
after.head = false;
|
|
2059
|
+
existingIntentRow.after = [...serialize(after)];
|
|
2060
|
+
}
|
|
2061
|
+
continue;
|
|
2062
|
+
}
|
|
2063
|
+
const previous = (await this.log.entryIndex.getShallow(nextHash))?.value;
|
|
2064
|
+
if (!previous) {
|
|
2065
|
+
continue;
|
|
2066
|
+
}
|
|
2067
|
+
const after = deserialize(serialize(previous), ShallowEntry);
|
|
2068
|
+
after.head = false;
|
|
2069
|
+
const intentRow = {
|
|
2070
|
+
hash: nextHash,
|
|
2071
|
+
before: [...serialize(previous)],
|
|
2072
|
+
after: [...serialize(after)],
|
|
2073
|
+
};
|
|
2074
|
+
handle.intent.lowerIndexRows.push(intentRow);
|
|
2075
|
+
rowsByHash.set(nextHash, intentRow);
|
|
2076
|
+
}
|
|
2077
|
+
handle.intent.coordinateDeleteHashes = [
|
|
2078
|
+
...new Set([
|
|
2079
|
+
...(handle.intent.coordinateDeleteHashes ?? []),
|
|
2080
|
+
...handle.intent.trimHashes,
|
|
2081
|
+
...rows.flatMap((row) => row.meta.next),
|
|
2082
|
+
]),
|
|
2083
|
+
];
|
|
2084
|
+
await this.writeNativeStrictDurableTransactionIntent(handle.intent);
|
|
2085
|
+
}
|
|
2086
|
+
async markNativeStrictDurableTransactionLowerMarker(handle) {
|
|
2087
|
+
if (!handle || handle.released) {
|
|
2088
|
+
return;
|
|
2089
|
+
}
|
|
2090
|
+
handle.intent.lowerMarkerCommitted = true;
|
|
2091
|
+
await this.writeNativeStrictDurableTransactionIntent(handle.intent);
|
|
2092
|
+
}
|
|
2093
|
+
async markNativeStrictDurableTransactionRollback(handle) {
|
|
2094
|
+
if (!handle || handle.released) {
|
|
2095
|
+
return;
|
|
2096
|
+
}
|
|
2097
|
+
const previousMarker = handle.intent.lowerMarkerCommitted;
|
|
2098
|
+
handle.intent.lowerMarkerCommitted = false;
|
|
2099
|
+
try {
|
|
2100
|
+
await this.writeNativeStrictDurableTransactionIntent(handle.intent);
|
|
2101
|
+
}
|
|
2102
|
+
catch (error) {
|
|
2103
|
+
// The last valid generation may still contain a true marker. Preserve
|
|
2104
|
+
// that in-memory knowledge and keep the handle held until the caller has
|
|
2105
|
+
// retained the lower finalizer. Releasing first would let concurrent close
|
|
2106
|
+
// compensate lower facts while recovery still sees a committed marker.
|
|
2107
|
+
handle.intent.lowerMarkerCommitted = previousMarker;
|
|
2108
|
+
this.poisonNativeStrictDurableTransaction(error);
|
|
2109
|
+
throw error;
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
async completeNativeStrictDurableTrimCleanup(intent, committed = intent.lowerMarkerCommitted === true, reconstructMissing = false) {
|
|
2113
|
+
if (!committed || intent.trimHashes.length === 0) {
|
|
2114
|
+
return;
|
|
2115
|
+
}
|
|
2116
|
+
const localStore = this.remoteBlocks?.localStore;
|
|
2117
|
+
if (typeof localStore?.completeCommittedNativeDeleteCleanup === "function") {
|
|
2118
|
+
await localStore.completeCommittedNativeDeleteCleanup(intent.trimHashes, {
|
|
2119
|
+
reconstructMissing,
|
|
2120
|
+
});
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
async completeNativeStrictDurableCoordinateCleanup(intent, committed = intent.lowerMarkerCommitted === true) {
|
|
2124
|
+
const hashes = intent.coordinateDeleteHashes ?? [];
|
|
2125
|
+
if (!committed || hashes.length === 0) {
|
|
2126
|
+
return;
|
|
2127
|
+
}
|
|
2128
|
+
await this.deleteCoordinatesForHashes(hashes);
|
|
2129
|
+
const flushed = this.flushNativeBackboneCoordinateJournal();
|
|
2130
|
+
if (isPromiseLike(flushed)) {
|
|
2131
|
+
await flushed;
|
|
2132
|
+
}
|
|
2133
|
+
}
|
|
2134
|
+
async completeNativeStrictDurableTransaction(handle) {
|
|
2135
|
+
if (!handle || handle.released) {
|
|
2136
|
+
return;
|
|
2137
|
+
}
|
|
2138
|
+
try {
|
|
2139
|
+
await this.completeNativeStrictDurableCoordinateCleanup(handle.intent);
|
|
2140
|
+
await this.completeNativeStrictDurableTrimCleanup(handle.intent);
|
|
2141
|
+
await this.writeNativeStrictDurableTransactionIntent(undefined);
|
|
2142
|
+
this.clearNativeStrictDurableTransactionFailure();
|
|
2143
|
+
}
|
|
2144
|
+
catch (error) {
|
|
2145
|
+
// The lower marker may already be acknowledged. Retain the intent and
|
|
2146
|
+
// reject every later mutation until reopen can finish recovery; allowing a
|
|
2147
|
+
// new transaction to overwrite this generation would make rollback/GC debt
|
|
2148
|
+
// ambiguous and can erase acknowledged data.
|
|
2149
|
+
this.poisonNativeStrictDurableTransaction(error);
|
|
2150
|
+
throw error;
|
|
2151
|
+
}
|
|
2152
|
+
finally {
|
|
2153
|
+
if (handle.lowerHashMutationLockOwner) {
|
|
2154
|
+
this.log.entryIndex.releaseHashMutationLocks(handle.lowerHashMutationLockOwner);
|
|
2155
|
+
handle.lowerHashMutationLockOwner = undefined;
|
|
2156
|
+
}
|
|
2157
|
+
handle.released = true;
|
|
2158
|
+
this._nativeStrictDurableTransactions?.delete(handle);
|
|
2159
|
+
handle.release();
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2162
|
+
releaseNativeStrictDurableTransaction(handle, cause = new Error("Native durable transaction intent was retained for recovery")) {
|
|
2163
|
+
if (!handle || handle.released) {
|
|
2164
|
+
return;
|
|
2165
|
+
}
|
|
2166
|
+
this.poisonNativeStrictDurableTransaction(cause);
|
|
2167
|
+
if (handle.lowerHashMutationLockOwner) {
|
|
2168
|
+
this.log.entryIndex.releaseHashMutationLocks(handle.lowerHashMutationLockOwner);
|
|
2169
|
+
handle.lowerHashMutationLockOwner = undefined;
|
|
2170
|
+
}
|
|
2171
|
+
handle.released = true;
|
|
2172
|
+
this._nativeStrictDurableTransactions?.delete(handle);
|
|
2173
|
+
handle.release();
|
|
2174
|
+
}
|
|
2175
|
+
retainNativeStrictDurableTransactionAfterMarkerFailure(handle, finalizer, cause) {
|
|
2176
|
+
const failures = [cause];
|
|
2177
|
+
try {
|
|
2178
|
+
finalizer?.retainForRecovery();
|
|
2179
|
+
}
|
|
2180
|
+
catch (error) {
|
|
2181
|
+
failures.push(error);
|
|
2182
|
+
}
|
|
2183
|
+
finally {
|
|
2184
|
+
// retainForRecovery finalizes its lower transaction even when one of its
|
|
2185
|
+
// internal cleanup steps reports an error. Only release the strict handle
|
|
2186
|
+
// after that synchronous state transition has been attempted.
|
|
2187
|
+
this.releaseNativeStrictDurableTransaction(handle, cause);
|
|
2188
|
+
}
|
|
2189
|
+
return failures;
|
|
2190
|
+
}
|
|
2191
|
+
async settleNativeStrictDurableTransactionsForClose() {
|
|
2192
|
+
while ((this._nativeStrictDurableTransactions?.size ?? 0) > 0) {
|
|
2193
|
+
const tail = this._nativeStrictDurableTransactionTail;
|
|
2194
|
+
if (!tail) {
|
|
2195
|
+
throw new Error("Native strict durable transaction has no settlement tail");
|
|
2196
|
+
}
|
|
2197
|
+
// A close racing an acknowledged lower marker must not release the strict
|
|
2198
|
+
// handle and let Log.close() compensate while the on-disk intent still says
|
|
2199
|
+
// committed. Wait until the owner either retires the intent or deliberately
|
|
2200
|
+
// retains it for recovery before closing the lower log or persistence stores.
|
|
2201
|
+
await tail;
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
async recoverNativeStrictDurableTransactionIntent(documentIndexReady = false) {
|
|
2205
|
+
const store = this._nativeBackboneCoordinatePersistenceStore;
|
|
2206
|
+
if (!store || !this._nativeBackbone) {
|
|
2207
|
+
this._nativeStrictDurableDocumentRecoveryDeferred = false;
|
|
2208
|
+
return true;
|
|
2209
|
+
}
|
|
2210
|
+
const journalState = await this.loadNativeStrictDurableTransactionJournalState();
|
|
2211
|
+
const intent = journalState.intent;
|
|
2212
|
+
if (!intent) {
|
|
2213
|
+
this._nativeStrictDurableDocumentRecoveryDeferred = false;
|
|
2214
|
+
this.clearNativeStrictDurableTransactionFailure();
|
|
2215
|
+
return true;
|
|
2216
|
+
}
|
|
2217
|
+
if (intent.version !== 1) {
|
|
2218
|
+
throw new Error("Unsupported native durable transaction recovery intent");
|
|
2219
|
+
}
|
|
2220
|
+
intent.trimHashes ??= [];
|
|
2221
|
+
intent.coordinateDeleteHashes ??= [];
|
|
2222
|
+
intent.lowerIndexRows ??= [];
|
|
2223
|
+
intent.coordinates ??= [];
|
|
2224
|
+
const bytesEqual = (left, right) => {
|
|
2225
|
+
if (!left || !right) {
|
|
2226
|
+
return left === undefined && right === undefined;
|
|
2227
|
+
}
|
|
2228
|
+
if (left.byteLength !== right.length) {
|
|
2229
|
+
return false;
|
|
2230
|
+
}
|
|
2231
|
+
for (let index = 0; index < left.byteLength; index++) {
|
|
2232
|
+
if (left[index] !== right[index]) {
|
|
2233
|
+
return false;
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
return true;
|
|
2237
|
+
};
|
|
2238
|
+
const immutableRowEquals = (current, expected) => {
|
|
2239
|
+
if (!current || !expected) {
|
|
2240
|
+
return current === undefined && expected === undefined;
|
|
2241
|
+
}
|
|
2242
|
+
const currentRow = deserialize(current, ShallowEntry);
|
|
2243
|
+
const expectedRow = deserialize(Uint8Array.from(expected), ShallowEntry);
|
|
2244
|
+
// `head` is a mutable graph projection. Hash, payload size, and metadata
|
|
2245
|
+
// are content-addressed append identity and are safe marker evidence even
|
|
2246
|
+
// when a later acknowledged entry has demoted this row.
|
|
2247
|
+
currentRow.head = false;
|
|
2248
|
+
expectedRow.head = false;
|
|
2249
|
+
return bytesEqual(serialize(currentRow), [...serialize(expectedRow)]);
|
|
2250
|
+
};
|
|
2251
|
+
const currentLowerRows = new Map();
|
|
2252
|
+
for (const row of intent.lowerIndexRows) {
|
|
2253
|
+
const current = (await this.log.entryIndex.properties.index.get(toId(row.hash)))?.value;
|
|
2254
|
+
currentLowerRows.set(row.hash, current ? serialize(current) : undefined);
|
|
2255
|
+
}
|
|
2256
|
+
const trimHashes = new Set(intent.trimHashes);
|
|
2257
|
+
const retainedMarkerRows = intent.lowerIndexRows.filter((row) => intent.appendHashes.includes(row.hash) &&
|
|
2258
|
+
!trimHashes.has(row.hash) &&
|
|
2259
|
+
row.after !== undefined);
|
|
2260
|
+
// Only a row known absent in the before-image is an unambiguous lower commit
|
|
2261
|
+
// marker. An existing content-addressed row can equal the after-image once
|
|
2262
|
+
// mutable `head` is ignored even before this transaction mutated anything.
|
|
2263
|
+
const expectedMarkerRows = retainedMarkerRows.filter((row) => row.before === undefined);
|
|
2264
|
+
let lowerMarkerCommitted = intent.lowerMarkerCommitted === true ||
|
|
2265
|
+
(expectedMarkerRows.length > 0 &&
|
|
2266
|
+
expectedMarkerRows.every((row) => immutableRowEquals(currentLowerRows.get(row.hash), row.after)));
|
|
2267
|
+
if (!lowerMarkerCommitted &&
|
|
2268
|
+
!documentIndexReady &&
|
|
2269
|
+
intent.documents.some((document) => document.value !== undefined)) {
|
|
2270
|
+
// SharedLog opens before Documents can attach its schema-aware native
|
|
2271
|
+
// index. Restoring an encoded before-image requires that schema. Keep the
|
|
2272
|
+
// intent authoritative and mutations poisoned until Documents has attached
|
|
2273
|
+
// the index and explicitly resumes recovery.
|
|
2274
|
+
this._nativeStrictDurableDocumentRecoveryDeferred = true;
|
|
2275
|
+
this.poisonNativeStrictDurableTransaction(new Error("Native strict durable document recovery is waiting for its document index"));
|
|
2276
|
+
return false;
|
|
2277
|
+
}
|
|
2278
|
+
const lowerIndex = this.log.entryIndex.properties
|
|
2279
|
+
.index;
|
|
2280
|
+
const deleteLowerIndexHash = async (hash) => {
|
|
2281
|
+
if (lowerIndex.delIds) {
|
|
2282
|
+
await lowerIndex.delIds([hash]);
|
|
2283
|
+
}
|
|
2284
|
+
else if (lowerIndex.delIdsNoReturn) {
|
|
2285
|
+
await lowerIndex.delIdsNoReturn([hash]);
|
|
2286
|
+
}
|
|
2287
|
+
else {
|
|
2288
|
+
await lowerIndex.del({ query: { hash } });
|
|
2289
|
+
}
|
|
2290
|
+
};
|
|
2291
|
+
let lowerIndexChanged = false;
|
|
2292
|
+
if (lowerMarkerCommitted) {
|
|
2293
|
+
for (const row of intent.lowerIndexRows) {
|
|
2294
|
+
if (trimHashes.has(row.hash) || !row.after) {
|
|
2295
|
+
continue;
|
|
2296
|
+
}
|
|
2297
|
+
const current = currentLowerRows.get(row.hash);
|
|
2298
|
+
if (immutableRowEquals(current, row.after)) {
|
|
2299
|
+
// Preserve the current mutable head projection. It may include a later
|
|
2300
|
+
// acknowledged Y -> X demotion that must survive recovery.
|
|
2301
|
+
continue;
|
|
2302
|
+
}
|
|
2303
|
+
if (!intent.appendHashes.includes(row.hash) || current !== undefined) {
|
|
2304
|
+
// External-next rows are not resurrected over a later delete, and a
|
|
2305
|
+
// conflicting present content-addressed row is never overwritten.
|
|
2306
|
+
continue;
|
|
2307
|
+
}
|
|
2308
|
+
await lowerIndex.put(deserialize(Uint8Array.from(row.after), ShallowEntry));
|
|
2309
|
+
lowerIndexChanged = true;
|
|
2310
|
+
}
|
|
2311
|
+
for (const hash of intent.trimHashes) {
|
|
2312
|
+
const current = await lowerIndex.get(toId(hash));
|
|
2313
|
+
const intentRow = intent.lowerIndexRows.find((row) => row.hash === hash);
|
|
2314
|
+
if (current &&
|
|
2315
|
+
intentRow?.before &&
|
|
2316
|
+
bytesEqual(serialize(current.value), intentRow.before)) {
|
|
2317
|
+
await deleteLowerIndexHash(hash);
|
|
2318
|
+
lowerIndexChanged = true;
|
|
2319
|
+
}
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
else {
|
|
2323
|
+
for (const row of intent.lowerIndexRows) {
|
|
2324
|
+
const current = currentLowerRows.get(row.hash);
|
|
2325
|
+
if (bytesEqual(current, row.before)) {
|
|
2326
|
+
continue;
|
|
2327
|
+
}
|
|
2328
|
+
// Exact after-image CAS: a later mutation (including only a `head`
|
|
2329
|
+
// change) owns the row and must not be erased or overwritten by recovery.
|
|
2330
|
+
if (!bytesEqual(current, row.after)) {
|
|
2331
|
+
continue;
|
|
2332
|
+
}
|
|
2333
|
+
if (row.before) {
|
|
2334
|
+
await lowerIndex.put(deserialize(Uint8Array.from(row.before), ShallowEntry));
|
|
2335
|
+
}
|
|
2336
|
+
else {
|
|
2337
|
+
await deleteLowerIndexHash(row.hash);
|
|
2338
|
+
}
|
|
2339
|
+
lowerIndexChanged = true;
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
if (lowerIndexChanged) {
|
|
2343
|
+
await this.log.entryIndex.init();
|
|
2344
|
+
}
|
|
2345
|
+
if (!lowerMarkerCommitted) {
|
|
2346
|
+
if (intent.coordinates.length > 0) {
|
|
2347
|
+
const mutationGenerations = (this._nativeCoordinateMutationGenerations ??= new Map());
|
|
2348
|
+
const rollback = {
|
|
2349
|
+
hashes: new Set(),
|
|
2350
|
+
entries: new Map(),
|
|
2351
|
+
generations: new Map(),
|
|
2352
|
+
};
|
|
2353
|
+
for (const coordinate of intent.coordinates) {
|
|
2354
|
+
rollback.hashes.add(coordinate.hash);
|
|
2355
|
+
const generation = (mutationGenerations.get(coordinate.hash) ?? 0) + 1;
|
|
2356
|
+
mutationGenerations.set(coordinate.hash, generation);
|
|
2357
|
+
rollback.generations.set(coordinate.hash, generation);
|
|
2358
|
+
if (coordinate.value) {
|
|
2359
|
+
const number = (value) => (this.domain.resolution === "u32"
|
|
2360
|
+
? Number(value)
|
|
2361
|
+
: BigInt(value));
|
|
2362
|
+
rollback.entries.set(coordinate.hash, new this.indexableDomain.constructorEntry({
|
|
2363
|
+
hash: coordinate.hash,
|
|
2364
|
+
hashNumber: number(coordinate.value.hashNumber),
|
|
2365
|
+
gid: coordinate.value.gid,
|
|
2366
|
+
coordinates: coordinate.value.coordinates.map(number),
|
|
2367
|
+
wallTime: BigInt(coordinate.value.wallTime),
|
|
2368
|
+
assignedToRangeBoundary: coordinate.value.assignedToRangeBoundary,
|
|
2369
|
+
metaBytes: Uint8Array.from(coordinate.value.metaBytes),
|
|
2370
|
+
}));
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
await this.rollbackNativeBackboneCoordinateAppendDurably("", rollback);
|
|
2374
|
+
}
|
|
2375
|
+
for (const document of intent.documents) {
|
|
2376
|
+
this.restoreNativeBackboneDocument({
|
|
2377
|
+
key: document.key,
|
|
2378
|
+
value: document.value ? Uint8Array.from(document.value) : undefined,
|
|
2379
|
+
byteElementIndexLimit: document.byteElementIndexLimit,
|
|
2380
|
+
});
|
|
2381
|
+
}
|
|
2382
|
+
const flushed = this.flushNativeBackboneCoordinateJournal();
|
|
2383
|
+
if (isPromiseLike(flushed)) {
|
|
2384
|
+
await flushed;
|
|
2385
|
+
}
|
|
2386
|
+
}
|
|
2387
|
+
if (lowerMarkerCommitted) {
|
|
2388
|
+
await this.completeNativeStrictDurableCoordinateCleanup(intent, true);
|
|
2389
|
+
await this.completeNativeStrictDurableTrimCleanup(intent, true, true);
|
|
2390
|
+
}
|
|
2391
|
+
await this.writeNativeStrictDurableTransactionIntent(undefined);
|
|
2392
|
+
this._nativeStrictDurableDocumentRecoveryDeferred = false;
|
|
2393
|
+
this.clearNativeStrictDurableTransactionFailure();
|
|
2394
|
+
return true;
|
|
2395
|
+
}
|
|
2396
|
+
/** @internal Complete a deferred rollback after Documents attaches its schema. */
|
|
2397
|
+
async finishNativeStrictDurableDocumentRecovery() {
|
|
2398
|
+
if (!this._nativeStrictDurableDocumentRecoveryDeferred) {
|
|
2399
|
+
return;
|
|
2400
|
+
}
|
|
2401
|
+
const completed = await this.recoverNativeStrictDurableTransactionIntent(true);
|
|
2402
|
+
if (!completed) {
|
|
2403
|
+
throw new Error("Native strict durable document recovery did not complete");
|
|
2404
|
+
}
|
|
2405
|
+
await this.reconcileNativeCoordinatesWithLowerCommitMarkers();
|
|
2406
|
+
}
|
|
2407
|
+
async rollbackFailedNativeBackboneTransaction(properties) {
|
|
2408
|
+
const backbone = this._nativeBackbone;
|
|
2409
|
+
if (!backbone)
|
|
2410
|
+
return;
|
|
2411
|
+
for (let index = properties.committedHashes.length - 1; index >= 0; index--) {
|
|
2412
|
+
const hash = properties.committedHashes[index];
|
|
2413
|
+
backbone.graph.delete(hash);
|
|
2414
|
+
this.rollbackNativeBackboneCoordinateAppend(hash, properties.coordinateEntries);
|
|
2415
|
+
}
|
|
2416
|
+
if (properties.restoreGraphFromIndex) {
|
|
2417
|
+
await this.log.entryIndex.restoreNativeGraphFromIndex();
|
|
2418
|
+
}
|
|
2419
|
+
else {
|
|
2420
|
+
if (properties.trimmedEntries?.length) {
|
|
2421
|
+
backbone.graph.putBatch(properties.trimmedEntries);
|
|
2422
|
+
}
|
|
2423
|
+
}
|
|
2424
|
+
for (const document of properties.documents ?? []) {
|
|
2425
|
+
this.restoreNativeBackboneDocument(document);
|
|
2426
|
+
}
|
|
2427
|
+
const flushed = this.flushNativeBackboneCoordinateJournal();
|
|
2428
|
+
if (isPromiseLike(flushed)) {
|
|
2429
|
+
await flushed;
|
|
2430
|
+
}
|
|
2431
|
+
if (properties.skipBlockCompensation) {
|
|
2432
|
+
return;
|
|
2433
|
+
}
|
|
2434
|
+
let compensated = false;
|
|
2435
|
+
try {
|
|
2436
|
+
if (properties.unmirroredBlockCompensation &&
|
|
2437
|
+
properties.durableWrapper?.rollbackUnmirroredNativeCommits) {
|
|
2438
|
+
await properties.durableWrapper.rollbackUnmirroredNativeCommits(properties.committedHashes, properties.trimmedEntries?.map((entry) => entry.hash));
|
|
2439
|
+
}
|
|
2440
|
+
else if (properties.durableWrapper?.rollbackFailedNativeCommits) {
|
|
2441
|
+
await properties.durableWrapper.rollbackFailedNativeCommits(properties.committedHashes, properties.trimmedEntries?.map((entry) => entry.hash));
|
|
2442
|
+
}
|
|
2443
|
+
else {
|
|
2444
|
+
await backbone.blocks.rmMany(properties.committedHashes);
|
|
2445
|
+
}
|
|
2446
|
+
compensated = true;
|
|
2447
|
+
}
|
|
2448
|
+
finally {
|
|
2449
|
+
this._nativeDurableRecoveryReadyForReopen = compensated;
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
openTime;
|
|
2453
|
+
oldestOpenTime;
|
|
2454
|
+
keep;
|
|
2455
|
+
// A fn that we can call many times that recalculates the participation role
|
|
2456
|
+
rebalanceParticipationDebounced;
|
|
2457
|
+
replicationAnnouncementRetryDebounced;
|
|
2458
|
+
_replicationAnnouncementRetryPending;
|
|
2459
|
+
_replicationAnnouncementRetryGeneration;
|
|
2460
|
+
_replicationAnnouncementRetryController;
|
|
2461
|
+
// A fn for debouncing the calls for pruning
|
|
2462
|
+
pruneDebouncedFn;
|
|
2463
|
+
responseToPruneDebouncedFn;
|
|
2464
|
+
get _requestIPruneSent() {
|
|
2465
|
+
return this._checkedPrune.requestIPruneSent;
|
|
2466
|
+
}
|
|
2467
|
+
get _requestIPruneResponseReplicatorSet() {
|
|
2468
|
+
return this._checkedPrune.responseReplicatorSet;
|
|
2469
|
+
}
|
|
2470
|
+
get _checkedPruneRetries() {
|
|
2471
|
+
return this._checkedPrune.retries;
|
|
2472
|
+
}
|
|
2473
|
+
replicationChangeDebounceFn;
|
|
2474
|
+
_repairRetryTimers;
|
|
2475
|
+
_recentRepairDispatch;
|
|
2476
|
+
_repairSweepRunning;
|
|
2477
|
+
_repairSweepPendingModes;
|
|
2478
|
+
_repairSweepPendingPeersByMode;
|
|
2479
|
+
_repairFrontierByMode;
|
|
2480
|
+
_repairFrontierActiveTargetsByMode;
|
|
2481
|
+
_repairFrontierBypassKnownPeersByMode;
|
|
2482
|
+
_repairSweepOptimisticGidPeersPending;
|
|
2483
|
+
_entryKnownPeers;
|
|
2484
|
+
_entryKnownPeerObservedAt;
|
|
2485
|
+
_joinAuthoritativeRepairTimersByDelay;
|
|
2486
|
+
_joinAuthoritativeRepairPeersByDelay;
|
|
2487
|
+
_assumeSyncedRepairSuppressedUntil;
|
|
2488
|
+
_appendBackfillTimer;
|
|
2489
|
+
_appendBackfillPendingByTarget;
|
|
2490
|
+
_repairMetrics;
|
|
2491
|
+
_topicSubscribersCache;
|
|
2492
|
+
_leaderSelectionContextCache;
|
|
2493
|
+
// Sync capability bits advertised by peers (SyncCapabilitiesMessage), keyed
|
|
2494
|
+
// by public key hash. Entries are dropped on unsubscribe/disconnect.
|
|
2495
|
+
_peerSyncCapabilities;
|
|
2496
|
+
// Pending live raw exchange-head gossip, coalesced per recipient set and
|
|
2497
|
+
// flushed at the end of the current event-loop turn (or when a batch cap
|
|
2498
|
+
// is hit). Only used when every recipient advertised raw capability.
|
|
2499
|
+
_liveRawGossipBatches;
|
|
2500
|
+
_liveRawGossipFlushScheduled;
|
|
2501
|
+
// regular distribution checks
|
|
2502
|
+
distributeQueue;
|
|
2503
|
+
syncronizer;
|
|
2504
|
+
replicas;
|
|
2505
|
+
cpuUsage;
|
|
2506
|
+
_lastLocalAppendAt;
|
|
2507
|
+
adaptiveRebalanceIdleMs;
|
|
2508
|
+
timeUntilRoleMaturity;
|
|
2509
|
+
waitForReplicatorTimeout;
|
|
2510
|
+
waitForReplicatorRequestIntervalMs;
|
|
2511
|
+
waitForReplicatorRequestMaxAttempts;
|
|
2512
|
+
waitForPruneDelay;
|
|
2513
|
+
distributionDebounceTime;
|
|
2514
|
+
repairSweepTargetBufferSize;
|
|
2515
|
+
replicationController;
|
|
2516
|
+
history;
|
|
2517
|
+
domain;
|
|
2518
|
+
indexableDomain;
|
|
2519
|
+
interval;
|
|
2520
|
+
constructor(properties) {
|
|
2521
|
+
super();
|
|
2522
|
+
this.ensureNativeDurabilityRuntimeState();
|
|
2523
|
+
this.log = new Log(properties);
|
|
2524
|
+
this.rpc = new RPC();
|
|
2525
|
+
this._checkedPrune = new CheckedPruneCoordinator();
|
|
2526
|
+
this._pendingIHave = new Map();
|
|
2527
|
+
this.latestReplicationInfoMessage = new Map();
|
|
997
2528
|
this._replicationInfoBlockedPeers = new Set();
|
|
998
2529
|
this._replicationInfoRequestByPeer = new Map();
|
|
999
2530
|
this._replicationInfoApplyQueueByPeer = new Map();
|
|
@@ -1025,9 +2556,23 @@ let SharedLog = (() => {
|
|
|
1025
2556
|
this._replicatorLivenessCursor = 0;
|
|
1026
2557
|
this._replicatorLivenessFailures = new Map();
|
|
1027
2558
|
this._replicatorLastActivityAt = new Map();
|
|
2559
|
+
this._replicationAnnouncementRetryPending = false;
|
|
2560
|
+
this._replicationAnnouncementRetryGeneration = 0;
|
|
2561
|
+
this._replicationAnnouncementRetryController = new AbortController();
|
|
1028
2562
|
this.pendingMaturity = new Map();
|
|
1029
2563
|
this._closeController = new AbortController();
|
|
1030
2564
|
}
|
|
2565
|
+
ensureNativeDurabilityRuntimeState() {
|
|
2566
|
+
// Program clones are borsh-created without running class field initializers.
|
|
2567
|
+
// Keep recovery state from an existing generation, while supplying fresh
|
|
2568
|
+
// defaults only when the runtime-only fields are absent.
|
|
2569
|
+
this._nativeDurableRecoveryReadyForReopen ??= false;
|
|
2570
|
+
this._nativeDurableRecoveryCids ??= new Set();
|
|
2571
|
+
this._nativeBackboneDropStarted ??= false;
|
|
2572
|
+
this._nativeBackboneCoordinateJournalLastFlushMs ??= 0;
|
|
2573
|
+
this._nativeStrictDurableDocumentRecoveryDeferred ??= false;
|
|
2574
|
+
this._nativeStrictDurableTransactionsClosing ??= false;
|
|
2575
|
+
}
|
|
1031
2576
|
get compatibility() {
|
|
1032
2577
|
return this._logProperties?.compatibility;
|
|
1033
2578
|
}
|
|
@@ -2031,6 +3576,7 @@ let SharedLog = (() => {
|
|
|
2031
3576
|
return undefined;
|
|
2032
3577
|
}
|
|
2033
3578
|
setupRebalanceDebounceFunction(interval = RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL) {
|
|
3579
|
+
this.rebalanceParticipationDebounced?.close();
|
|
2034
3580
|
this.rebalanceParticipationDebounced = undefined;
|
|
2035
3581
|
this.rebalanceParticipationDebounced = debounceFixedInterval(() => this.rebalanceParticipation(),
|
|
2036
3582
|
/* Math.max(
|
|
@@ -2040,7 +3586,133 @@ let SharedLog = (() => {
|
|
|
2040
3586
|
REBALANCE_DEBOUNCE_INTERVAL
|
|
2041
3587
|
)
|
|
2042
3588
|
) */
|
|
2043
|
-
interval
|
|
3589
|
+
interval, // TODO make this dynamic on the number of replicators
|
|
3590
|
+
{
|
|
3591
|
+
onError: (error) => this.onRebalanceParticipationError(error),
|
|
3592
|
+
});
|
|
3593
|
+
}
|
|
3594
|
+
queueCurrentReplicationStateAnnouncementRetry(error) {
|
|
3595
|
+
if (this.closed ||
|
|
3596
|
+
this._closeController.signal.aborted ||
|
|
3597
|
+
this._replicationAnnouncementRetryController.signal.aborted ||
|
|
3598
|
+
!isTransientReplicationAnnouncementError(error)) {
|
|
3599
|
+
return false;
|
|
3600
|
+
}
|
|
3601
|
+
this._replicationAnnouncementRetryPending = true;
|
|
3602
|
+
void this.replicationAnnouncementRetryDebounced?.call();
|
|
3603
|
+
return true;
|
|
3604
|
+
}
|
|
3605
|
+
onRebalanceParticipationError(error) {
|
|
3606
|
+
if (this.closed ||
|
|
3607
|
+
isNotStartedError(error) ||
|
|
3608
|
+
(isTransientReplicationAnnouncementError(error) &&
|
|
3609
|
+
this._replicationAnnouncementRetryPending)) {
|
|
3610
|
+
return;
|
|
3611
|
+
}
|
|
3612
|
+
// Debounced invocations run from an un-awaited timer. Throwing here would
|
|
3613
|
+
// create an unhandled rejection (and a browser pageerror), so surface
|
|
3614
|
+
// unexpected failures through the logger instead.
|
|
3615
|
+
logger.error(error);
|
|
3616
|
+
}
|
|
3617
|
+
setupReplicationAnnouncementRetryFunction(interval = REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL) {
|
|
3618
|
+
this.replicationAnnouncementRetryDebounced?.close();
|
|
3619
|
+
this._replicationAnnouncementRetryController?.abort();
|
|
3620
|
+
this._replicationAnnouncementRetryController = new AbortController();
|
|
3621
|
+
this.replicationAnnouncementRetryDebounced = debounceFixedInterval(() => this.retryCurrentReplicationStateAnnouncement(), interval, {
|
|
3622
|
+
leading: false,
|
|
3623
|
+
onError: (error) => {
|
|
3624
|
+
if (this.closed ||
|
|
3625
|
+
this._closeController.signal.aborted ||
|
|
3626
|
+
isNotStartedError(error)) {
|
|
3627
|
+
return;
|
|
3628
|
+
}
|
|
3629
|
+
logger.error(error);
|
|
3630
|
+
},
|
|
3631
|
+
});
|
|
3632
|
+
}
|
|
3633
|
+
cancelCurrentReplicationStateAnnouncementRetry() {
|
|
3634
|
+
this._replicationAnnouncementRetryGeneration += 1;
|
|
3635
|
+
this._replicationAnnouncementRetryPending = false;
|
|
3636
|
+
this._replicationAnnouncementRetryController?.abort();
|
|
3637
|
+
this.replicationAnnouncementRetryDebounced?.close();
|
|
3638
|
+
}
|
|
3639
|
+
async sendReplicationAnnouncement(message) {
|
|
3640
|
+
// Advance before every post-mutation send, including successful ones. An
|
|
3641
|
+
// authoritative retry already in flight may have captured the previous
|
|
3642
|
+
// local state; the generation mismatch forces one more current snapshot
|
|
3643
|
+
// after that stale send settles.
|
|
3644
|
+
this._replicationAnnouncementRetryGeneration += 1;
|
|
3645
|
+
try {
|
|
3646
|
+
await this.rpc.send(message, {
|
|
3647
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
3648
|
+
});
|
|
3649
|
+
}
|
|
3650
|
+
catch (error) {
|
|
3651
|
+
// The local replication-index mutation precedes all calls to this
|
|
3652
|
+
// wrapper. Preserve the explicit caller's rejection, but independently
|
|
3653
|
+
// schedule an authoritative snapshot so peers eventually observe the
|
|
3654
|
+
// already-committed local state.
|
|
3655
|
+
this.queueCurrentReplicationStateAnnouncementRetry(error);
|
|
3656
|
+
throw error;
|
|
3657
|
+
}
|
|
3658
|
+
}
|
|
3659
|
+
async retryCurrentReplicationStateAnnouncement() {
|
|
3660
|
+
const generation = this._replicationAnnouncementRetryGeneration;
|
|
3661
|
+
const controller = this._replicationAnnouncementRetryController;
|
|
3662
|
+
try {
|
|
3663
|
+
const segments = (await this.getMyReplicationSegments()).map((range) => range.toReplicationRange());
|
|
3664
|
+
if (this.closed ||
|
|
3665
|
+
this._closeController.signal.aborted ||
|
|
3666
|
+
controller.signal.aborted) {
|
|
3667
|
+
return;
|
|
3668
|
+
}
|
|
3669
|
+
if (generation !== this._replicationAnnouncementRetryGeneration) {
|
|
3670
|
+
void this.replicationAnnouncementRetryDebounced?.call();
|
|
3671
|
+
return;
|
|
3672
|
+
}
|
|
3673
|
+
await this.rpc.send(new AllReplicatingSegmentsMessage({ segments }), {
|
|
3674
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
3675
|
+
signal: controller.signal,
|
|
3676
|
+
});
|
|
3677
|
+
}
|
|
3678
|
+
catch (error) {
|
|
3679
|
+
if (this.closed ||
|
|
3680
|
+
this._closeController.signal.aborted ||
|
|
3681
|
+
controller.signal.aborted) {
|
|
3682
|
+
return;
|
|
3683
|
+
}
|
|
3684
|
+
if (this.queueCurrentReplicationStateAnnouncementRetry(error)) {
|
|
3685
|
+
return;
|
|
3686
|
+
}
|
|
3687
|
+
if (generation === this._replicationAnnouncementRetryGeneration) {
|
|
3688
|
+
this._replicationAnnouncementRetryPending = false;
|
|
3689
|
+
}
|
|
3690
|
+
else {
|
|
3691
|
+
void this.replicationAnnouncementRetryDebounced?.call();
|
|
3692
|
+
}
|
|
3693
|
+
throw error;
|
|
3694
|
+
}
|
|
3695
|
+
if (this.closed ||
|
|
3696
|
+
this._closeController.signal.aborted ||
|
|
3697
|
+
controller.signal.aborted) {
|
|
3698
|
+
return;
|
|
3699
|
+
}
|
|
3700
|
+
// A newer mutation announcement may have started while this snapshot was
|
|
3701
|
+
// in flight. In that case keep the repair pending so the newer current
|
|
3702
|
+
// state is also announced in full, regardless of whether its incremental
|
|
3703
|
+
// send succeeded or failed.
|
|
3704
|
+
if (generation === this._replicationAnnouncementRetryGeneration) {
|
|
3705
|
+
this._replicationAnnouncementRetryPending = false;
|
|
3706
|
+
if (!this.closed &&
|
|
3707
|
+
!this._closeController.signal.aborted &&
|
|
3708
|
+
!controller.signal.aborted &&
|
|
3709
|
+
this._isAdaptiveReplicating) {
|
|
3710
|
+
void this.rebalanceParticipationDebounced?.call();
|
|
3711
|
+
}
|
|
3712
|
+
}
|
|
3713
|
+
else {
|
|
3714
|
+
void this.replicationAnnouncementRetryDebounced?.call();
|
|
3715
|
+
}
|
|
2044
3716
|
}
|
|
2045
3717
|
markLocalAppendActivity(timestamp = Date.now()) {
|
|
2046
3718
|
this._lastLocalAppendAt = Math.max(this._lastLocalAppendAt ?? 0, timestamp);
|
|
@@ -2278,11 +3950,9 @@ let SharedLog = (() => {
|
|
|
2278
3950
|
rebalance,
|
|
2279
3951
|
});
|
|
2280
3952
|
if (rangesToUnreplicate.length > 0) {
|
|
2281
|
-
await this.
|
|
3953
|
+
await this.sendReplicationAnnouncement(new StoppedReplicating({
|
|
2282
3954
|
segmentIds: rangesToUnreplicate.map((x) => x.id),
|
|
2283
|
-
})
|
|
2284
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
2285
|
-
});
|
|
3955
|
+
}));
|
|
2286
3956
|
}
|
|
2287
3957
|
return rangesToReplicate;
|
|
2288
3958
|
}
|
|
@@ -2406,9 +4076,7 @@ let SharedLog = (() => {
|
|
|
2406
4076
|
}
|
|
2407
4077
|
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(segmentIds, this.node.identity.publicKey);
|
|
2408
4078
|
await this.removeReplicationRanges(rangesToRemove, this.node.identity.publicKey);
|
|
2409
|
-
await this.
|
|
2410
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
2411
|
-
});
|
|
4079
|
+
await this.sendReplicationAnnouncement(new StoppedReplicating({ segmentIds }));
|
|
2412
4080
|
}
|
|
2413
4081
|
async removeReplicator(key, options) {
|
|
2414
4082
|
const keyHash = typeof key === "string" ? key : key.hashcode();
|
|
@@ -2427,9 +4095,7 @@ let SharedLog = (() => {
|
|
|
2427
4095
|
const isMe = this.node.identity.publicKey.hashcode() === keyHash;
|
|
2428
4096
|
if (isMe) {
|
|
2429
4097
|
// announce that we are no longer replicating
|
|
2430
|
-
await this.
|
|
2431
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
2432
|
-
});
|
|
4098
|
+
await this.sendReplicationAnnouncement(new AllReplicatingSegmentsMessage({ segments: [] }));
|
|
2433
4099
|
}
|
|
2434
4100
|
if (options?.noEvent !== true) {
|
|
2435
4101
|
const publicKey = toLocalPublicSignKey(key);
|
|
@@ -2803,9 +4469,7 @@ let SharedLog = (() => {
|
|
|
2803
4469
|
return options.announce(message);
|
|
2804
4470
|
}
|
|
2805
4471
|
else {
|
|
2806
|
-
await this.
|
|
2807
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
2808
|
-
});
|
|
4472
|
+
await this.sendReplicationAnnouncement(message);
|
|
2809
4473
|
}
|
|
2810
4474
|
}
|
|
2811
4475
|
}
|
|
@@ -4055,6 +5719,7 @@ let SharedLog = (() => {
|
|
|
4055
5719
|
}
|
|
4056
5720
|
}
|
|
4057
5721
|
async append(data, options) {
|
|
5722
|
+
this.throwIfNativeDurableCommitFailed();
|
|
4058
5723
|
if (this._isAdaptiveReplicating) {
|
|
4059
5724
|
this.markLocalAppendActivity();
|
|
4060
5725
|
}
|
|
@@ -4067,6 +5732,7 @@ let SharedLog = (() => {
|
|
|
4067
5732
|
}
|
|
4068
5733
|
// Trusted local append path for callers that already validated the entry.
|
|
4069
5734
|
async appendLocallyValidated(data, options) {
|
|
5735
|
+
this.throwIfNativeDurableCommitFailed();
|
|
4070
5736
|
if (options?.canAppend || options?.onChange) {
|
|
4071
5737
|
throw new Error("appendLocallyValidated does not accept canAppend or onChange hooks");
|
|
4072
5738
|
}
|
|
@@ -4084,6 +5750,7 @@ let SharedLog = (() => {
|
|
|
4084
5750
|
}
|
|
4085
5751
|
// Trusted local append path that lets the shared log own change application.
|
|
4086
5752
|
async appendLocallyPrepared(data, options, properties) {
|
|
5753
|
+
this.throwIfNativeDurableCommitFailed();
|
|
4087
5754
|
if (options?.canAppend || options?.onChange) {
|
|
4088
5755
|
throw new Error("appendLocallyPrepared does not accept canAppend or onChange hooks");
|
|
4089
5756
|
}
|
|
@@ -4221,6 +5888,7 @@ let SharedLog = (() => {
|
|
|
4221
5888
|
}
|
|
4222
5889
|
// Trusted local payload append path that keeps the public Entry lazy.
|
|
4223
5890
|
appendLocallyPreparedPayloadCommitOnly(payloadData, options, properties) {
|
|
5891
|
+
this.throwIfNativeDurableCommitFailed();
|
|
4224
5892
|
if (options?.canAppend || options?.onChange) {
|
|
4225
5893
|
throw new Error("appendLocallyPreparedPayloadCommitOnly does not accept canAppend or onChange hooks");
|
|
4226
5894
|
}
|
|
@@ -4250,6 +5918,7 @@ let SharedLog = (() => {
|
|
|
4250
5918
|
}
|
|
4251
5919
|
// Strict native document path. Never falls back to compatibility append.
|
|
4252
5920
|
appendStrictNativeDocumentPayloadCommitOnly(payloadData, options, properties) {
|
|
5921
|
+
this.throwIfNativeDurableCommitFailed();
|
|
4253
5922
|
if (options?.canAppend || options?.onChange) {
|
|
4254
5923
|
throw new Error("appendStrictNativeDocumentPayloadCommitOnly does not accept canAppend or onChange hooks");
|
|
4255
5924
|
}
|
|
@@ -4316,116 +5985,300 @@ let SharedLog = (() => {
|
|
|
4316
5985
|
// commit-only block can be mirrored to durable WITHOUT routing it through
|
|
4317
5986
|
// the log's finishBlocks/putKnown* (which would disturb the strict-native
|
|
4318
5987
|
// resident-coordinate append path). `mirrorToDurable` writes to the durable
|
|
4319
|
-
// side only
|
|
5988
|
+
// side only; the lower-log result is held behind that durability barrier.
|
|
4320
5989
|
const durableWrapper = durableWrapperActive
|
|
4321
5990
|
? this.remoteBlocks?.localStore
|
|
4322
5991
|
: undefined;
|
|
4323
5992
|
let nativeBackboneDocumentIndexCommitted = false;
|
|
4324
|
-
let
|
|
5993
|
+
let nativeDeleteCleanupToken;
|
|
5994
|
+
let nativeDocumentRollback;
|
|
5995
|
+
let nativeStrictTransaction;
|
|
5996
|
+
let lowerPublicationRollback;
|
|
4325
5997
|
const nativeCommitProperties = {
|
|
4326
5998
|
payloadData,
|
|
4327
5999
|
resolveTrimmedEntries: properties?.resolveTrimmedEntries,
|
|
4328
6000
|
};
|
|
4329
|
-
nativeCommitProperties.skipMissingNextJoin =
|
|
4330
|
-
properties?.skipMissingNextJoin;
|
|
4331
|
-
nativeCommitProperties.retainMaterializationBytes =
|
|
4332
|
-
this._logProperties?.trim != null;
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4336
|
-
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
6001
|
+
nativeCommitProperties.skipMissingNextJoin =
|
|
6002
|
+
properties?.skipMissingNextJoin;
|
|
6003
|
+
nativeCommitProperties.retainMaterializationBytes =
|
|
6004
|
+
this._logProperties?.trim != null;
|
|
6005
|
+
nativeCommitProperties.deferNativeTransactionAcknowledgement = true;
|
|
6006
|
+
const rollbackLowerPublication = async (error) => {
|
|
6007
|
+
durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6008
|
+
const rollbackFailures = [];
|
|
6009
|
+
if (lowerPublicationRollback &&
|
|
6010
|
+
!(error instanceof NativeDurableCommitError)) {
|
|
6011
|
+
try {
|
|
6012
|
+
const lowerPublicationStarted = lowerPublicationRollback.lowerPublicationStarted;
|
|
6013
|
+
await this.rollbackFailedNativeBackboneTransaction({
|
|
6014
|
+
committedHashes: lowerPublicationRollback.committedHashes,
|
|
6015
|
+
trimmedEntries: lowerPublicationRollback.trimmedEntries,
|
|
6016
|
+
coordinateEntries: lowerPublicationRollback.coordinateEntries,
|
|
6017
|
+
documents: lowerPublicationRollback.documents,
|
|
6018
|
+
durableWrapper: lowerPublicationStarted
|
|
6019
|
+
? undefined
|
|
6020
|
+
: lowerPublicationRollback.durableWrapper,
|
|
6021
|
+
skipBlockCompensation: lowerPublicationStarted,
|
|
6022
|
+
unmirroredBlockCompensation: !lowerPublicationStarted,
|
|
6023
|
+
restoreGraphFromIndex: true,
|
|
6024
|
+
});
|
|
6025
|
+
}
|
|
6026
|
+
catch (rollbackError) {
|
|
6027
|
+
rollbackFailures.push(rollbackError);
|
|
6028
|
+
}
|
|
6029
|
+
}
|
|
6030
|
+
else if (nativeDocumentRollback &&
|
|
6031
|
+
!(error instanceof NativeDurableCommitError)) {
|
|
6032
|
+
try {
|
|
6033
|
+
this.restoreNativeBackboneDocument(nativeDocumentRollback);
|
|
6034
|
+
const flushed = this.flushNativeBackboneCoordinateJournal();
|
|
6035
|
+
if (isPromiseLike(flushed)) {
|
|
6036
|
+
await flushed;
|
|
6037
|
+
}
|
|
6038
|
+
}
|
|
6039
|
+
catch (rollbackError) {
|
|
6040
|
+
rollbackFailures.push(rollbackError);
|
|
6041
|
+
}
|
|
6042
|
+
}
|
|
6043
|
+
if (rollbackFailures.length > 0) {
|
|
6044
|
+
this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6045
|
+
throw new AggregateError([error, ...rollbackFailures], "Lower-log publication and native compensation both failed");
|
|
6046
|
+
}
|
|
6047
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6048
|
+
throw error;
|
|
6049
|
+
};
|
|
6050
|
+
let result;
|
|
6051
|
+
try {
|
|
6052
|
+
result = asTrustedLowerLog(this.log).appendLocallyPreparedNativeNoNextCommitOnly(undefined, appendOptions, nativeCommitProperties, async (input) => {
|
|
6053
|
+
const next = "next" in input && Array.isArray(input.next) ? input.next : [];
|
|
6054
|
+
const nativeBackboneDocumentIndex = properties?.nativeBackboneDocumentIndex ??
|
|
6055
|
+
properties?.prepareNativeBackboneDocumentIndex?.({
|
|
6056
|
+
wallTime: input.wallTime,
|
|
6057
|
+
gid: input.gid,
|
|
6058
|
+
payloadSize: input.payloadData.byteLength,
|
|
6059
|
+
});
|
|
6060
|
+
const nativeBackboneDocumentIndexForAppend = nativeBackboneDocumentIndex &&
|
|
6061
|
+
input.trimLengthTo == null &&
|
|
6062
|
+
nativeBackboneDocumentIndex.deleteTrimmedHeads === true
|
|
4361
6063
|
? {
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
...(useLatestDocumentContext
|
|
4365
|
-
? { useLatestContext: true }
|
|
4366
|
-
: {}),
|
|
4367
|
-
},
|
|
6064
|
+
...nativeBackboneDocumentIndex,
|
|
6065
|
+
deleteTrimmedHeads: false,
|
|
4368
6066
|
}
|
|
4369
|
-
:
|
|
4370
|
-
|
|
4371
|
-
|
|
4372
|
-
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
|
|
4385
|
-
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
6067
|
+
: nativeBackboneDocumentIndex;
|
|
6068
|
+
if (nativeBackboneDocumentIndex) {
|
|
6069
|
+
nativeBackboneDocumentIndexCommitted = true;
|
|
6070
|
+
}
|
|
6071
|
+
const useLatestDocumentContext = properties?.useNativeExistingDocumentContext === true;
|
|
6072
|
+
nativeDocumentRollback = this.snapshotNativeBackboneDocument(nativeBackboneDocumentIndexForAppend);
|
|
6073
|
+
nativeStrictTransaction =
|
|
6074
|
+
await this.beginNativeStrictDurableTransaction(nativeDocumentRollback ? [nativeDocumentRollback] : []);
|
|
6075
|
+
const prepared = backbone.graph.prepareEntryV0PlainEntryCommit({
|
|
6076
|
+
...input,
|
|
6077
|
+
next,
|
|
6078
|
+
includeMaterializationBytes: false,
|
|
6079
|
+
includeAppendFactsBytes: true,
|
|
6080
|
+
trimLengthTo: input.trimLengthTo,
|
|
6081
|
+
...(nativeBackboneDocumentIndexForAppend
|
|
6082
|
+
? {
|
|
6083
|
+
documentIndex: {
|
|
6084
|
+
...nativeBackboneDocumentIndexForAppend,
|
|
6085
|
+
...(useLatestDocumentContext
|
|
6086
|
+
? { useLatestContext: true }
|
|
6087
|
+
: {}),
|
|
6088
|
+
},
|
|
6089
|
+
}
|
|
6090
|
+
: {}),
|
|
6091
|
+
}, backbone.blocks);
|
|
6092
|
+
if (prepared) {
|
|
4392
6093
|
const preparedHash = prepared.cid ?? prepared.hash;
|
|
4393
|
-
const
|
|
4394
|
-
|
|
4395
|
-
:
|
|
4396
|
-
|
|
4397
|
-
|
|
6094
|
+
const preparedNext = prepared.next ?? next;
|
|
6095
|
+
const nativeTrimmedHashes = prepared.trimmedEntryHashes ??
|
|
6096
|
+
prepared.trimmedEntries?.flatMap((entry) => (entry.hash ? [entry.hash] : [])) ??
|
|
6097
|
+
[];
|
|
6098
|
+
const coordinateRollback = this.snapshotResidentCoordinateEntries([
|
|
6099
|
+
...(preparedHash ? [preparedHash] : []),
|
|
6100
|
+
...preparedNext,
|
|
6101
|
+
...nativeTrimmedHashes,
|
|
6102
|
+
]);
|
|
6103
|
+
lowerPublicationRollback = {
|
|
6104
|
+
committedHashes: preparedHash ? [preparedHash] : [],
|
|
6105
|
+
trimmedEntries: prepared.trimmedEntries,
|
|
6106
|
+
coordinateEntries: coordinateRollback,
|
|
6107
|
+
documents: nativeDocumentRollback
|
|
6108
|
+
? [nativeDocumentRollback]
|
|
6109
|
+
: undefined,
|
|
6110
|
+
durableWrapper,
|
|
6111
|
+
lowerPublicationStarted: false,
|
|
6112
|
+
};
|
|
6113
|
+
await this.setNativeStrictDurableTransactionOperation(nativeStrictTransaction, preparedHash ? [preparedHash] : [], nativeTrimmedHashes, coordinateRollback, combineCoordinateDeleteHashes(preparedNext, nativeTrimmedHashes));
|
|
6114
|
+
if (prepared.bytes) {
|
|
6115
|
+
lowerPublicationRollback.lowerPublicationStarted = true;
|
|
6116
|
+
return {
|
|
6117
|
+
...prepared,
|
|
6118
|
+
nativeIndexMutationLockOwner: nativeStrictTransaction?.lowerHashMutationLockOwner,
|
|
6119
|
+
};
|
|
6120
|
+
}
|
|
6121
|
+
const rollbackCommitted = async (cause, committedCids) => {
|
|
6122
|
+
durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6123
|
+
let compensated = false;
|
|
6124
|
+
try {
|
|
6125
|
+
await this.rollbackFailedNativeBackboneTransaction({
|
|
6126
|
+
committedHashes: committedCids,
|
|
6127
|
+
trimmedEntries: prepared.trimmedEntries,
|
|
6128
|
+
coordinateEntries: coordinateRollback,
|
|
6129
|
+
documents: nativeDocumentRollback
|
|
6130
|
+
? [nativeDocumentRollback]
|
|
6131
|
+
: undefined,
|
|
6132
|
+
durableWrapper,
|
|
6133
|
+
});
|
|
6134
|
+
compensated = true;
|
|
6135
|
+
}
|
|
6136
|
+
catch {
|
|
6137
|
+
// Keep recovery marked incomplete; close will discard pending native
|
|
6138
|
+
// journals. Reopen preserves uncertain content-addressed bytes and
|
|
6139
|
+
// recovers liveness from the authoritative lower-log facts.
|
|
6140
|
+
}
|
|
6141
|
+
if (compensated) {
|
|
6142
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6143
|
+
}
|
|
6144
|
+
else {
|
|
6145
|
+
this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6146
|
+
}
|
|
6147
|
+
return this.failNativeDurableCommit(cause, {
|
|
6148
|
+
committedCids,
|
|
6149
|
+
failedCids: committedCids,
|
|
6150
|
+
});
|
|
6151
|
+
};
|
|
6152
|
+
if (durableWrapper &&
|
|
6153
|
+
nativeTrimmedHashes.length > 0 &&
|
|
6154
|
+
!durableWrapper.beginNativeDeleteCleanup) {
|
|
6155
|
+
return rollbackCommitted(new Error("Native durable block wrapper cannot preannounce trim cleanup"), preparedHash ? [preparedHash] : []);
|
|
6156
|
+
}
|
|
6157
|
+
nativeDeleteCleanupToken =
|
|
6158
|
+
durableWrapper?.beginNativeDeleteCleanup?.(nativeTrimmedHashes);
|
|
6159
|
+
const preparedResult = {
|
|
6160
|
+
...prepared,
|
|
6161
|
+
nativeIndexMutationLockOwner: nativeStrictTransaction?.lowerHashMutationLockOwner,
|
|
6162
|
+
getBytes: (hash) => backbone.blocks.get(hash),
|
|
6163
|
+
nativeBlocksDeleted: true,
|
|
6164
|
+
nativeDeleteCleanupToken,
|
|
6165
|
+
};
|
|
6166
|
+
if (durableWrapper) {
|
|
6167
|
+
// The durable write-through wrapper is active, so the block store
|
|
6168
|
+
// the log writes through (this.remoteBlocks.localStore) is the
|
|
6169
|
+
// wrapper, NOT the raw wasm block map. prepareEntryV0PlainEntryCommit
|
|
6170
|
+
// committed the block into the wasm map ONLY and returned no raw
|
|
6171
|
+
// bytes, so on its own the block would never reach durable (log's
|
|
6172
|
+
// finishBlocks only calls putKnown* when prepared.bytes is set) and
|
|
6173
|
+
// a non-replicating native node would lose it on restart.
|
|
6174
|
+
//
|
|
6175
|
+
// Mirror the just-committed block (read back from the wasm store)
|
|
6176
|
+
// straight to the DURABLE side of the wrapper. Crucially we do NOT
|
|
6177
|
+
// attach prepared.bytes: doing so would make the log's finishBlocks
|
|
6178
|
+
// call putKnown*, which changes the commit-only append path and
|
|
6179
|
+
// breaks the strict-native resident-coordinate optimization (the
|
|
6180
|
+
// reopen tests assert the append stays native and resolves no entry
|
|
6181
|
+
// block). Instead the prepared result is returned exactly as the
|
|
6182
|
+
// memory-only branch below (getBytes only, no bytes), so the log's
|
|
6183
|
+
// finishBlocks path is UNCHANGED. Returning the mirror promise from
|
|
6184
|
+
// this prepare callback holds lower-log index/head/trim publication
|
|
6185
|
+
// until durable succeeds.
|
|
6186
|
+
if (!preparedHash) {
|
|
6187
|
+
durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6188
|
+
return rollbackCommitted(new Error("Native commit returned no entry CID to mirror"), []);
|
|
6189
|
+
}
|
|
6190
|
+
if (!durableWrapper.mirrorToDurable) {
|
|
6191
|
+
durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6192
|
+
return rollbackCommitted(new Error("Native durable block wrapper has no mirror method"), [preparedHash]);
|
|
6193
|
+
}
|
|
6194
|
+
const committedBytes = backbone.blocks.get(preparedHash);
|
|
6195
|
+
if (!committedBytes) {
|
|
6196
|
+
durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6197
|
+
return rollbackCommitted(new Error(`Native committed block ${preparedHash} is missing from the hot store`), [preparedHash]);
|
|
6198
|
+
}
|
|
6199
|
+
return durableWrapper
|
|
6200
|
+
.mirrorToDurable(preparedHash, committedBytes, {
|
|
6201
|
+
nativeTrimmed: nativeTrimmedHashes.includes(preparedHash),
|
|
6202
|
+
})
|
|
6203
|
+
.then((nativeCommitOwnershipToken) => {
|
|
6204
|
+
lowerPublicationRollback.lowerPublicationStarted = true;
|
|
6205
|
+
return {
|
|
6206
|
+
...preparedResult,
|
|
6207
|
+
nativeCommitOwnershipToken,
|
|
6208
|
+
};
|
|
6209
|
+
}, (error) => {
|
|
6210
|
+
durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6211
|
+
return rollbackCommitted(error, [preparedHash]);
|
|
6212
|
+
});
|
|
4398
6213
|
}
|
|
6214
|
+
lowerPublicationRollback.lowerPublicationStarted = true;
|
|
6215
|
+
return preparedResult;
|
|
4399
6216
|
}
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
getBytes: (hash) => backbone.blocks.get(hash),
|
|
4406
|
-
};
|
|
6217
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6218
|
+
return prepared;
|
|
6219
|
+
});
|
|
6220
|
+
if (isPromiseLike(result)) {
|
|
6221
|
+
result = result.catch(rollbackLowerPublication);
|
|
4407
6222
|
}
|
|
4408
|
-
|
|
4409
|
-
|
|
6223
|
+
}
|
|
6224
|
+
catch (error) {
|
|
6225
|
+
return rollbackLowerPublication(error);
|
|
6226
|
+
}
|
|
4410
6227
|
if (!result) {
|
|
4411
|
-
return undefined;
|
|
6228
|
+
return this.completeNativeStrictDurableTransaction(nativeStrictTransaction).then(() => undefined);
|
|
4412
6229
|
}
|
|
4413
|
-
return mapMaybePromise(result, (prepared) => {
|
|
6230
|
+
return mapMaybePromise(result, async (prepared) => {
|
|
4414
6231
|
if (!prepared) {
|
|
6232
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
4415
6233
|
return undefined;
|
|
4416
6234
|
}
|
|
4417
|
-
const rollback = (error) => {
|
|
4418
|
-
|
|
4419
|
-
|
|
6235
|
+
const rollback = async (error) => {
|
|
6236
|
+
const rollbackFailures = [];
|
|
6237
|
+
try {
|
|
6238
|
+
await this.markNativeStrictDurableTransactionRollback(nativeStrictTransaction);
|
|
6239
|
+
}
|
|
6240
|
+
catch (rollbackError) {
|
|
6241
|
+
const retentionFailures = this.retainNativeStrictDurableTransactionAfterMarkerFailure(nativeStrictTransaction, prepared.nativeCommittedAppendFinalizer, rollbackError);
|
|
6242
|
+
throw new AggregateError([error, ...retentionFailures], "Native rollback marker could not be persisted; recovery is required");
|
|
6243
|
+
}
|
|
6244
|
+
try {
|
|
6245
|
+
await prepared.nativeCommittedAppendFinalizer?.rollback();
|
|
6246
|
+
}
|
|
6247
|
+
catch (rollbackError) {
|
|
6248
|
+
rollbackFailures.push(rollbackError);
|
|
6249
|
+
}
|
|
6250
|
+
try {
|
|
6251
|
+
await this.rollbackNativeBackboneCoordinateAppendDurably(prepared.appendFacts.hash, lowerPublicationRollback?.coordinateEntries);
|
|
6252
|
+
for (const document of lowerPublicationRollback?.documents ?? []) {
|
|
6253
|
+
this.restoreNativeBackboneDocument(document);
|
|
6254
|
+
}
|
|
6255
|
+
const flushed = this.flushNativeBackboneCoordinateJournal();
|
|
6256
|
+
if (isPromiseLike(flushed)) {
|
|
6257
|
+
await flushed;
|
|
6258
|
+
}
|
|
6259
|
+
}
|
|
6260
|
+
catch (rollbackError) {
|
|
6261
|
+
rollbackFailures.push(rollbackError);
|
|
6262
|
+
}
|
|
6263
|
+
if (rollbackFailures.length === 0) {
|
|
6264
|
+
try {
|
|
6265
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6266
|
+
}
|
|
6267
|
+
catch (rollbackError) {
|
|
6268
|
+
rollbackFailures.push(rollbackError);
|
|
6269
|
+
}
|
|
6270
|
+
}
|
|
6271
|
+
else {
|
|
6272
|
+
this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6273
|
+
}
|
|
6274
|
+
if (rollbackFailures.length > 0) {
|
|
6275
|
+
throw new AggregateError([error, ...rollbackFailures], "Shared-log append and compensation both failed");
|
|
4420
6276
|
}
|
|
4421
6277
|
throw error;
|
|
4422
6278
|
};
|
|
6279
|
+
let finishResult;
|
|
4423
6280
|
try {
|
|
4424
|
-
|
|
4425
|
-
const deleteHashes = deferredCoordinateDeleteHashes &&
|
|
4426
|
-
deferredCoordinateDeleteHashes.length > 0
|
|
4427
|
-
? deferredCoordinateDeleteHashes
|
|
4428
|
-
: [];
|
|
6281
|
+
await this.setNativeStrictDurableTransactionExpectedRows(nativeStrictTransaction, [prepared.shallowEntry]);
|
|
4429
6282
|
const finish = () => {
|
|
4430
6283
|
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(prepared.appendFacts);
|
|
4431
6284
|
if (nativeBackboneDocumentIndexCommitted) {
|
|
@@ -4442,16 +6295,27 @@ let SharedLog = (() => {
|
|
|
4442
6295
|
appendCommit,
|
|
4443
6296
|
};
|
|
4444
6297
|
};
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
6298
|
+
if (!prepared.nativeCommittedAppendFinalizer) {
|
|
6299
|
+
throw new Error("Missing deferred native append finalizer");
|
|
6300
|
+
}
|
|
6301
|
+
// Strict success cannot honor batching thresholds: native
|
|
6302
|
+
// coordinate/document/signer facts must be physically durable before
|
|
6303
|
+
// the lower commit marker is acknowledged and its intent is retired.
|
|
6304
|
+
await this.flushNativeBackboneCoordinateJournal();
|
|
6305
|
+
await prepared.nativeCommittedAppendFinalizer.acknowledge(() => this.markNativeStrictDurableTransactionLowerMarker(nativeStrictTransaction));
|
|
6306
|
+
finishResult = finish();
|
|
4451
6307
|
}
|
|
4452
6308
|
catch (error) {
|
|
4453
6309
|
return rollback(error);
|
|
4454
6310
|
}
|
|
6311
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, { removedHashes: prepared.removedHashes });
|
|
6312
|
+
try {
|
|
6313
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6314
|
+
}
|
|
6315
|
+
catch (error) {
|
|
6316
|
+
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
6317
|
+
}
|
|
6318
|
+
return finishResult;
|
|
4455
6319
|
});
|
|
4456
6320
|
}
|
|
4457
6321
|
appendLocallyPreparedPayloadNativeBackboneStorageTransaction(payloadData, appendOptions, properties, minReplicasValue, runtimeOnlyCoordinates) {
|
|
@@ -4484,8 +6348,11 @@ let SharedLog = (() => {
|
|
|
4484
6348
|
durableWrapperActive;
|
|
4485
6349
|
let nativeBackboneDocumentIndexCommitted = false;
|
|
4486
6350
|
let nativeBackboneDocumentDeleteCommitted = false;
|
|
4487
|
-
let
|
|
4488
|
-
|
|
6351
|
+
let nativeDocumentRollback;
|
|
6352
|
+
let nativeCoordinateRollback;
|
|
6353
|
+
let nativeDeleteCleanupToken;
|
|
6354
|
+
let nativeStrictTransaction;
|
|
6355
|
+
const prepareBackboneAppend = async (input) => {
|
|
4489
6356
|
const next = input.next ?? [];
|
|
4490
6357
|
const appendInput = {
|
|
4491
6358
|
wallTime: input.wallTime,
|
|
@@ -4536,6 +6403,12 @@ let SharedLog = (() => {
|
|
|
4536
6403
|
documentDeleteKey: nativeBackboneDocumentDeleteKey,
|
|
4537
6404
|
}
|
|
4538
6405
|
: appendInput;
|
|
6406
|
+
nativeDocumentRollback = this.snapshotNativeBackboneDocument(nativeBackboneDocumentIndexForAppend ??
|
|
6407
|
+
(nativeBackboneDocumentDeleteKey
|
|
6408
|
+
? { key: nativeBackboneDocumentDeleteKey }
|
|
6409
|
+
: undefined));
|
|
6410
|
+
nativeStrictTransaction =
|
|
6411
|
+
await this.beginNativeStrictDurableTransaction(nativeDocumentRollback ? [nativeDocumentRollback] : []);
|
|
4539
6412
|
if (next.length === 0) {
|
|
4540
6413
|
if (commitBlocksInBackbone) {
|
|
4541
6414
|
if (nativeBackboneDocumentIndex &&
|
|
@@ -4568,29 +6441,61 @@ let SharedLog = (() => {
|
|
|
4568
6441
|
}
|
|
4569
6442
|
if (nativeBackboneDocumentIndex) {
|
|
4570
6443
|
nativeBackboneDocumentIndexCommitted = true;
|
|
4571
|
-
committedNativeBackboneDocumentIndex = nativeBackboneDocumentIndex;
|
|
4572
6444
|
}
|
|
4573
6445
|
nativeBackboneDocumentDeleteCommitted =
|
|
4574
6446
|
!!nativeBackboneDocumentDeleteKey;
|
|
4575
6447
|
const useTrimmedHashesOnly = properties?.resolveTrimmedEntries === false;
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4581
|
-
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
|
|
4585
|
-
|
|
4586
|
-
|
|
4587
|
-
|
|
4588
|
-
|
|
4589
|
-
|
|
6448
|
+
const nativeTrimmedHashes = backboneAppend.trimmedHashes ??
|
|
6449
|
+
backboneAppend.trimmed.map((entry) => entry.hash);
|
|
6450
|
+
const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
|
|
6451
|
+
const committedNext = backboneAppend.entry.next ?? next;
|
|
6452
|
+
nativeCoordinateRollback = this.snapshotResidentCoordinateEntries([
|
|
6453
|
+
...(committedHash ? [committedHash] : []),
|
|
6454
|
+
...committedNext,
|
|
6455
|
+
...nativeTrimmedHashes,
|
|
6456
|
+
]);
|
|
6457
|
+
await this.setNativeStrictDurableTransactionOperation(nativeStrictTransaction, committedHash ? [committedHash] : [], nativeTrimmedHashes, nativeCoordinateRollback, combineCoordinateDeleteHashes(committedNext, nativeTrimmedHashes));
|
|
6458
|
+
const rollbackCommitted = async (cause, committedCids) => {
|
|
6459
|
+
durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6460
|
+
let compensated = false;
|
|
6461
|
+
try {
|
|
6462
|
+
await this.rollbackFailedNativeBackboneTransaction({
|
|
6463
|
+
committedHashes: committedCids,
|
|
6464
|
+
trimmedEntries: backboneAppend?.trimmed,
|
|
6465
|
+
coordinateEntries: nativeCoordinateRollback,
|
|
6466
|
+
documents: nativeDocumentRollback
|
|
6467
|
+
? [nativeDocumentRollback]
|
|
6468
|
+
: undefined,
|
|
6469
|
+
durableWrapper,
|
|
6470
|
+
});
|
|
6471
|
+
compensated = true;
|
|
4590
6472
|
}
|
|
4591
|
-
|
|
4592
|
-
|
|
6473
|
+
catch {
|
|
6474
|
+
// close/reopen completes recovery if durable compensation failed
|
|
6475
|
+
}
|
|
6476
|
+
if (compensated) {
|
|
6477
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6478
|
+
}
|
|
6479
|
+
else {
|
|
6480
|
+
this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6481
|
+
}
|
|
6482
|
+
return this.failNativeDurableCommit(cause, {
|
|
6483
|
+
committedCids,
|
|
6484
|
+
failedCids: committedCids,
|
|
6485
|
+
});
|
|
6486
|
+
};
|
|
6487
|
+
if (durableWrapper &&
|
|
6488
|
+
commitBlocksInBackbone &&
|
|
6489
|
+
nativeTrimmedHashes.length > 0 &&
|
|
6490
|
+
!durableWrapper.beginNativeDeleteCleanup) {
|
|
6491
|
+
return rollbackCommitted(new Error("Native durable block wrapper cannot preannounce trim cleanup"), committedHash ? [committedHash] : []);
|
|
6492
|
+
}
|
|
6493
|
+
nativeDeleteCleanupToken = commitBlocksInBackbone
|
|
6494
|
+
? durableWrapper?.beginNativeDeleteCleanup?.(nativeTrimmedHashes)
|
|
6495
|
+
: undefined;
|
|
6496
|
+
const preparedResult = {
|
|
4593
6497
|
...backboneAppend.entry,
|
|
6498
|
+
nativeIndexMutationLockOwner: nativeStrictTransaction?.lowerHashMutationLockOwner,
|
|
4594
6499
|
gid: backboneAppend.coordinate.gid,
|
|
4595
6500
|
getBytes: commitBlocksInBackbone
|
|
4596
6501
|
? (hash) => backbone.blocks.get(hash)
|
|
@@ -4601,8 +6506,42 @@ let SharedLog = (() => {
|
|
|
4601
6506
|
trimmedEntryHashes: useTrimmedHashesOnly
|
|
4602
6507
|
? backboneAppend.trimmedHashes
|
|
4603
6508
|
: undefined,
|
|
6509
|
+
nativeBlocksDeleted: commitBlocksInBackbone,
|
|
6510
|
+
nativeDeleteCleanupToken,
|
|
4604
6511
|
documentPreviousContext: backboneAppend.documentPreviousContext,
|
|
4605
6512
|
};
|
|
6513
|
+
if (durableWrapper?.mirrorToDurable) {
|
|
6514
|
+
// The block was committed into the wasm map (commitBlocksInBackbone is
|
|
6515
|
+
// true) but not into durable, because the log's finishBlocks path is
|
|
6516
|
+
// left UNCHANGED for strict-native mode (getBytes only, no bytes, so
|
|
6517
|
+
// no putKnown* through the wrapper). Returning the promise here prevents
|
|
6518
|
+
// lower-log index/head/trim publication until the mirror settles.
|
|
6519
|
+
if (!committedHash) {
|
|
6520
|
+
durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6521
|
+
return rollbackCommitted(new Error("Native commit returned no entry CID to mirror"), []);
|
|
6522
|
+
}
|
|
6523
|
+
const committedBytes = backboneAppend.entry.bytes ?? backbone.blocks.get(committedHash);
|
|
6524
|
+
if (!committedBytes) {
|
|
6525
|
+
durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6526
|
+
return rollbackCommitted(new Error(`Native committed block ${committedHash} is missing from the hot store`), [committedHash]);
|
|
6527
|
+
}
|
|
6528
|
+
return durableWrapper
|
|
6529
|
+
.mirrorToDurable(committedHash, committedBytes, {
|
|
6530
|
+
nativeTrimmed: nativeTrimmedHashes.includes(committedHash),
|
|
6531
|
+
})
|
|
6532
|
+
.then((nativeCommitOwnershipToken) => ({
|
|
6533
|
+
...preparedResult,
|
|
6534
|
+
nativeCommitOwnershipToken,
|
|
6535
|
+
}), (error) => {
|
|
6536
|
+
durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6537
|
+
return rollbackCommitted(error, [committedHash]);
|
|
6538
|
+
});
|
|
6539
|
+
}
|
|
6540
|
+
if (durableWrapper) {
|
|
6541
|
+
durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6542
|
+
return rollbackCommitted(new Error("Native durable block wrapper has no mirror method"), committedHash ? [committedHash] : []);
|
|
6543
|
+
}
|
|
6544
|
+
return preparedResult;
|
|
4606
6545
|
};
|
|
4607
6546
|
const hasKnownNoNext = appendOptions.meta?.next != null &&
|
|
4608
6547
|
appendOptions.meta.next.length === 0;
|
|
@@ -4611,19 +6550,62 @@ let SharedLog = (() => {
|
|
|
4611
6550
|
resolveTrimmedEntries: properties?.resolveTrimmedEntries,
|
|
4612
6551
|
skipMissingNextJoin: properties?.skipMissingNextJoin,
|
|
4613
6552
|
retainMaterializationBytes: this._logProperties?.trim != null,
|
|
6553
|
+
deferNativeTransactionAcknowledgement: true,
|
|
4614
6554
|
}, prepareBackboneAppend);
|
|
4615
|
-
const
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
6555
|
+
const rollbackLowerPublication = async (error) => {
|
|
6556
|
+
durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
6557
|
+
let compensated = !backboneAppend;
|
|
6558
|
+
if (backboneAppend && !(error instanceof NativeDurableCommitError)) {
|
|
6559
|
+
const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
|
|
6560
|
+
try {
|
|
6561
|
+
await this.rollbackFailedNativeBackboneTransaction({
|
|
6562
|
+
committedHashes: committedHash ? [committedHash] : [],
|
|
6563
|
+
coordinateEntries: nativeCoordinateRollback,
|
|
6564
|
+
documents: nativeDocumentRollback
|
|
6565
|
+
? [nativeDocumentRollback]
|
|
6566
|
+
: undefined,
|
|
6567
|
+
skipBlockCompensation: true,
|
|
6568
|
+
restoreGraphFromIndex: true,
|
|
6569
|
+
});
|
|
6570
|
+
compensated = true;
|
|
6571
|
+
}
|
|
6572
|
+
catch {
|
|
6573
|
+
// Lower-log compensation already handled durable/native blocks.
|
|
6574
|
+
// Preserve the index publication error for this caller.
|
|
6575
|
+
}
|
|
6576
|
+
}
|
|
6577
|
+
if (compensated) {
|
|
6578
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6579
|
+
}
|
|
6580
|
+
else {
|
|
6581
|
+
this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6582
|
+
}
|
|
6583
|
+
throw error;
|
|
6584
|
+
};
|
|
6585
|
+
let result;
|
|
6586
|
+
try {
|
|
6587
|
+
const directNoNextResult = hasKnownNoNext
|
|
6588
|
+
? asTrustedLowerLog(this.log).appendLocallyPreparedNativeKnownNoNextCommitOnly(undefined, appendOptions, {
|
|
6589
|
+
payloadData,
|
|
6590
|
+
resolveTrimmedEntries: properties?.resolveTrimmedEntries,
|
|
6591
|
+
retainMaterializationBytes: this._logProperties?.trim != null,
|
|
6592
|
+
deferNativeTransactionAcknowledgement: true,
|
|
6593
|
+
}, prepareBackboneAppend)
|
|
6594
|
+
: undefined;
|
|
6595
|
+
result =
|
|
6596
|
+
directNoNextResult === undefined
|
|
6597
|
+
? appendGenericNativeCommit()
|
|
6598
|
+
: directNoNextResult;
|
|
6599
|
+
if (isPromiseLike(result)) {
|
|
6600
|
+
result = result.catch(rollbackLowerPublication);
|
|
6601
|
+
}
|
|
6602
|
+
}
|
|
6603
|
+
catch (error) {
|
|
6604
|
+
return rollbackLowerPublication(error);
|
|
6605
|
+
}
|
|
6606
|
+
return mapMaybePromise(result, async (prepared) => {
|
|
4626
6607
|
if (!prepared || !backboneAppend) {
|
|
6608
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
4627
6609
|
return undefined;
|
|
4628
6610
|
}
|
|
4629
6611
|
const coordinateFields = this.createCoordinateFieldsFromNativePlanFacts({
|
|
@@ -4639,11 +6621,11 @@ let SharedLog = (() => {
|
|
|
4639
6621
|
fields: coordinateFields,
|
|
4640
6622
|
});
|
|
4641
6623
|
const plannedCoordinateDeleteHashes = combineCoordinateDeleteHashes(prepared.appendFacts.next, prepared.removedHashes ?? prepared.removed.map((entry) => entry.hash));
|
|
4642
|
-
const rollbackCoordinateEntries =
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
6624
|
+
const rollbackCoordinateEntries = nativeCoordinateRollback ??
|
|
6625
|
+
this.snapshotResidentCoordinateEntries([
|
|
6626
|
+
prepared.appendFacts.hash,
|
|
6627
|
+
...plannedCoordinateDeleteHashes,
|
|
6628
|
+
]);
|
|
4647
6629
|
const finish = () => {
|
|
4648
6630
|
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(prepared.appendFacts, {
|
|
4649
6631
|
hashNumber: backboneAppend.coordinate
|
|
@@ -4671,14 +6653,48 @@ let SharedLog = (() => {
|
|
|
4671
6653
|
};
|
|
4672
6654
|
};
|
|
4673
6655
|
const coordinateIndex = this.entryCoordinatesIndex;
|
|
4674
|
-
const rollback = (error) => {
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
6656
|
+
const rollback = async (error) => {
|
|
6657
|
+
const rollbackFailures = [];
|
|
6658
|
+
try {
|
|
6659
|
+
await this.markNativeStrictDurableTransactionRollback(nativeStrictTransaction);
|
|
6660
|
+
}
|
|
6661
|
+
catch (rollbackError) {
|
|
6662
|
+
const retentionFailures = this.retainNativeStrictDurableTransactionAfterMarkerFailure(nativeStrictTransaction, prepared.nativeCommittedAppendFinalizer, rollbackError);
|
|
6663
|
+
throw new AggregateError([error, ...retentionFailures], "Native rollback marker could not be persisted; recovery is required");
|
|
6664
|
+
}
|
|
6665
|
+
try {
|
|
6666
|
+
await prepared.nativeCommittedAppendFinalizer?.rollback();
|
|
6667
|
+
}
|
|
6668
|
+
catch (rollbackError) {
|
|
6669
|
+
rollbackFailures.push(rollbackError);
|
|
6670
|
+
}
|
|
6671
|
+
try {
|
|
6672
|
+
await this.rollbackNativeBackboneCoordinateAppendDurably(prepared.appendFacts.hash, rollbackCoordinateEntries);
|
|
6673
|
+
}
|
|
6674
|
+
catch (rollbackError) {
|
|
6675
|
+
rollbackFailures.push(rollbackError);
|
|
6676
|
+
}
|
|
6677
|
+
if (nativeDocumentRollback) {
|
|
6678
|
+
try {
|
|
6679
|
+
this.restoreNativeBackboneDocument(nativeDocumentRollback);
|
|
6680
|
+
const flushed = this.flushNativeBackboneCoordinateJournal();
|
|
6681
|
+
if (isPromiseLike(flushed)) {
|
|
6682
|
+
await flushed;
|
|
6683
|
+
}
|
|
6684
|
+
}
|
|
6685
|
+
catch (rollbackError) {
|
|
6686
|
+
rollbackFailures.push(rollbackError);
|
|
6687
|
+
}
|
|
4678
6688
|
}
|
|
6689
|
+
if (rollbackFailures.length > 0) {
|
|
6690
|
+
this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6691
|
+
throw new AggregateError([error, ...rollbackFailures], "Shared-log append and compensation both failed");
|
|
6692
|
+
}
|
|
6693
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
4679
6694
|
throw error;
|
|
4680
6695
|
};
|
|
4681
6696
|
try {
|
|
6697
|
+
await this.setNativeStrictDurableTransactionExpectedRows(nativeStrictTransaction, [prepared.shallowEntry]);
|
|
4682
6698
|
const hasNativeCoordinatePut = this.canUseBackboneOnlyCoordinatePersistence() ||
|
|
4683
6699
|
coordinateIndex.putSharedLogCoordinateFieldsEncodedAndDeleteHashesNoReturn ||
|
|
4684
6700
|
coordinateIndex.putSharedLogCoordinateFieldsAndDeleteHashesNoReturn;
|
|
@@ -4687,7 +6703,7 @@ let SharedLog = (() => {
|
|
|
4687
6703
|
coordinateIndex,
|
|
4688
6704
|
fields: coordinateFields,
|
|
4689
6705
|
hash: prepared.appendFacts.hash,
|
|
4690
|
-
deleteHashes:
|
|
6706
|
+
deleteHashes: [],
|
|
4691
6707
|
coordinates: backboneAppend.coordinate
|
|
4692
6708
|
.coordinates,
|
|
4693
6709
|
skipGenericTransientCoordinateIndex: runtimeOnlyCoordinates,
|
|
@@ -4695,43 +6711,56 @@ let SharedLog = (() => {
|
|
|
4695
6711
|
: this.persistPreparedCoordinate({
|
|
4696
6712
|
prepared: getPreparedCoordinate(),
|
|
4697
6713
|
hash: prepared.appendFacts.hash,
|
|
4698
|
-
nextHashes:
|
|
4699
|
-
deleteHashes:
|
|
6714
|
+
nextHashes: [],
|
|
6715
|
+
deleteHashes: [],
|
|
4700
6716
|
coordinates: backboneAppend.coordinate
|
|
4701
6717
|
.coordinates,
|
|
4702
6718
|
replicas: backboneAppend.coordinate.coordinates.length,
|
|
4703
6719
|
commitNative: true,
|
|
4704
6720
|
commitNativeBackbone: false,
|
|
4705
6721
|
});
|
|
4706
|
-
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
|
|
4712
|
-
|
|
4713
|
-
|
|
4714
|
-
const leaders = backboneAppend.leaders;
|
|
4715
|
-
if (leaders) {
|
|
4716
|
-
const pruneEntry = this.materializePreparedCoordinateEntry(getPreparedCoordinate());
|
|
4717
|
-
this.pruneDebouncedFnAddIfNotKeeping({
|
|
4718
|
-
key: pruneEntry.hash,
|
|
4719
|
-
value: { entry: pruneEntry, leaders },
|
|
4720
|
-
});
|
|
4721
|
-
}
|
|
4722
|
-
}
|
|
4723
|
-
if (!delayAdaptiveRebalance) {
|
|
4724
|
-
this.rebalanceParticipationDebounced?.call();
|
|
4725
|
-
}
|
|
4726
|
-
return finish();
|
|
4727
|
-
});
|
|
4728
|
-
return isPromiseLike(completed)
|
|
4729
|
-
? completed.catch((error) => rollback(error))
|
|
4730
|
-
: completed;
|
|
6722
|
+
if (isPromiseLike(persisted)) {
|
|
6723
|
+
await persisted;
|
|
6724
|
+
}
|
|
6725
|
+
if (!prepared.nativeCommittedAppendFinalizer) {
|
|
6726
|
+
throw new Error("Missing deferred native append finalizer");
|
|
6727
|
+
}
|
|
6728
|
+
await this.flushNativeBackboneCoordinateJournal();
|
|
6729
|
+
await prepared.nativeCommittedAppendFinalizer.acknowledge(() => this.markNativeStrictDurableTransactionLowerMarker(nativeStrictTransaction));
|
|
4731
6730
|
}
|
|
4732
6731
|
catch (error) {
|
|
4733
6732
|
return rollback(error);
|
|
4734
6733
|
}
|
|
6734
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, {
|
|
6735
|
+
forgetNativeCoordinates: false,
|
|
6736
|
+
removedHashes: prepared.removedHashes,
|
|
6737
|
+
});
|
|
6738
|
+
try {
|
|
6739
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
6740
|
+
}
|
|
6741
|
+
catch (error) {
|
|
6742
|
+
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
6743
|
+
}
|
|
6744
|
+
if (commitBlocksInBackbone &&
|
|
6745
|
+
!runtimeOnlyCoordinates &&
|
|
6746
|
+
this.remoteBlocks.hasNotifyStoredHook()) {
|
|
6747
|
+
this.remoteBlocks.notifyStoredDeferred(prepared.appendFacts.hash);
|
|
6748
|
+
}
|
|
6749
|
+
const delayAdaptiveRebalance = this.shouldDelayAdaptiveRebalance();
|
|
6750
|
+
if (!backboneAppend.isLeader && !delayAdaptiveRebalance) {
|
|
6751
|
+
const leaders = backboneAppend.leaders;
|
|
6752
|
+
if (leaders) {
|
|
6753
|
+
const pruneEntry = this.materializePreparedCoordinateEntry(getPreparedCoordinate());
|
|
6754
|
+
this.pruneDebouncedFnAddIfNotKeeping({
|
|
6755
|
+
key: pruneEntry.hash,
|
|
6756
|
+
value: { entry: pruneEntry, leaders },
|
|
6757
|
+
});
|
|
6758
|
+
}
|
|
6759
|
+
}
|
|
6760
|
+
if (!delayAdaptiveRebalance) {
|
|
6761
|
+
this.rebalanceParticipationDebounced?.call();
|
|
6762
|
+
}
|
|
6763
|
+
return finish();
|
|
4735
6764
|
});
|
|
4736
6765
|
});
|
|
4737
6766
|
}
|
|
@@ -4985,16 +7014,27 @@ let SharedLog = (() => {
|
|
|
4985
7014
|
const context = await this.createLeaderSelectionContext();
|
|
4986
7015
|
const nativeLeaderOptions = this.createNativeLeaderOptions(context);
|
|
4987
7016
|
let backboneAppends;
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
|
|
4992
|
-
|
|
4993
|
-
|
|
4994
|
-
|
|
4995
|
-
|
|
4996
|
-
|
|
4997
|
-
usesLatestDocumentContext
|
|
7017
|
+
let batchDocumentRollbacks = [];
|
|
7018
|
+
let batchCoordinateRollback;
|
|
7019
|
+
let nativeDeleteCleanupToken;
|
|
7020
|
+
let nativeStrictTransaction;
|
|
7021
|
+
let appended;
|
|
7022
|
+
try {
|
|
7023
|
+
appended = await asTrustedLowerLog(this.log).appendLocallyPreparedNativeKnownNoNextCommitOnlyBatch(data, appendOptions, {
|
|
7024
|
+
payloadDatas,
|
|
7025
|
+
resolveTrimmedEntries: properties?.resolveTrimmedEntries,
|
|
7026
|
+
allowPreparedNexts: usesLatestDocumentContext,
|
|
7027
|
+
retainMaterializationBytes: properties?.retainMaterializationBytes === true ||
|
|
7028
|
+
this._logProperties?.trim != null,
|
|
7029
|
+
deferNativeTransactionAcknowledgement: true,
|
|
7030
|
+
}, async (inputs) => {
|
|
7031
|
+
batchDocumentRollbacks = documentIndexes
|
|
7032
|
+
.map((index) => this.snapshotNativeBackboneDocument(index))
|
|
7033
|
+
.filter((value) => !!value);
|
|
7034
|
+
nativeStrictTransaction =
|
|
7035
|
+
await this.beginNativeStrictDurableTransaction(batchDocumentRollbacks);
|
|
7036
|
+
const documentDeleteTrimmedHeadsForAppend = deleteTrimmedHeads && inputs[0]?.trimLengthTo != null;
|
|
7037
|
+
backboneAppends = usesLatestDocumentContext
|
|
4998
7038
|
? backbone.preparePlainCommittedStorageAppendDocumentIndexLatestBatchTransaction({
|
|
4999
7039
|
entries: inputs.map((input, index) => ({
|
|
5000
7040
|
wallTime: input.wallTime,
|
|
@@ -5034,72 +7074,294 @@ let SharedLog = (() => {
|
|
|
5034
7074
|
documentDeleteTrimmedHeads: documentDeleteTrimmedHeadsForAppend,
|
|
5035
7075
|
trimLengthTo: inputs[0]?.trimLengthTo,
|
|
5036
7076
|
});
|
|
5037
|
-
|
|
5038
|
-
|
|
5039
|
-
|
|
5040
|
-
|
|
5041
|
-
|
|
5042
|
-
|
|
5043
|
-
|
|
5044
|
-
|
|
5045
|
-
|
|
5046
|
-
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
7077
|
+
if (!backboneAppends) {
|
|
7078
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7079
|
+
return undefined;
|
|
7080
|
+
}
|
|
7081
|
+
const committedAppends = backboneAppends;
|
|
7082
|
+
const committedCids = committedAppends
|
|
7083
|
+
.map((append) => append.entry.cid ?? append.entry.hash)
|
|
7084
|
+
.filter((cid) => !!cid);
|
|
7085
|
+
const nativeTrimmedHashSet = new Set(committedAppends.flatMap((append) => append.trimmedHashes ??
|
|
7086
|
+
append.trimmed.map((entry) => entry.hash)));
|
|
7087
|
+
const nativeTrimmedHashes = [...nativeTrimmedHashSet];
|
|
7088
|
+
batchCoordinateRollback = this.snapshotResidentCoordinateEntries(committedAppends.flatMap((append) => [
|
|
7089
|
+
...((append.entry.cid ?? append.entry.hash)
|
|
7090
|
+
? [append.entry.cid ?? append.entry.hash]
|
|
7091
|
+
: []),
|
|
7092
|
+
...append.entry.next,
|
|
7093
|
+
...(append.trimmedHashes ??
|
|
7094
|
+
append.trimmed.map((entry) => entry.hash)),
|
|
7095
|
+
]));
|
|
7096
|
+
await this.setNativeStrictDurableTransactionOperation(nativeStrictTransaction, committedCids, nativeTrimmedHashes, batchCoordinateRollback, committedAppends.flatMap((append) => [
|
|
7097
|
+
...append.entry.next,
|
|
7098
|
+
...(append.trimmedHashes ??
|
|
7099
|
+
append.trimmed.map((entry) => entry.hash)),
|
|
7100
|
+
]));
|
|
7101
|
+
const rollbackCommitted = async (cause) => {
|
|
7102
|
+
durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
7103
|
+
let compensated = false;
|
|
7104
|
+
try {
|
|
7105
|
+
await this.rollbackFailedNativeBackboneTransaction({
|
|
7106
|
+
committedHashes: committedCids,
|
|
7107
|
+
trimmedEntries: committedAppends.flatMap((append) => append.trimmed),
|
|
7108
|
+
coordinateEntries: batchCoordinateRollback,
|
|
7109
|
+
documents: batchDocumentRollbacks,
|
|
7110
|
+
durableWrapper,
|
|
7111
|
+
});
|
|
7112
|
+
compensated = true;
|
|
7113
|
+
}
|
|
7114
|
+
catch {
|
|
7115
|
+
// close/reopen completes recovery if durable compensation failed
|
|
7116
|
+
}
|
|
7117
|
+
if (compensated) {
|
|
7118
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7119
|
+
}
|
|
7120
|
+
else {
|
|
7121
|
+
this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7122
|
+
}
|
|
7123
|
+
return this.failNativeDurableCommit(cause, {
|
|
7124
|
+
committedCids,
|
|
7125
|
+
failedCids: committedCids,
|
|
7126
|
+
});
|
|
7127
|
+
};
|
|
7128
|
+
if (durableWrapper &&
|
|
7129
|
+
nativeTrimmedHashes.length > 0 &&
|
|
7130
|
+
!durableWrapper.beginNativeDeleteCleanup) {
|
|
7131
|
+
return rollbackCommitted(new Error("Native durable block wrapper cannot preannounce trim cleanup"));
|
|
7132
|
+
}
|
|
7133
|
+
nativeDeleteCleanupToken =
|
|
7134
|
+
durableWrapper?.beginNativeDeleteCleanup?.(nativeTrimmedHashes);
|
|
7135
|
+
const preparedRows = committedAppends.map((append) => ({
|
|
7136
|
+
cid: append.entry.hash,
|
|
7137
|
+
hash: append.entry.hash,
|
|
7138
|
+
gid: append.coordinate.gid,
|
|
7139
|
+
next: append.entry.next,
|
|
7140
|
+
bytes: append.entry.bytes,
|
|
7141
|
+
byteLength: append.entry.byteLength,
|
|
7142
|
+
metaBytes: append.entry.metaBytes,
|
|
7143
|
+
hashDigestBytes: append.entry.hashDigestBytes,
|
|
7144
|
+
getBytes: (hash) => backbone.blocks.get(hash),
|
|
7145
|
+
nativeIndexMutationLockOwner: nativeStrictTransaction?.lowerHashMutationLockOwner,
|
|
7146
|
+
trimmedEntryHashes: append.trimmedHashes,
|
|
7147
|
+
nativeBlocksDeleted: true,
|
|
7148
|
+
nativeDeleteCleanupToken,
|
|
7149
|
+
documentTrimmedHeadsProcessed: append.documentTrimmedHeadsProcessed,
|
|
7150
|
+
documentPreviousContext: append.documentPreviousContext,
|
|
7151
|
+
}));
|
|
7152
|
+
if (!durableWrapper) {
|
|
7153
|
+
return preparedRows;
|
|
7154
|
+
}
|
|
7155
|
+
if (!durableWrapper.mirrorManyToDurable) {
|
|
7156
|
+
durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
7157
|
+
return rollbackCommitted(new Error("Native durable block wrapper has no batch mirror method"));
|
|
7158
|
+
}
|
|
7159
|
+
const durableMirrorBlocks = [];
|
|
7160
|
+
const missingCommittedCids = [];
|
|
7161
|
+
let missingCommittedHash = false;
|
|
7162
|
+
for (const backboneAppend of committedAppends) {
|
|
7163
|
+
const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
|
|
7164
|
+
if (!committedHash) {
|
|
7165
|
+
missingCommittedHash = true;
|
|
7166
|
+
continue;
|
|
7167
|
+
}
|
|
7168
|
+
// Earlier rows can be trimmed by later rows in this one native batch.
|
|
7169
|
+
// The native result retains their bytes even though the final hot map
|
|
7170
|
+
// no longer does; mirror those bytes, then let the explicit trim cleanup
|
|
7171
|
+
// remove the durable copy.
|
|
7172
|
+
const committedBytes = backboneAppend.entry.bytes ?? backbone.blocks.get(committedHash);
|
|
7173
|
+
if (!committedBytes) {
|
|
7174
|
+
missingCommittedCids.push(committedHash);
|
|
7175
|
+
continue;
|
|
7176
|
+
}
|
|
7177
|
+
durableMirrorBlocks.push([committedHash, committedBytes]);
|
|
7178
|
+
}
|
|
7179
|
+
// One strict putKnownMany WAL mutation gives the whole native batch one
|
|
7180
|
+
// durability barrier instead of issuing and fsyncing one record per row.
|
|
7181
|
+
const durableMirror = durableMirrorBlocks.length > 0
|
|
7182
|
+
? durableWrapper.mirrorManyToDurable(durableMirrorBlocks, {
|
|
7183
|
+
nativeTrimmedCids: nativeTrimmedHashSet,
|
|
7184
|
+
})
|
|
7185
|
+
: Promise.resolve();
|
|
7186
|
+
return Promise.allSettled([durableMirror]).then(async (settled) => {
|
|
7187
|
+
const rejected = settled[0]?.status === "rejected"
|
|
7188
|
+
? settled[0].reason
|
|
7189
|
+
: undefined;
|
|
7190
|
+
if (missingCommittedHash ||
|
|
7191
|
+
missingCommittedCids.length > 0 ||
|
|
7192
|
+
rejected !== undefined) {
|
|
7193
|
+
durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
7194
|
+
const cause = rejected === undefined
|
|
7195
|
+
? new Error(missingCommittedHash
|
|
7196
|
+
? "Native batch commit returned an entry with no CID to mirror"
|
|
7197
|
+
: `Native committed blocks are missing from the hot store: ${missingCommittedCids.join(", ")}`)
|
|
7198
|
+
: rejected;
|
|
7199
|
+
const rejectedCids = cause instanceof NativeDurableCommitError
|
|
7200
|
+
? cause.failedCids.filter((cid) => committedCids.includes(cid))
|
|
7201
|
+
: durableMirrorBlocks.map(([cid]) => cid);
|
|
7202
|
+
if (cause instanceof NativeDurableCommitError) {
|
|
7203
|
+
cause.addCommitContext({
|
|
7204
|
+
committedCids,
|
|
7205
|
+
failedCids: [...missingCommittedCids, ...rejectedCids],
|
|
7206
|
+
});
|
|
7207
|
+
}
|
|
7208
|
+
return rollbackCommitted(cause);
|
|
7209
|
+
}
|
|
7210
|
+
const nativeCommitOwnershipToken = settled[0]?.status === "fulfilled" ? settled[0].value : undefined;
|
|
7211
|
+
return preparedRows.map((row) => ({
|
|
7212
|
+
...row,
|
|
7213
|
+
nativeCommitOwnershipToken,
|
|
7214
|
+
}));
|
|
7215
|
+
});
|
|
7216
|
+
});
|
|
7217
|
+
}
|
|
7218
|
+
catch (error) {
|
|
7219
|
+
durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
7220
|
+
let compensated = !backboneAppends;
|
|
7221
|
+
if (backboneAppends && !(error instanceof NativeDurableCommitError)) {
|
|
7222
|
+
try {
|
|
7223
|
+
await this.rollbackFailedNativeBackboneTransaction({
|
|
7224
|
+
committedHashes: backboneAppends
|
|
7225
|
+
.map((append) => append.entry.cid ?? append.entry.hash)
|
|
7226
|
+
.filter((hash) => !!hash),
|
|
7227
|
+
coordinateEntries: batchCoordinateRollback,
|
|
7228
|
+
documents: batchDocumentRollbacks,
|
|
7229
|
+
skipBlockCompensation: true,
|
|
7230
|
+
restoreGraphFromIndex: true,
|
|
7231
|
+
});
|
|
7232
|
+
compensated = true;
|
|
7233
|
+
}
|
|
7234
|
+
catch {
|
|
7235
|
+
// Preserve the lower index publication failure.
|
|
7236
|
+
}
|
|
7237
|
+
}
|
|
7238
|
+
if (!(error instanceof NativeDurableCommitError)) {
|
|
7239
|
+
if (compensated) {
|
|
7240
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7241
|
+
}
|
|
7242
|
+
else {
|
|
7243
|
+
this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7244
|
+
}
|
|
7245
|
+
}
|
|
7246
|
+
throw error;
|
|
7247
|
+
}
|
|
5052
7248
|
if (!appended || !backboneAppends) {
|
|
7249
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
5053
7250
|
return undefined;
|
|
5054
7251
|
}
|
|
5055
|
-
const appendCommits = [];
|
|
5056
7252
|
const runtimeOnlyCoordinates = options?.replicate === false;
|
|
5057
|
-
|
|
5058
|
-
const
|
|
5059
|
-
|
|
5060
|
-
|
|
5061
|
-
|
|
5062
|
-
|
|
5063
|
-
|
|
5064
|
-
|
|
5065
|
-
|
|
5066
|
-
|
|
5067
|
-
|
|
5068
|
-
|
|
5069
|
-
|
|
5070
|
-
|
|
5071
|
-
|
|
7253
|
+
const rollbackBatch = async (error) => {
|
|
7254
|
+
const rollbackFailures = [];
|
|
7255
|
+
try {
|
|
7256
|
+
await this.markNativeStrictDurableTransactionRollback(nativeStrictTransaction);
|
|
7257
|
+
}
|
|
7258
|
+
catch (rollbackError) {
|
|
7259
|
+
const retentionFailures = this.retainNativeStrictDurableTransactionAfterMarkerFailure(nativeStrictTransaction, appended.nativeCommittedAppendFinalizer, rollbackError);
|
|
7260
|
+
throw new AggregateError([error, ...retentionFailures], "Native rollback marker could not be persisted; recovery is required");
|
|
7261
|
+
}
|
|
7262
|
+
try {
|
|
7263
|
+
await appended.nativeCommittedAppendFinalizer?.rollback();
|
|
7264
|
+
}
|
|
7265
|
+
catch (rollbackError) {
|
|
7266
|
+
rollbackFailures.push(rollbackError);
|
|
7267
|
+
}
|
|
7268
|
+
try {
|
|
7269
|
+
await this.rollbackNativeBackboneCoordinateAppendDurably(appended.appendFacts[0]?.hash ?? "", batchCoordinateRollback);
|
|
7270
|
+
}
|
|
7271
|
+
catch (rollbackError) {
|
|
7272
|
+
rollbackFailures.push(rollbackError);
|
|
7273
|
+
}
|
|
7274
|
+
try {
|
|
7275
|
+
for (const document of batchDocumentRollbacks) {
|
|
7276
|
+
this.restoreNativeBackboneDocument(document);
|
|
7277
|
+
}
|
|
7278
|
+
const flushed = this.flushNativeBackboneCoordinateJournal();
|
|
7279
|
+
if (isPromiseLike(flushed)) {
|
|
7280
|
+
await flushed;
|
|
7281
|
+
}
|
|
7282
|
+
}
|
|
7283
|
+
catch (rollbackError) {
|
|
7284
|
+
rollbackFailures.push(rollbackError);
|
|
7285
|
+
}
|
|
7286
|
+
if (rollbackFailures.length > 0) {
|
|
7287
|
+
this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7288
|
+
throw new AggregateError([error, ...rollbackFailures], "Shared-log append batch and compensation both failed");
|
|
7289
|
+
}
|
|
7290
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7291
|
+
throw error;
|
|
7292
|
+
};
|
|
7293
|
+
const coordinateRows = [];
|
|
7294
|
+
try {
|
|
7295
|
+
const batchExternalNextHashes = new Set(appended.appendFacts.flatMap((facts) => facts.next));
|
|
7296
|
+
await this.setNativeStrictDurableTransactionExpectedRows(nativeStrictTransaction, appended.appendFacts.map((facts) => new ShallowEntry({
|
|
7297
|
+
hash: facts.hash,
|
|
7298
|
+
payloadSize: facts.payloadSize,
|
|
7299
|
+
head: !batchExternalNextHashes.has(facts.hash),
|
|
7300
|
+
meta: new ShallowMeta({
|
|
7301
|
+
gid: facts.gid,
|
|
7302
|
+
clock: new LamportClock({
|
|
7303
|
+
id: facts.clockId ?? this.node.identity.publicKey.bytes,
|
|
7304
|
+
timestamp: new Timestamp({
|
|
7305
|
+
wallTime: facts.wallTime,
|
|
7306
|
+
logical: facts.logical,
|
|
7307
|
+
}),
|
|
7308
|
+
}),
|
|
7309
|
+
data: facts.metaData,
|
|
7310
|
+
next: facts.next,
|
|
7311
|
+
type: facts.type ?? EntryType.APPEND,
|
|
7312
|
+
}),
|
|
7313
|
+
})));
|
|
7314
|
+
for (let i = 0; i < appended.appendFacts.length; i++) {
|
|
7315
|
+
const facts = appended.appendFacts[i];
|
|
7316
|
+
const backboneAppend = backboneAppends[i];
|
|
7317
|
+
const coordinateFields = this.createCoordinateFieldsFromNativePlanFacts({
|
|
7318
|
+
appendFacts: facts,
|
|
7319
|
+
plan: backboneAppend.coordinate,
|
|
7320
|
+
});
|
|
7321
|
+
if (!coordinateFields) {
|
|
7322
|
+
throw new Error("Native backbone batch append transaction returned mismatched coordinate facts");
|
|
7323
|
+
}
|
|
7324
|
+
const plannedCoordinateDeleteHashes = combineCoordinateDeleteHashes(facts.next, backboneAppend.trimmedHashes ?? []);
|
|
7325
|
+
coordinateRows.push({
|
|
7326
|
+
facts,
|
|
7327
|
+
backboneAppend,
|
|
7328
|
+
coordinateFields,
|
|
7329
|
+
plannedCoordinateDeleteHashes,
|
|
7330
|
+
});
|
|
7331
|
+
const persisted = this.persistBackboneCoordinateFieldsNativeTransaction({
|
|
7332
|
+
coordinateIndex: this.entryCoordinatesIndex,
|
|
7333
|
+
fields: coordinateFields,
|
|
7334
|
+
hash: facts.hash,
|
|
7335
|
+
deleteHashes: [],
|
|
7336
|
+
coordinates: backboneAppend.coordinate
|
|
7337
|
+
.coordinates,
|
|
7338
|
+
skipGenericTransientCoordinateIndex: runtimeOnlyCoordinates,
|
|
7339
|
+
});
|
|
7340
|
+
if (isPromiseLike(persisted)) {
|
|
7341
|
+
await persisted;
|
|
5072
7342
|
}
|
|
5073
7343
|
}
|
|
5074
|
-
|
|
5075
|
-
|
|
5076
|
-
plan: backboneAppend.coordinate,
|
|
5077
|
-
});
|
|
5078
|
-
if (!coordinateFields) {
|
|
5079
|
-
throw new Error("Native backbone batch append transaction returned mismatched coordinate facts");
|
|
7344
|
+
if (!appended.nativeCommittedAppendFinalizer) {
|
|
7345
|
+
throw new Error("Missing deferred native append batch finalizer");
|
|
5080
7346
|
}
|
|
5081
|
-
|
|
7347
|
+
await this.flushNativeBackboneCoordinateJournal();
|
|
7348
|
+
await appended.nativeCommittedAppendFinalizer.acknowledge(() => this.markNativeStrictDurableTransactionLowerMarker(nativeStrictTransaction));
|
|
7349
|
+
}
|
|
7350
|
+
catch (error) {
|
|
7351
|
+
return rollbackBatch(error);
|
|
7352
|
+
}
|
|
7353
|
+
const appendCommits = [];
|
|
7354
|
+
for (let i = 0; i < coordinateRows.length; i++) {
|
|
7355
|
+
const { facts, backboneAppend, coordinateFields, plannedCoordinateDeleteHashes, } = coordinateRows[i];
|
|
5082
7356
|
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(facts, [], appended.materializeEntries[i], {
|
|
5083
7357
|
forgetNativeCoordinates: false,
|
|
5084
7358
|
removedHashes: plannedCoordinateDeleteHashes,
|
|
5085
7359
|
});
|
|
5086
|
-
const persisted = this.persistBackboneCoordinateFieldsNativeTransaction({
|
|
5087
|
-
coordinateIndex: this.entryCoordinatesIndex,
|
|
5088
|
-
fields: coordinateFields,
|
|
5089
|
-
hash: facts.hash,
|
|
5090
|
-
deleteHashes: plannedCoordinateDeleteHashes,
|
|
5091
|
-
coordinates: backboneAppend.coordinate.coordinates,
|
|
5092
|
-
skipGenericTransientCoordinateIndex: runtimeOnlyCoordinates,
|
|
5093
|
-
});
|
|
5094
|
-
if (isPromiseLike(persisted)) {
|
|
5095
|
-
await persisted;
|
|
5096
|
-
}
|
|
5097
7360
|
if (!runtimeOnlyCoordinates && this.remoteBlocks.hasNotifyStoredHook()) {
|
|
5098
7361
|
this.remoteBlocks.notifyStoredDeferred(facts.hash);
|
|
5099
7362
|
}
|
|
5100
7363
|
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(facts, {
|
|
5101
|
-
hashNumber: backboneAppend.coordinate
|
|
5102
|
-
.hashNumber,
|
|
7364
|
+
hashNumber: backboneAppend.coordinate.hashNumber,
|
|
5103
7365
|
coordinateFields,
|
|
5104
7366
|
});
|
|
5105
7367
|
appendCommit.nativeBackboneDocumentIndexCommitted = true;
|
|
@@ -5109,6 +7371,12 @@ let SharedLog = (() => {
|
|
|
5109
7371
|
backboneAppend.documentPreviousContext;
|
|
5110
7372
|
appendCommits.push(appendCommit);
|
|
5111
7373
|
}
|
|
7374
|
+
try {
|
|
7375
|
+
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7376
|
+
}
|
|
7377
|
+
catch (error) {
|
|
7378
|
+
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
7379
|
+
}
|
|
5112
7380
|
const delayAdaptiveRebalance = this.shouldDelayAdaptiveRebalance();
|
|
5113
7381
|
if (!delayAdaptiveRebalance) {
|
|
5114
7382
|
this.rebalanceParticipationDebounced?.call();
|
|
@@ -5123,6 +7391,7 @@ let SharedLog = (() => {
|
|
|
5123
7391
|
};
|
|
5124
7392
|
}
|
|
5125
7393
|
async appendLocallyPreparedManyIndependent(data, options, properties) {
|
|
7394
|
+
this.throwIfNativeDurableCommitFailed();
|
|
5126
7395
|
if (data.length === 0) {
|
|
5127
7396
|
return { entries: [], removed: [], appendCommits: [] };
|
|
5128
7397
|
}
|
|
@@ -5204,6 +7473,7 @@ let SharedLog = (() => {
|
|
|
5204
7473
|
});
|
|
5205
7474
|
}
|
|
5206
7475
|
async appendMany(data, options) {
|
|
7476
|
+
this.throwIfNativeDurableCommitFailed();
|
|
5207
7477
|
if (data.length === 0) {
|
|
5208
7478
|
return { entries: [], removed: [] };
|
|
5209
7479
|
}
|
|
@@ -5838,6 +8108,9 @@ let SharedLog = (() => {
|
|
|
5838
8108
|
return this.createPreparedLocalAppendCommits(entries, nativeAppendPlans);
|
|
5839
8109
|
}
|
|
5840
8110
|
async open(options) {
|
|
8111
|
+
this.ensureNativeDurabilityRuntimeState();
|
|
8112
|
+
this._nativeStrictDurableTransactionsClosing = false;
|
|
8113
|
+
const recoveringNativeDurableFailure = this._nativeDurableCommitFailure !== undefined;
|
|
5841
8114
|
options = applySharedLogNativeDefaults(options, this.node
|
|
5842
8115
|
.sharedLogNativeDefaults);
|
|
5843
8116
|
this.replicas = {
|
|
@@ -5900,6 +8173,8 @@ let SharedLog = (() => {
|
|
|
5900
8173
|
this._replicatorLivenessFailures = new Map();
|
|
5901
8174
|
this._replicatorLastActivityAt = new Map();
|
|
5902
8175
|
this._lastLocalAppendAt = 0;
|
|
8176
|
+
this._replicationAnnouncementRetryPending = false;
|
|
8177
|
+
this._replicationAnnouncementRetryGeneration = 0;
|
|
5903
8178
|
const adaptiveReplicateOptions = options?.replicate && isAdaptiveReplicatorOption(options.replicate)
|
|
5904
8179
|
? options.replicate
|
|
5905
8180
|
: undefined;
|
|
@@ -5937,6 +8212,7 @@ let SharedLog = (() => {
|
|
|
5937
8212
|
throw new Error("waitForReplicatorRequestMaxAttempts must be a positive number");
|
|
5938
8213
|
}
|
|
5939
8214
|
this._closeController = new AbortController();
|
|
8215
|
+
this.setupReplicationAnnouncementRetryFunction();
|
|
5940
8216
|
this._closeController.signal.addEventListener("abort", () => {
|
|
5941
8217
|
for (const [_peer, state] of this._replicationInfoRequestByPeer) {
|
|
5942
8218
|
if (state.timer)
|
|
@@ -6002,7 +8278,7 @@ let SharedLog = (() => {
|
|
|
6002
8278
|
let localBlocks;
|
|
6003
8279
|
if (this._nativeBackbone) {
|
|
6004
8280
|
if (this.node.directory != null) {
|
|
6005
|
-
const durable =
|
|
8281
|
+
const durable = await createNativeDurableBlockStore(storage);
|
|
6006
8282
|
localBlocks = new NativeBackboneWriteThroughBlockStore(this._nativeBackbone.blocks, durable);
|
|
6007
8283
|
}
|
|
6008
8284
|
else {
|
|
@@ -6010,7 +8286,7 @@ let SharedLog = (() => {
|
|
|
6010
8286
|
}
|
|
6011
8287
|
}
|
|
6012
8288
|
else {
|
|
6013
|
-
localBlocks =
|
|
8289
|
+
localBlocks = await createDefaultDurableBlockStore(storage);
|
|
6014
8290
|
}
|
|
6015
8291
|
this.remoteBlocks = new RemoteBlocks({
|
|
6016
8292
|
local: localBlocks,
|
|
@@ -6160,6 +8436,10 @@ let SharedLog = (() => {
|
|
|
6160
8436
|
};
|
|
6161
8437
|
}, PRUNE_DEBOUNCE_INTERVAL);
|
|
6162
8438
|
await remoteBlocksStartPromise;
|
|
8439
|
+
// Failed native prepares can leave content-addressed bytes behind. Recovery
|
|
8440
|
+
// deliberately preserves them: the reopened lower log is the liveness
|
|
8441
|
+
// authority, while these unreachable bytes are safer than deleting a CID that
|
|
8442
|
+
// may also belong to an acknowledged, restored, or concurrent operation.
|
|
6163
8443
|
const useNativeBackboneBlocks = this._nativeBackbone && this._logProperties?.replicate === false;
|
|
6164
8444
|
const nativeBackboneGraph = this._nativeBackbone
|
|
6165
8445
|
? useNativeBackboneBlocks
|
|
@@ -6202,6 +8482,29 @@ let SharedLog = (() => {
|
|
|
6202
8482
|
},
|
|
6203
8483
|
indexer: logIndex,
|
|
6204
8484
|
});
|
|
8485
|
+
try {
|
|
8486
|
+
const recovered = await this.recoverNativeStrictDurableTransactionIntent();
|
|
8487
|
+
if (recovered) {
|
|
8488
|
+
await this.reconcileNativeCoordinatesWithLowerCommitMarkers();
|
|
8489
|
+
}
|
|
8490
|
+
}
|
|
8491
|
+
catch (error) {
|
|
8492
|
+
this.poisonNativeStrictDurableTransaction(error);
|
|
8493
|
+
throw error;
|
|
8494
|
+
}
|
|
8495
|
+
// A fresh wrapper alone is not proof of recovery. Clear the cached poison
|
|
8496
|
+
// only after the failed native transaction was compensated (or its pending
|
|
8497
|
+
// native journals were deliberately discarded during close) and the lower log
|
|
8498
|
+
// reopened successfully. Unreferenced content-addressed bytes are preserved;
|
|
8499
|
+
// the reopened lower-log facts, not block presence, determine liveness.
|
|
8500
|
+
if (localBlocks instanceof NativeBackboneWriteThroughBlockStore &&
|
|
8501
|
+
!localBlocks.getNativeDurableCommitFailure() &&
|
|
8502
|
+
(!recoveringNativeDurableFailure ||
|
|
8503
|
+
this._nativeDurableRecoveryReadyForReopen)) {
|
|
8504
|
+
this._nativeDurableCommitFailure = undefined;
|
|
8505
|
+
this._nativeDurableRecoveryReadyForReopen = false;
|
|
8506
|
+
this._nativeDurableRecoveryCids.clear();
|
|
8507
|
+
}
|
|
6205
8508
|
const resolveHashesForSymbols = (symbols) => {
|
|
6206
8509
|
const nativeState = this._nativeBackbone ?? this._nativeSharedLogState;
|
|
6207
8510
|
if (!nativeState) {
|
|
@@ -6410,6 +8713,10 @@ let SharedLog = (() => {
|
|
|
6410
8713
|
await rangeIterator.close();
|
|
6411
8714
|
}
|
|
6412
8715
|
if (this._nativeBackboneCoordinatePersistence) {
|
|
8716
|
+
// A previous explicit drop may have been interrupted after its durable
|
|
8717
|
+
// tombstone was written. Complete that erase before the adapter can expose
|
|
8718
|
+
// any stale coordinate or document state to this backbone.
|
|
8719
|
+
await this._nativeBackboneCoordinatePersistence.resumeDrop?.();
|
|
6413
8720
|
await this._nativeBackboneCoordinatePersistence.hydrate(backbone);
|
|
6414
8721
|
this._nativeBackboneCoordinateJournalLastFlushMs = Date.now();
|
|
6415
8722
|
this.hydrateNativeCoordinateStateFromBackbone(backbone);
|
|
@@ -6437,6 +8744,58 @@ let SharedLog = (() => {
|
|
|
6437
8744
|
await iterator.close();
|
|
6438
8745
|
}
|
|
6439
8746
|
}
|
|
8747
|
+
async reconcileNativeCoordinatesWithLowerCommitMarkers() {
|
|
8748
|
+
if (!this._nativeBackbone) {
|
|
8749
|
+
return;
|
|
8750
|
+
}
|
|
8751
|
+
const hashes = new Set(this._residentEntryCoordinatesByHash?.keys() ?? []);
|
|
8752
|
+
const iterator = this.entryCoordinatesIndex.iterate({});
|
|
8753
|
+
try {
|
|
8754
|
+
for (;;) {
|
|
8755
|
+
const batch = await iterator.next(256);
|
|
8756
|
+
if (batch.length === 0) {
|
|
8757
|
+
break;
|
|
8758
|
+
}
|
|
8759
|
+
for (const result of batch) {
|
|
8760
|
+
hashes.add(result.value.hash);
|
|
8761
|
+
}
|
|
8762
|
+
}
|
|
8763
|
+
}
|
|
8764
|
+
finally {
|
|
8765
|
+
await iterator.close();
|
|
8766
|
+
}
|
|
8767
|
+
if (hashes.size === 0) {
|
|
8768
|
+
return;
|
|
8769
|
+
}
|
|
8770
|
+
const committed = await this.log.entryIndex.hasMany(hashes);
|
|
8771
|
+
const orphaned = [...hashes].filter((hash) => !committed.has(hash));
|
|
8772
|
+
if (orphaned.length === 0) {
|
|
8773
|
+
return;
|
|
8774
|
+
}
|
|
8775
|
+
const coordinateIndex = this.entryCoordinatesIndex;
|
|
8776
|
+
if (coordinateIndex.delIdsNoReturn) {
|
|
8777
|
+
await coordinateIndex.delIdsNoReturn(orphaned);
|
|
8778
|
+
}
|
|
8779
|
+
else if (coordinateIndex.delIds) {
|
|
8780
|
+
await coordinateIndex.delIds(orphaned);
|
|
8781
|
+
}
|
|
8782
|
+
else {
|
|
8783
|
+
await coordinateIndex.del({
|
|
8784
|
+
query: orphaned.length === 1
|
|
8785
|
+
? { hash: orphaned[0] }
|
|
8786
|
+
: new Or(orphaned.map((hash) => new StringMatch({ key: "hash", value: hash }))),
|
|
8787
|
+
});
|
|
8788
|
+
}
|
|
8789
|
+
for (const hash of orphaned) {
|
|
8790
|
+
this._nativeBackbone.deleteEntryCoordinates(hash);
|
|
8791
|
+
this._nativeSharedLogState?.deleteEntryCoordinates(hash);
|
|
8792
|
+
this._residentEntryCoordinatesByHash?.delete(hash);
|
|
8793
|
+
}
|
|
8794
|
+
const flushed = this.flushNativeBackboneCoordinateJournal();
|
|
8795
|
+
if (isPromiseLike(flushed)) {
|
|
8796
|
+
await flushed;
|
|
8797
|
+
}
|
|
8798
|
+
}
|
|
6440
8799
|
hydrateNativeCoordinateStateFromBackbone(backbone) {
|
|
6441
8800
|
const fields = backbone.getEntryCoordinateFields();
|
|
6442
8801
|
this._nativeSharedLogState?.clearEntryCoordinates();
|
|
@@ -6511,7 +8870,10 @@ let SharedLog = (() => {
|
|
|
6511
8870
|
}
|
|
6512
8871
|
async openNativeBackbone(options) {
|
|
6513
8872
|
this._nativeBackboneCoordinatePersistence = undefined;
|
|
8873
|
+
this._nativeBackboneCoordinatePersistenceStore = undefined;
|
|
8874
|
+
this._nativeBackboneDropStarted = false;
|
|
6514
8875
|
this._nativeBackboneCoordinateJournalLastFlushMs = 0;
|
|
8876
|
+
this._nativeStrictDurableTransactionJournalState = undefined;
|
|
6515
8877
|
if (!options) {
|
|
6516
8878
|
return undefined;
|
|
6517
8879
|
}
|
|
@@ -6548,6 +8910,17 @@ let SharedLog = (() => {
|
|
|
6548
8910
|
// peer to re-derive from. Memory-only nodes (no directory) keep the
|
|
6549
8911
|
// previous in-memory behavior.
|
|
6550
8912
|
if (options.coordinatePersistence) {
|
|
8913
|
+
if ("store" in options.coordinatePersistence) {
|
|
8914
|
+
this._nativeBackboneCoordinatePersistenceStore =
|
|
8915
|
+
options.coordinatePersistence.store;
|
|
8916
|
+
}
|
|
8917
|
+
else if (options.coordinatePersistence.intentStore) {
|
|
8918
|
+
this._nativeBackboneCoordinatePersistenceStore =
|
|
8919
|
+
options.coordinatePersistence.intentStore;
|
|
8920
|
+
}
|
|
8921
|
+
else if (this.node.directory != null) {
|
|
8922
|
+
throw new Error("Durable nativeBackbone.coordinatePersistence adapters must expose intentStore");
|
|
8923
|
+
}
|
|
6551
8924
|
this._nativeBackboneCoordinatePersistence =
|
|
6552
8925
|
createNativeBackboneCoordinatePersistence(options.coordinatePersistence);
|
|
6553
8926
|
}
|
|
@@ -6555,6 +8928,24 @@ let SharedLog = (() => {
|
|
|
6555
8928
|
this._nativeBackboneCoordinatePersistence =
|
|
6556
8929
|
await this.createAutoDerivedCoordinatePersistence(nativeBackboneModule);
|
|
6557
8930
|
}
|
|
8931
|
+
if (this.node.directory != null &&
|
|
8932
|
+
this._nativeBackboneCoordinatePersistence) {
|
|
8933
|
+
if (this._nativeBackboneCoordinatePersistence.durableBarrier !== true ||
|
|
8934
|
+
typeof this._nativeBackboneCoordinatePersistenceStore
|
|
8935
|
+
?.durableBarrier !== "function") {
|
|
8936
|
+
throw new Error("Durable nativeBackbone coordinate persistence requires an explicit physical durability barrier");
|
|
8937
|
+
}
|
|
8938
|
+
}
|
|
8939
|
+
if (this._nativeBackboneCoordinatePersistence &&
|
|
8940
|
+
(this._nativeBackboneCoordinatePersistence.compactMaxJournalBytes !=
|
|
8941
|
+
null ||
|
|
8942
|
+
this._nativeBackboneCoordinatePersistence.compactMaxJournalRecords !=
|
|
8943
|
+
null) &&
|
|
8944
|
+
this._nativeBackboneCoordinatePersistence.crashSafeCompaction !== true) {
|
|
8945
|
+
// Durable custom adapters must explicitly advertise an atomic generation
|
|
8946
|
+
// protocol before SharedLog permits automatic WAL compaction.
|
|
8947
|
+
throw new Error("Durable native coordinate persistence compaction thresholds require crashSafeCompaction");
|
|
8948
|
+
}
|
|
6558
8949
|
return backbone;
|
|
6559
8950
|
}
|
|
6560
8951
|
catch (error) {
|
|
@@ -6605,6 +8996,7 @@ let SharedLog = (() => {
|
|
|
6605
8996
|
directory: ["coordinates", namespace],
|
|
6606
8997
|
});
|
|
6607
8998
|
}
|
|
8999
|
+
this._nativeBackboneCoordinatePersistenceStore = store;
|
|
6608
9000
|
return createNativeBackboneCoordinatePersistence({
|
|
6609
9001
|
store,
|
|
6610
9002
|
buffered: true,
|
|
@@ -7563,115 +9955,167 @@ let SharedLog = (() => {
|
|
|
7563
9955
|
throw error;
|
|
7564
9956
|
}
|
|
7565
9957
|
}
|
|
7566
|
-
async _close() {
|
|
7567
|
-
|
|
7568
|
-
|
|
7569
|
-
|
|
7570
|
-
|
|
7571
|
-
|
|
7572
|
-
this._wireSyncSession = undefined;
|
|
7573
|
-
}
|
|
7574
|
-
await this.closeNativeBackboneCoordinatePersistence();
|
|
7575
|
-
await this.syncronizer?.close();
|
|
7576
|
-
for (const [_key, peerMap] of this.pendingMaturity ?? []) {
|
|
7577
|
-
for (const [_key2, info] of peerMap) {
|
|
7578
|
-
clearTimeout(info.timeout);
|
|
9958
|
+
async _close(options) {
|
|
9959
|
+
const preserveDropRetryResources = options?.preserveDropRetryResources === true;
|
|
9960
|
+
let firstError;
|
|
9961
|
+
const capture = async (operation) => {
|
|
9962
|
+
try {
|
|
9963
|
+
await operation();
|
|
7579
9964
|
}
|
|
7580
|
-
|
|
7581
|
-
|
|
7582
|
-
|
|
7583
|
-
|
|
7584
|
-
|
|
7585
|
-
|
|
7586
|
-
|
|
7587
|
-
|
|
7588
|
-
|
|
7589
|
-
|
|
9965
|
+
catch (error) {
|
|
9966
|
+
firstError ??= error;
|
|
9967
|
+
}
|
|
9968
|
+
};
|
|
9969
|
+
const captureSync = (operation) => {
|
|
9970
|
+
try {
|
|
9971
|
+
operation();
|
|
9972
|
+
}
|
|
9973
|
+
catch (error) {
|
|
9974
|
+
firstError ??= error;
|
|
9975
|
+
}
|
|
9976
|
+
};
|
|
9977
|
+
captureSync(() => this.cancelCurrentReplicationStateAnnouncementRetry());
|
|
9978
|
+
this.replicationAnnouncementRetryDebounced = undefined;
|
|
9979
|
+
captureSync(() => {
|
|
9980
|
+
if (this._wireSyncSession) {
|
|
9981
|
+
this._wireSyncSession.unregisterTopic(this.topic);
|
|
9982
|
+
this._wireSyncSession = undefined;
|
|
9983
|
+
}
|
|
9984
|
+
});
|
|
9985
|
+
await capture(() => this.closeNativeBackboneCoordinatePersistence());
|
|
9986
|
+
await capture(() => this.syncronizer?.close());
|
|
9987
|
+
captureSync(() => {
|
|
9988
|
+
for (const [_key, peerMap] of this.pendingMaturity ?? []) {
|
|
9989
|
+
for (const [_key2, info] of peerMap)
|
|
9990
|
+
clearTimeout(info.timeout);
|
|
9991
|
+
}
|
|
9992
|
+
this.pendingMaturity?.clear();
|
|
9993
|
+
this.distributeQueue?.clear();
|
|
9994
|
+
});
|
|
9995
|
+
captureSync(() => this._closeFanoutChannel());
|
|
9996
|
+
captureSync(() => this._providerHandle?.close());
|
|
7590
9997
|
this._providerHandle = undefined;
|
|
7591
|
-
|
|
7592
|
-
|
|
7593
|
-
|
|
7594
|
-
|
|
7595
|
-
|
|
7596
|
-
|
|
7597
|
-
|
|
7598
|
-
|
|
7599
|
-
|
|
7600
|
-
|
|
7601
|
-
|
|
7602
|
-
|
|
7603
|
-
|
|
7604
|
-
|
|
7605
|
-
|
|
7606
|
-
|
|
7607
|
-
|
|
7608
|
-
|
|
7609
|
-
|
|
7610
|
-
|
|
7611
|
-
|
|
7612
|
-
|
|
7613
|
-
|
|
7614
|
-
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
7618
|
-
|
|
7619
|
-
|
|
7620
|
-
|
|
7621
|
-
|
|
7622
|
-
|
|
7623
|
-
|
|
7624
|
-
|
|
7625
|
-
|
|
7626
|
-
targets.
|
|
7627
|
-
|
|
7628
|
-
|
|
7629
|
-
|
|
7630
|
-
|
|
7631
|
-
|
|
7632
|
-
|
|
7633
|
-
|
|
7634
|
-
this.
|
|
7635
|
-
|
|
7636
|
-
|
|
7637
|
-
|
|
7638
|
-
|
|
7639
|
-
|
|
7640
|
-
|
|
7641
|
-
|
|
7642
|
-
|
|
7643
|
-
|
|
7644
|
-
|
|
7645
|
-
|
|
7646
|
-
|
|
7647
|
-
|
|
7648
|
-
|
|
7649
|
-
|
|
7650
|
-
|
|
7651
|
-
|
|
7652
|
-
|
|
7653
|
-
|
|
7654
|
-
|
|
7655
|
-
//
|
|
7656
|
-
|
|
7657
|
-
this.
|
|
7658
|
-
this.
|
|
7659
|
-
this.
|
|
9998
|
+
captureSync(() => {
|
|
9999
|
+
this.coordinateToHash?.clear();
|
|
10000
|
+
this.recentlyRebalanced?.clear();
|
|
10001
|
+
this.uniqueReplicators?.clear();
|
|
10002
|
+
this._topicSubscribersCache?.clear();
|
|
10003
|
+
this._closeController.abort();
|
|
10004
|
+
clearInterval(this.interval);
|
|
10005
|
+
this.stopReplicatorLivenessSweep();
|
|
10006
|
+
});
|
|
10007
|
+
captureSync(() => this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn));
|
|
10008
|
+
captureSync(() => this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn));
|
|
10009
|
+
captureSync(() => {
|
|
10010
|
+
for (const timer of this._repairRetryTimers ?? [])
|
|
10011
|
+
clearTimeout(timer);
|
|
10012
|
+
this._repairRetryTimers?.clear();
|
|
10013
|
+
this._recentRepairDispatch?.clear();
|
|
10014
|
+
this._repairSweepRunning = false;
|
|
10015
|
+
this._repairSweepPendingModes?.clear();
|
|
10016
|
+
for (const peers of this._repairSweepPendingPeersByMode?.values() ?? [])
|
|
10017
|
+
peers.clear();
|
|
10018
|
+
this._repairSweepOptimisticGidPeersPending?.clear();
|
|
10019
|
+
this._entryKnownPeers?.clear();
|
|
10020
|
+
this._entryKnownPeerObservedAt?.clear();
|
|
10021
|
+
this._nativeSharedLogState?.clearEntryKnownPeers();
|
|
10022
|
+
this._nativeBackbone?.clearEntryKnownPeers();
|
|
10023
|
+
for (const timer of this._joinAuthoritativeRepairTimersByDelay?.values() ??
|
|
10024
|
+
[])
|
|
10025
|
+
clearTimeout(timer);
|
|
10026
|
+
this._joinAuthoritativeRepairTimersByDelay?.clear();
|
|
10027
|
+
this._joinAuthoritativeRepairPeersByDelay?.clear();
|
|
10028
|
+
for (const targets of this._repairFrontierByMode?.values() ?? [])
|
|
10029
|
+
targets.clear();
|
|
10030
|
+
for (const targets of this._repairFrontierActiveTargetsByMode?.values() ??
|
|
10031
|
+
[])
|
|
10032
|
+
targets.clear();
|
|
10033
|
+
for (const targets of this._repairFrontierBypassKnownPeersByMode?.values() ??
|
|
10034
|
+
[])
|
|
10035
|
+
targets.clear();
|
|
10036
|
+
if (this._appendBackfillTimer) {
|
|
10037
|
+
clearTimeout(this._appendBackfillTimer);
|
|
10038
|
+
this._appendBackfillTimer = undefined;
|
|
10039
|
+
}
|
|
10040
|
+
this._appendBackfillPendingByTarget?.clear();
|
|
10041
|
+
for (const [_key, value] of this._pendingIHave ?? [])
|
|
10042
|
+
value.clear();
|
|
10043
|
+
if (this._pendingIHaveExpiryTimer) {
|
|
10044
|
+
clearTimeout(this._pendingIHaveExpiryTimer);
|
|
10045
|
+
this._pendingIHaveExpiryTimer = undefined;
|
|
10046
|
+
this._pendingIHaveExpiryDeadline = Number.POSITIVE_INFINITY;
|
|
10047
|
+
}
|
|
10048
|
+
});
|
|
10049
|
+
captureSync(() => this._checkedPrune.close());
|
|
10050
|
+
if (!preserveDropRetryResources) {
|
|
10051
|
+
await capture(() => this.remoteBlocks?.stop?.());
|
|
10052
|
+
}
|
|
10053
|
+
captureSync(() => {
|
|
10054
|
+
this._pendingIHave?.clear();
|
|
10055
|
+
this.latestReplicationInfoMessage?.clear();
|
|
10056
|
+
this._gidPeersHistory?.clear();
|
|
10057
|
+
this._peerSyncCapabilities?.clear();
|
|
10058
|
+
this._liveRawGossipBatches?.clear();
|
|
10059
|
+
this._nativeSharedLogState?.clearGidPeers();
|
|
10060
|
+
this._nativeBackbone?.clearGidPeers();
|
|
10061
|
+
});
|
|
10062
|
+
// Cancel every debounce independently so one faulty close hook cannot keep
|
|
10063
|
+
// the remaining timers or indexes alive.
|
|
10064
|
+
captureSync(() => this.rebalanceParticipationDebounced?.close());
|
|
10065
|
+
captureSync(() => this.replicationChangeDebounceFn?.close?.());
|
|
10066
|
+
captureSync(() => this.pruneDebouncedFn?.close?.());
|
|
10067
|
+
captureSync(() => this.responseToPruneDebouncedFn?.close?.());
|
|
7660
10068
|
this.pruneDebouncedFn = undefined;
|
|
7661
10069
|
this.rebalanceParticipationDebounced = undefined;
|
|
7662
|
-
|
|
7663
|
-
|
|
7664
|
-
|
|
7665
|
-
|
|
7666
|
-
|
|
7667
|
-
|
|
10070
|
+
if (!preserveDropRetryResources) {
|
|
10071
|
+
const stopIndex = async (index, forget) => {
|
|
10072
|
+
if (!index) {
|
|
10073
|
+
return;
|
|
10074
|
+
}
|
|
10075
|
+
try {
|
|
10076
|
+
await index.stop?.();
|
|
10077
|
+
forget();
|
|
10078
|
+
}
|
|
10079
|
+
catch (error) {
|
|
10080
|
+
firstError ??= error;
|
|
10081
|
+
}
|
|
10082
|
+
};
|
|
10083
|
+
await stopIndex(this._replicationRangeIndex, () => {
|
|
10084
|
+
this._replicationRangeIndex = undefined;
|
|
10085
|
+
});
|
|
10086
|
+
await stopIndex(this._entryCoordinatesIndex, () => {
|
|
10087
|
+
this._entryCoordinatesIndex = undefined;
|
|
10088
|
+
});
|
|
10089
|
+
}
|
|
7668
10090
|
this._nativeRangePlanner = undefined;
|
|
7669
10091
|
this._nativeSharedLogState = undefined;
|
|
7670
10092
|
this._residentEntryCoordinatesByHash = undefined;
|
|
7671
|
-
this.cpuUsage?.stop?.();
|
|
7672
|
-
|
|
10093
|
+
captureSync(() => this.cpuUsage?.stop?.());
|
|
10094
|
+
if (firstError !== undefined) {
|
|
10095
|
+
throw firstError;
|
|
10096
|
+
}
|
|
10097
|
+
}
|
|
10098
|
+
classifyTerminalOwnership(from) {
|
|
10099
|
+
if (this.closed) {
|
|
10100
|
+
return "terminal";
|
|
10101
|
+
}
|
|
10102
|
+
const parentIndex = this.parents?.findIndex((parent) => parent === from) ?? -1;
|
|
10103
|
+
if (from && parentIndex === -1) {
|
|
10104
|
+
throw new TerminalOperationNotStartedError("Could not find from in parents");
|
|
10105
|
+
}
|
|
10106
|
+
return parentIndex !== -1 && (this.parents?.length ?? 0) > 1
|
|
10107
|
+
? "nonterminal"
|
|
10108
|
+
: "terminal";
|
|
7673
10109
|
}
|
|
7674
10110
|
async close(from) {
|
|
10111
|
+
if (this.classifyTerminalOwnership(from) === "nonterminal") {
|
|
10112
|
+
return super.close(from);
|
|
10113
|
+
}
|
|
10114
|
+
// Match Program.end()'s synchronous terminal admission fence before any
|
|
10115
|
+
// SharedLog-specific await or observable teardown can admit a new owner.
|
|
10116
|
+
this.preventParentAttachments();
|
|
10117
|
+
this.ensureNativeDurabilityRuntimeState();
|
|
10118
|
+
this.cancelCurrentReplicationStateAnnouncementRetry();
|
|
7675
10119
|
// Best-effort: announce that we are going offline before tearing down
|
|
7676
10120
|
// RPC/subscription state.
|
|
7677
10121
|
//
|
|
@@ -7724,15 +10168,71 @@ let SharedLog = (() => {
|
|
|
7724
10168
|
catch {
|
|
7725
10169
|
// ignore: close should be resilient even if we were never fully started
|
|
7726
10170
|
}
|
|
7727
|
-
|
|
7728
|
-
|
|
7729
|
-
|
|
10171
|
+
let firstError;
|
|
10172
|
+
let superClosed = false;
|
|
10173
|
+
try {
|
|
10174
|
+
superClosed = await super.close(from);
|
|
10175
|
+
}
|
|
10176
|
+
catch (error) {
|
|
10177
|
+
if (!this.closed || this.pendingTerminalOperation !== "close") {
|
|
10178
|
+
// Child/base admission failed before Program committed this terminal
|
|
10179
|
+
// transition (including a cleanly closed instance). Lower resources are
|
|
10180
|
+
// still live data or belong to a completed generation, so do not mutate
|
|
10181
|
+
// them while merely propagating the base error.
|
|
10182
|
+
throw error;
|
|
10183
|
+
}
|
|
10184
|
+
firstError = error;
|
|
10185
|
+
}
|
|
10186
|
+
if (!superClosed && firstError === undefined) {
|
|
10187
|
+
return false;
|
|
10188
|
+
}
|
|
10189
|
+
this._nativeStrictDurableTransactionsClosing = true;
|
|
10190
|
+
let strictTransactionsSettled = false;
|
|
10191
|
+
try {
|
|
10192
|
+
await this.settleNativeStrictDurableTransactionsForClose();
|
|
10193
|
+
strictTransactionsSettled = true;
|
|
10194
|
+
}
|
|
10195
|
+
catch (error) {
|
|
10196
|
+
firstError ??= error;
|
|
10197
|
+
}
|
|
10198
|
+
if (!strictTransactionsSettled) {
|
|
10199
|
+
throw firstError;
|
|
10200
|
+
}
|
|
10201
|
+
try {
|
|
10202
|
+
await this.log.close();
|
|
10203
|
+
}
|
|
10204
|
+
catch (error) {
|
|
10205
|
+
firstError ??= error;
|
|
10206
|
+
}
|
|
10207
|
+
try {
|
|
10208
|
+
await this._close();
|
|
10209
|
+
}
|
|
10210
|
+
catch (error) {
|
|
10211
|
+
firstError ??= error;
|
|
10212
|
+
}
|
|
10213
|
+
if (firstError !== undefined) {
|
|
10214
|
+
throw firstError;
|
|
7730
10215
|
}
|
|
7731
|
-
await this._close();
|
|
7732
|
-
await this.log.close();
|
|
7733
10216
|
return true;
|
|
7734
10217
|
}
|
|
7735
10218
|
async drop(from) {
|
|
10219
|
+
if (this.classifyTerminalOwnership(from) === "nonterminal") {
|
|
10220
|
+
return super.drop(from);
|
|
10221
|
+
}
|
|
10222
|
+
this.ensureNativeDurabilityRuntimeState();
|
|
10223
|
+
const nativePersistence = this._nativeBackboneCoordinatePersistence;
|
|
10224
|
+
if (nativePersistence &&
|
|
10225
|
+
(typeof nativePersistence.drop !== "function" ||
|
|
10226
|
+
typeof nativePersistence.resumeDrop !== "function" ||
|
|
10227
|
+
nativePersistence.supportsDrop !== true ||
|
|
10228
|
+
nativePersistence.dropIsTerminal !== true)) {
|
|
10229
|
+
// Reject before `super.drop()` can drop child programs or any lower index.
|
|
10230
|
+
throw new TerminalOperationNotStartedError("NativeBackbone coordinate persistence adapters must expose a terminal underlying drop capability and resumeDrop before SharedLog.drop() can erase their namespace");
|
|
10231
|
+
}
|
|
10232
|
+
// Adapter capability validation above is explicitly unstarted. Establish
|
|
10233
|
+
// the terminal fence only after that precondition succeeds.
|
|
10234
|
+
this.preventParentAttachments();
|
|
10235
|
+
this.cancelCurrentReplicationStateAnnouncementRetry();
|
|
7736
10236
|
// Best-effort: announce that we are going offline before tearing down
|
|
7737
10237
|
// RPC/subscription state (same reasoning as in `close()`).
|
|
7738
10238
|
try {
|
|
@@ -7769,14 +10269,132 @@ let SharedLog = (() => {
|
|
|
7769
10269
|
catch {
|
|
7770
10270
|
// ignore: drop should be resilient even if we were never fully started
|
|
7771
10271
|
}
|
|
7772
|
-
|
|
7773
|
-
|
|
7774
|
-
|
|
10272
|
+
let firstError;
|
|
10273
|
+
let superDropped = false;
|
|
10274
|
+
try {
|
|
10275
|
+
superDropped = await super.drop(from);
|
|
10276
|
+
}
|
|
10277
|
+
catch (error) {
|
|
10278
|
+
if (!this.closed || this.pendingTerminalOperation !== "drop") {
|
|
10279
|
+
// A fresh drop on a cleanly closed Program is an API rejection, not
|
|
10280
|
+
// permission to erase the already-closed lower log. Likewise, a child
|
|
10281
|
+
// failure before the base drop commits must leave all lower data intact.
|
|
10282
|
+
throw error;
|
|
10283
|
+
}
|
|
10284
|
+
firstError = error;
|
|
10285
|
+
}
|
|
10286
|
+
if (!superDropped && firstError === undefined) {
|
|
10287
|
+
return false;
|
|
10288
|
+
}
|
|
10289
|
+
this._nativeStrictDurableTransactionsClosing = true;
|
|
10290
|
+
const capture = async (operation) => {
|
|
10291
|
+
try {
|
|
10292
|
+
await operation();
|
|
10293
|
+
}
|
|
10294
|
+
catch (error) {
|
|
10295
|
+
firstError ??= error;
|
|
10296
|
+
}
|
|
10297
|
+
};
|
|
10298
|
+
let strictTransactionsSettled = false;
|
|
10299
|
+
try {
|
|
10300
|
+
await this.settleNativeStrictDurableTransactionsForClose();
|
|
10301
|
+
strictTransactionsSettled = true;
|
|
10302
|
+
}
|
|
10303
|
+
catch (error) {
|
|
10304
|
+
firstError ??= error;
|
|
10305
|
+
}
|
|
10306
|
+
if (!strictTransactionsSettled) {
|
|
10307
|
+
throw firstError;
|
|
10308
|
+
}
|
|
10309
|
+
if (nativePersistence) {
|
|
10310
|
+
try {
|
|
10311
|
+
const additionalFiles = this._nativeBackboneCoordinatePersistenceStore
|
|
10312
|
+
? NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILES
|
|
10313
|
+
: [];
|
|
10314
|
+
if (this._nativeBackboneDropStarted) {
|
|
10315
|
+
let resumed;
|
|
10316
|
+
try {
|
|
10317
|
+
resumed = await nativePersistence.resumeDrop();
|
|
10318
|
+
}
|
|
10319
|
+
catch (resumeError) {
|
|
10320
|
+
try {
|
|
10321
|
+
// A corrupt or partial tombstone deliberately restores the
|
|
10322
|
+
// adapter to active so explicit drop can overwrite it. For
|
|
10323
|
+
// transient read/remove failures the adapter stays dropping,
|
|
10324
|
+
// this fallback rejects, and the original recovery error wins.
|
|
10325
|
+
await nativePersistence.drop(additionalFiles);
|
|
10326
|
+
resumed = true;
|
|
10327
|
+
}
|
|
10328
|
+
catch (restartError) {
|
|
10329
|
+
throw new AggregateError([resumeError, restartError], "Failed to resume or restart native backbone drop");
|
|
10330
|
+
}
|
|
10331
|
+
}
|
|
10332
|
+
if (!resumed) {
|
|
10333
|
+
await nativePersistence.drop(additionalFiles);
|
|
10334
|
+
}
|
|
10335
|
+
}
|
|
10336
|
+
else {
|
|
10337
|
+
this._nativeBackboneDropStarted = true;
|
|
10338
|
+
await nativePersistence.drop(additionalFiles);
|
|
10339
|
+
}
|
|
10340
|
+
}
|
|
10341
|
+
catch (error) {
|
|
10342
|
+
firstError ??= error;
|
|
10343
|
+
// Quiesce the failed generation without closing the lower block/index
|
|
10344
|
+
// handles: exact drop retry still owns their destructive cleanup.
|
|
10345
|
+
await capture(() => this.log.close());
|
|
10346
|
+
await capture(() => this._close({ preserveDropRetryResources: true }));
|
|
10347
|
+
throw firstError;
|
|
10348
|
+
}
|
|
10349
|
+
// These in-memory states only stop being recovery-authoritative after all
|
|
10350
|
+
// six persistence files and both alternating intent slots are durably gone.
|
|
10351
|
+
this._nativeStrictDurableTransactionJournalState = undefined;
|
|
10352
|
+
this._nativeStrictDurableDocumentRecoveryDeferred = false;
|
|
10353
|
+
this._nativeStrictDurableTransactionTail = undefined;
|
|
10354
|
+
this._nativeStrictDurableTransactions?.clear();
|
|
10355
|
+
this.clearNativeStrictDurableTransactionFailure();
|
|
10356
|
+
this._nativeDurableCommitFailure = undefined;
|
|
10357
|
+
this._nativeDurableRecoveryReadyForReopen = false;
|
|
10358
|
+
this._nativeDurableRecoveryCids.clear();
|
|
10359
|
+
}
|
|
10360
|
+
let destructiveCleanupFailed = false;
|
|
10361
|
+
const dropIndex = async (index, forget) => {
|
|
10362
|
+
if (!index) {
|
|
10363
|
+
return;
|
|
10364
|
+
}
|
|
10365
|
+
try {
|
|
10366
|
+
await index.drop();
|
|
10367
|
+
forget();
|
|
10368
|
+
}
|
|
10369
|
+
catch (error) {
|
|
10370
|
+
firstError ??= error;
|
|
10371
|
+
destructiveCleanupFailed = true;
|
|
10372
|
+
}
|
|
10373
|
+
};
|
|
10374
|
+
await dropIndex(this._entryCoordinatesIndex, () => {
|
|
10375
|
+
this._entryCoordinatesIndex = undefined;
|
|
10376
|
+
});
|
|
10377
|
+
await dropIndex(this._replicationRangeIndex, () => {
|
|
10378
|
+
this._replicationRangeIndex = undefined;
|
|
10379
|
+
});
|
|
10380
|
+
try {
|
|
10381
|
+
await this.log.drop();
|
|
10382
|
+
}
|
|
10383
|
+
catch (error) {
|
|
10384
|
+
firstError ??= error;
|
|
10385
|
+
destructiveCleanupFailed = true;
|
|
10386
|
+
}
|
|
10387
|
+
if (destructiveCleanupFailed) {
|
|
10388
|
+
// Exact drop retry still owns every failed destructive handle. Quiesce
|
|
10389
|
+
// the rest of the generation without turning an erase failure into a
|
|
10390
|
+
// successful close or forgetting the only object that can retry it.
|
|
10391
|
+
await capture(() => this._close({ preserveDropRetryResources: true }));
|
|
10392
|
+
throw firstError;
|
|
10393
|
+
}
|
|
10394
|
+
await capture(() => this._close());
|
|
10395
|
+
if (firstError !== undefined) {
|
|
10396
|
+
throw firstError;
|
|
7775
10397
|
}
|
|
7776
|
-
await this._entryCoordinatesIndex.drop();
|
|
7777
|
-
await this._replicationRangeIndex.drop();
|
|
7778
|
-
await this.log.drop();
|
|
7779
|
-
await this._close();
|
|
7780
10398
|
return true;
|
|
7781
10399
|
}
|
|
7782
10400
|
async recover() {
|
|
@@ -7825,6 +10443,7 @@ let SharedLog = (() => {
|
|
|
7825
10443
|
? msg
|
|
7826
10444
|
: undefined;
|
|
7827
10445
|
try {
|
|
10446
|
+
this.throwIfNativeDurableCommitFailed();
|
|
7828
10447
|
if (!context.from) {
|
|
7829
10448
|
throw new Error("Missing from in update role message");
|
|
7830
10449
|
}
|
|
@@ -7932,8 +10551,7 @@ let SharedLog = (() => {
|
|
|
7932
10551
|
hashes,
|
|
7933
10552
|
from: rawFrom,
|
|
7934
10553
|
});
|
|
7935
|
-
rawPreparedReceiveSelectionValue =
|
|
7936
|
-
await rawPreparedReceiveSelection;
|
|
10554
|
+
rawPreparedReceiveSelectionValue = await rawPreparedReceiveSelection;
|
|
7937
10555
|
return rawPreparedReceiveSelectionValue;
|
|
7938
10556
|
};
|
|
7939
10557
|
// Receive fusion: when this message was resolved from the wire
|
|
@@ -7995,8 +10613,7 @@ let SharedLog = (() => {
|
|
|
7995
10613
|
if (!prepared) {
|
|
7996
10614
|
return undefined;
|
|
7997
10615
|
}
|
|
7998
|
-
rawPreparedReceiveSelectionValue =
|
|
7999
|
-
prepared.selection;
|
|
10616
|
+
rawPreparedReceiveSelectionValue = prepared.selection;
|
|
8000
10617
|
rawPreparedReceiveSelection = Promise.resolve(rawPreparedReceiveSelectionValue);
|
|
8001
10618
|
return { columns: prepared.columns };
|
|
8002
10619
|
}
|
|
@@ -8081,9 +10698,7 @@ let SharedLog = (() => {
|
|
|
8081
10698
|
? msg.preparedHashes
|
|
8082
10699
|
: heads.map(getExchangeHeadHash);
|
|
8083
10700
|
const isRepairHint = (msg.reserved[0] & EXCHANGE_HEADS_REPAIR_HINT) !== 0;
|
|
8084
|
-
logger.trace(`${this.node.identity.publicKey.hashcode()}: Recieved heads: ${heads.length === 1
|
|
8085
|
-
? headHashes[0]
|
|
8086
|
-
: "#" + heads.length}, logId: ${this.log.idString}`);
|
|
10701
|
+
logger.trace(`${this.node.identity.publicKey.hashcode()}: Recieved heads: ${heads.length === 1 ? headHashes[0] : "#" + heads.length}, logId: ${this.log.idString}`);
|
|
8087
10702
|
if (heads) {
|
|
8088
10703
|
let filteredHeads;
|
|
8089
10704
|
let filteredHeadHashes;
|
|
@@ -8196,8 +10811,7 @@ let SharedLog = (() => {
|
|
|
8196
10811
|
maxReplicas: this.replicas.max?.getValue(this),
|
|
8197
10812
|
};
|
|
8198
10813
|
if (!isReplicating &&
|
|
8199
|
-
rawPreparedReceiveSelectionValue
|
|
8200
|
-
?.retainedGroupLeaderPlans &&
|
|
10814
|
+
rawPreparedReceiveSelectionValue?.retainedGroupLeaderPlans &&
|
|
8201
10815
|
rawPreparedReceiveSelectionValue.retainedHashes.length ===
|
|
8202
10816
|
filteredHeadHashes.length &&
|
|
8203
10817
|
rawPreparedReceiveSelectionValue.retainedHashes.every((hash, index) => hash === filteredHeadHashes[index])) {
|
|
@@ -8217,8 +10831,7 @@ let SharedLog = (() => {
|
|
|
8217
10831
|
this._nativeBackbone.planPreparedRawReceiveGroupAssignments?.(filteredHeadHashes, replicaOptions, nativeLeaderOptions, contextFromHash);
|
|
8218
10832
|
if (nativeRawGroupAssignmentPlans &&
|
|
8219
10833
|
!nativeRawGroupAssignmentPlans.every((plan) => {
|
|
8220
|
-
const keepAsLeader = plan.isLeader ||
|
|
8221
|
-
(isRepairHint && plan.fromIsLeader);
|
|
10834
|
+
const keepAsLeader = plan.isLeader || (isRepairHint && plan.fromIsLeader);
|
|
8222
10835
|
const canKeepWithoutWait = isReplicating
|
|
8223
10836
|
? plan.isLeader
|
|
8224
10837
|
: keepAsLeader;
|
|
@@ -8494,8 +11107,7 @@ let SharedLog = (() => {
|
|
|
8494
11107
|
}
|
|
8495
11108
|
let usedNativeReceiveGroupLeaderPlans = false;
|
|
8496
11109
|
if (!isReplicating) {
|
|
8497
|
-
let leaderPlans = usedNativeRawGroupLeaderPlans ||
|
|
8498
|
-
usedNativeRawGroupAssignmentPlans
|
|
11110
|
+
let leaderPlans = usedNativeRawGroupLeaderPlans || usedNativeRawGroupAssignmentPlans
|
|
8499
11111
|
? receiveGroups.map((group) => group.leaderPlan)
|
|
8500
11112
|
: usedNativeRawGroups && this._nativeBackbone
|
|
8501
11113
|
? await this.planNativeBackboneReceiveGroupLeaders(receiveGroups)
|
|
@@ -8673,8 +11285,7 @@ let SharedLog = (() => {
|
|
|
8673
11285
|
for (const group of receiveGroups) {
|
|
8674
11286
|
const fromIsLeader = group.fromIsLeader ?? false;
|
|
8675
11287
|
const keepAsLeader = group.isLeader === true || (isRepairHint && fromIsLeader);
|
|
8676
|
-
if (group.maxReplicasFromNewEntries <
|
|
8677
|
-
group.maxReplicasFromHead) {
|
|
11288
|
+
if (group.maxReplicasFromNewEntries < group.maxReplicasFromHead) {
|
|
8678
11289
|
canUseAllKeptNativeJoinPlan = false;
|
|
8679
11290
|
break;
|
|
8680
11291
|
}
|
|
@@ -8757,8 +11368,7 @@ let SharedLog = (() => {
|
|
|
8757
11368
|
if (fromIsLeader) {
|
|
8758
11369
|
this.addPeersToGidPeerHistory(group.gid, [contextFromHash]);
|
|
8759
11370
|
}
|
|
8760
|
-
if (group.maxReplicasFromNewEntries <
|
|
8761
|
-
group.maxReplicasFromHead) {
|
|
11371
|
+
if (group.maxReplicasFromNewEntries < group.maxReplicasFromHead) {
|
|
8762
11372
|
(maybeDelete || (maybeDelete = [])).push(group.entries);
|
|
8763
11373
|
}
|
|
8764
11374
|
}
|
|
@@ -8801,7 +11411,8 @@ let SharedLog = (() => {
|
|
|
8801
11411
|
isLeader =
|
|
8802
11412
|
isLeader ||
|
|
8803
11413
|
this.node.identity.publicKey.hashcode() === key;
|
|
8804
|
-
fromIsLeader =
|
|
11414
|
+
fromIsLeader =
|
|
11415
|
+
fromIsLeader || contextFromHash === key;
|
|
8805
11416
|
},
|
|
8806
11417
|
});
|
|
8807
11418
|
}
|
|
@@ -8977,10 +11588,7 @@ let SharedLog = (() => {
|
|
|
8977
11588
|
this._pendingIHave.size === 0;
|
|
8978
11589
|
let mergeEntryByHash;
|
|
8979
11590
|
const materializeMergedEntry = (hash) => {
|
|
8980
|
-
mergeEntryByHash ??= new Map(allToMerge.map((entry) => [
|
|
8981
|
-
getExchangeHeadHash(entry),
|
|
8982
|
-
entry,
|
|
8983
|
-
]));
|
|
11591
|
+
mergeEntryByHash ??= new Map(allToMerge.map((entry) => [getExchangeHeadHash(entry), entry]));
|
|
8984
11592
|
const entryRef = mergeEntryByHash.get(hash);
|
|
8985
11593
|
if (!entryRef) {
|
|
8986
11594
|
throw new Error("Missing merged entry for appended hash");
|
|
@@ -9471,18 +12079,20 @@ let SharedLog = (() => {
|
|
|
9471
12079
|
}
|
|
9472
12080
|
}
|
|
9473
12081
|
else {
|
|
9474
|
-
const gid = nativeEntryGid ??
|
|
12082
|
+
const gid = nativeEntryGid ??
|
|
12083
|
+
nativeEntry?.gid ??
|
|
12084
|
+
indexedEntry.value.meta.gid;
|
|
9475
12085
|
const replicaCountByIndex = nativeLeaderHints.replicaCountsByIndex?.[i];
|
|
9476
12086
|
const replicas = replicaCountByIndex != null && replicaCountByIndex > 0
|
|
9477
12087
|
? replicaCountByIndex
|
|
9478
|
-
: nativeLeaderHints.replicaCounts.get(hash) ??
|
|
12088
|
+
: (nativeLeaderHints.replicaCounts.get(hash) ??
|
|
9479
12089
|
decodeReplicas({
|
|
9480
12090
|
meta: {
|
|
9481
12091
|
data: nativeEntryData ??
|
|
9482
12092
|
nativeEntry?.data ??
|
|
9483
12093
|
indexedEntry.value.meta.data,
|
|
9484
12094
|
},
|
|
9485
|
-
}).getValue(this);
|
|
12095
|
+
}).getValue(this));
|
|
9486
12096
|
if (!nativeLeaderHints.peerHistoryRemovedFlags?.[i] &&
|
|
9487
12097
|
!nativeLeaderHints.peerHistoryRemovedHashes.has(hash)) {
|
|
9488
12098
|
this.removePeerFromGidPeerHistory(from, gid);
|
|
@@ -9740,6 +12350,9 @@ let SharedLog = (() => {
|
|
|
9740
12350
|
}
|
|
9741
12351
|
}
|
|
9742
12352
|
catch (e) {
|
|
12353
|
+
if (e instanceof NativeDurableCommitError) {
|
|
12354
|
+
throw e;
|
|
12355
|
+
}
|
|
9743
12356
|
if (e instanceof AbortError ||
|
|
9744
12357
|
e instanceof NotStartedError ||
|
|
9745
12358
|
e instanceof IndexNotStartedError) {
|
|
@@ -9756,20 +12369,28 @@ let SharedLog = (() => {
|
|
|
9756
12369
|
logger.error(e);
|
|
9757
12370
|
}
|
|
9758
12371
|
finally {
|
|
9759
|
-
|
|
9760
|
-
|
|
9761
|
-
|
|
9762
|
-
|
|
9763
|
-
|
|
9764
|
-
|
|
9765
|
-
|
|
9766
|
-
|
|
9767
|
-
|
|
9768
|
-
|
|
9769
|
-
|
|
9770
|
-
|
|
12372
|
+
try {
|
|
12373
|
+
if (stashBackedRawMessage && stashBackedRawMessage.release()) {
|
|
12374
|
+
const syncProfile = this._logProperties?.sync?.profile;
|
|
12375
|
+
if (syncProfile) {
|
|
12376
|
+
emitSyncProfileEvent(syncProfile, {
|
|
12377
|
+
name: "sharedLog.rawReceive.wireStashRelease",
|
|
12378
|
+
component: "shared-log",
|
|
12379
|
+
entries: stashBackedRawMessage.heads.length,
|
|
12380
|
+
messages: 1,
|
|
12381
|
+
details: {
|
|
12382
|
+
bytesMaterialized: stashBackedRawMessage.bytesMaterializedCount,
|
|
12383
|
+
},
|
|
12384
|
+
});
|
|
12385
|
+
}
|
|
9771
12386
|
}
|
|
9772
12387
|
}
|
|
12388
|
+
finally {
|
|
12389
|
+
// Every return and every locally swallowed receive error passes this
|
|
12390
|
+
// boundary. Release a native wire stash exactly once first, then surface
|
|
12391
|
+
// any durable mutation poison that arose while handling the message.
|
|
12392
|
+
this.throwIfNativeDurableCommitFailed();
|
|
12393
|
+
}
|
|
9773
12394
|
}
|
|
9774
12395
|
}
|
|
9775
12396
|
async calculateTotalParticipation(options) {
|
|
@@ -9880,6 +12501,7 @@ let SharedLog = (() => {
|
|
|
9880
12501
|
return set;
|
|
9881
12502
|
}
|
|
9882
12503
|
async join(entries, options) {
|
|
12504
|
+
this.throwIfNativeDurableCommitFailed();
|
|
9883
12505
|
let entriesToReplicate = [];
|
|
9884
12506
|
const localHashes = options?.replicate && this.log.length > 0
|
|
9885
12507
|
? await this.log.entryIndex.hasMany(entries.map((element) => typeof element === "string" ? element : element.hash))
|
|
@@ -10007,9 +12629,7 @@ let SharedLog = (() => {
|
|
|
10007
12629
|
await persistCoordinate(entry);
|
|
10008
12630
|
}
|
|
10009
12631
|
if (messageToSend) {
|
|
10010
|
-
await this.
|
|
10011
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
10012
|
-
});
|
|
12632
|
+
await this.sendReplicationAnnouncement(messageToSend);
|
|
10013
12633
|
}
|
|
10014
12634
|
}
|
|
10015
12635
|
}
|
|
@@ -10431,7 +13051,8 @@ let SharedLog = (() => {
|
|
|
10431
13051
|
const skipHashes = this._checkedPrune.pendingDeletes.size > 0
|
|
10432
13052
|
? [...this._checkedPrune.pendingDeletes.keys()]
|
|
10433
13053
|
: [];
|
|
10434
|
-
const nativeBackboneHintArrays = this
|
|
13054
|
+
const nativeBackboneHintArrays = this
|
|
13055
|
+
._nativeBackbone;
|
|
10435
13056
|
if (skipHashes.length === 0) {
|
|
10436
13057
|
const allConfirmed = nativeBackboneHintArrays.planRequestPruneAllConfirmed?.(hashes, from, {
|
|
10437
13058
|
...this.createNativeLeaderOptions(context),
|
|
@@ -10937,29 +13558,46 @@ let SharedLog = (() => {
|
|
|
10937
13558
|
return materialized;
|
|
10938
13559
|
}
|
|
10939
13560
|
snapshotResidentCoordinateEntries(hashes) {
|
|
10940
|
-
|
|
13561
|
+
const uniqueHashes = new Set([...hashes].filter(Boolean));
|
|
13562
|
+
if (uniqueHashes.size === 0) {
|
|
10941
13563
|
return undefined;
|
|
10942
13564
|
}
|
|
10943
|
-
const
|
|
10944
|
-
|
|
10945
|
-
|
|
13565
|
+
const entries = new Map();
|
|
13566
|
+
const generations = new Map();
|
|
13567
|
+
const mutationGenerations = (this._nativeCoordinateMutationGenerations ??=
|
|
13568
|
+
new Map());
|
|
13569
|
+
for (const hash of uniqueHashes) {
|
|
13570
|
+
const generation = (mutationGenerations.get(hash) ?? 0) + 1;
|
|
13571
|
+
mutationGenerations.set(hash, generation);
|
|
13572
|
+
generations.set(hash, generation);
|
|
13573
|
+
const entry = this._residentEntryCoordinatesByHash?.get(hash);
|
|
10946
13574
|
if (entry) {
|
|
10947
|
-
|
|
13575
|
+
entries.set(hash, entry);
|
|
10948
13576
|
}
|
|
10949
13577
|
}
|
|
10950
|
-
return
|
|
13578
|
+
return { hashes: uniqueHashes, entries, generations };
|
|
10951
13579
|
}
|
|
10952
|
-
rollbackNativeBackboneCoordinateAppend(appendHash,
|
|
13580
|
+
rollbackNativeBackboneCoordinateAppend(appendHash, rollback) {
|
|
10953
13581
|
const backbone = this._nativeBackbone;
|
|
10954
13582
|
if (!backbone) {
|
|
10955
13583
|
return;
|
|
10956
13584
|
}
|
|
10957
|
-
|
|
10958
|
-
this.
|
|
10959
|
-
|
|
10960
|
-
|
|
10961
|
-
|
|
10962
|
-
|
|
13585
|
+
const hashes = rollback?.hashes ?? new Set([appendHash]);
|
|
13586
|
+
const mutationGenerations = (this._nativeCoordinateMutationGenerations ??=
|
|
13587
|
+
new Map());
|
|
13588
|
+
for (const hash of hashes) {
|
|
13589
|
+
const expectedGeneration = rollback?.generations.get(hash);
|
|
13590
|
+
if (expectedGeneration !== undefined &&
|
|
13591
|
+
mutationGenerations.get(hash) !== expectedGeneration) {
|
|
13592
|
+
continue;
|
|
13593
|
+
}
|
|
13594
|
+
backbone.deleteEntryCoordinates(hash);
|
|
13595
|
+
this._nativeSharedLogState?.deleteEntryCoordinates(hash);
|
|
13596
|
+
this._residentEntryCoordinatesByHash?.delete(hash);
|
|
13597
|
+
const entry = rollback?.entries.get(hash);
|
|
13598
|
+
if (!entry) {
|
|
13599
|
+
continue;
|
|
13600
|
+
}
|
|
10963
13601
|
const fields = isEntryReplicated(entry)
|
|
10964
13602
|
? {
|
|
10965
13603
|
hash: entry.hash,
|
|
@@ -10973,9 +13611,41 @@ let SharedLog = (() => {
|
|
|
10973
13611
|
? decodeReplicas(entry).getValue(this)
|
|
10974
13612
|
: fields.coordinates.length;
|
|
10975
13613
|
backbone.putEntryCoordinates(fields.hash, fields.gid, fields.coordinates, fields.assignedToRangeBoundary, requestedReplicas, fields.hashNumber);
|
|
13614
|
+
this._nativeSharedLogState?.putEntryCoordinates(fields.hash, fields.gid, fields.coordinates, fields.assignedToRangeBoundary, requestedReplicas, fields.hashNumber);
|
|
10976
13615
|
this._residentEntryCoordinatesByHash?.set(hash, entry);
|
|
10977
13616
|
}
|
|
10978
13617
|
}
|
|
13618
|
+
async rollbackNativeBackboneCoordinateAppendDurably(appendHash, rollback) {
|
|
13619
|
+
this.rollbackNativeBackboneCoordinateAppend(appendHash, rollback);
|
|
13620
|
+
const coordinateIndex = this.entryCoordinatesIndex;
|
|
13621
|
+
const hashes = rollback?.hashes ?? new Set([appendHash]);
|
|
13622
|
+
const mutationGenerations = (this._nativeCoordinateMutationGenerations ??=
|
|
13623
|
+
new Map());
|
|
13624
|
+
for (const hash of hashes) {
|
|
13625
|
+
const expectedGeneration = rollback?.generations.get(hash);
|
|
13626
|
+
if (expectedGeneration !== undefined &&
|
|
13627
|
+
mutationGenerations.get(hash) !== expectedGeneration) {
|
|
13628
|
+
continue;
|
|
13629
|
+
}
|
|
13630
|
+
const previous = rollback?.entries.get(hash);
|
|
13631
|
+
if (previous) {
|
|
13632
|
+
await coordinateIndex.put(this.materializeResidentCoordinateEntry(previous));
|
|
13633
|
+
}
|
|
13634
|
+
else if (coordinateIndex.delIds) {
|
|
13635
|
+
await coordinateIndex.delIds([hash]);
|
|
13636
|
+
}
|
|
13637
|
+
else if (coordinateIndex.delIdsNoReturn) {
|
|
13638
|
+
await coordinateIndex.delIdsNoReturn([hash]);
|
|
13639
|
+
}
|
|
13640
|
+
else {
|
|
13641
|
+
await coordinateIndex.del({ query: { hash } });
|
|
13642
|
+
}
|
|
13643
|
+
}
|
|
13644
|
+
const flushed = this.flushNativeBackboneCoordinateJournal();
|
|
13645
|
+
if (isPromiseLike(flushed)) {
|
|
13646
|
+
await flushed;
|
|
13647
|
+
}
|
|
13648
|
+
}
|
|
10979
13649
|
persistPreparedCoordinate(properties) {
|
|
10980
13650
|
const { assignedToRangeBoundary, fields } = properties.prepared;
|
|
10981
13651
|
const deleteHashes = combineCoordinateDeleteHashes(properties.nextHashes, properties.deleteHashes);
|
|
@@ -11112,7 +13782,12 @@ let SharedLog = (() => {
|
|
|
11112
13782
|
flushNativeBackboneCoordinateJournal() {
|
|
11113
13783
|
const backbone = this._nativeBackbone;
|
|
11114
13784
|
const persistence = this._nativeBackboneCoordinatePersistence;
|
|
11115
|
-
if (!backbone || !persistence) {
|
|
13785
|
+
if (!backbone || !persistence || this._nativeBackboneDropStarted) {
|
|
13786
|
+
return undefined;
|
|
13787
|
+
}
|
|
13788
|
+
if (backbone.coordinatePendingJournalLength === 0 &&
|
|
13789
|
+
backbone.documentPendingJournalLength === 0 &&
|
|
13790
|
+
backbone.documentSignerPendingJournalLength === 0) {
|
|
11116
13791
|
return undefined;
|
|
11117
13792
|
}
|
|
11118
13793
|
return mapMaybePromise(persistence.flushJournal(backbone), () => {
|
|
@@ -11123,7 +13798,7 @@ let SharedLog = (() => {
|
|
|
11123
13798
|
flushNativeBackboneCoordinateJournalOnAppend() {
|
|
11124
13799
|
const backbone = this._nativeBackbone;
|
|
11125
13800
|
const persistence = this._nativeBackboneCoordinatePersistence;
|
|
11126
|
-
if (!backbone || !persistence) {
|
|
13801
|
+
if (!backbone || !persistence || this._nativeBackboneDropStarted) {
|
|
11127
13802
|
return undefined;
|
|
11128
13803
|
}
|
|
11129
13804
|
if (persistence.flushJournalOnAppend) {
|
|
@@ -11163,6 +13838,23 @@ let SharedLog = (() => {
|
|
|
11163
13838
|
if (!persistence) {
|
|
11164
13839
|
return;
|
|
11165
13840
|
}
|
|
13841
|
+
if (this._nativeBackboneDropStarted) {
|
|
13842
|
+
// `drop()` owns the durable namespace lifecycle. Never flush the live wasm
|
|
13843
|
+
// journals or invoke an ordinary custom close after its tombstone/erase has
|
|
13844
|
+
// started: a close implementation that rewrites cached state could resurrect
|
|
13845
|
+
// files after a successful terminal drop.
|
|
13846
|
+
return;
|
|
13847
|
+
}
|
|
13848
|
+
if (this._nativeDurableCommitFailure &&
|
|
13849
|
+
!this._nativeDurableRecoveryReadyForReopen) {
|
|
13850
|
+
// The failed native transaction was never published by the lower log.
|
|
13851
|
+
// Its coordinate/document/signer records are still only in the wasm
|
|
13852
|
+
// pending journals. Closing without flushing discards that generation;
|
|
13853
|
+
// the next backbone hydrates the last acknowledged checkpoint.
|
|
13854
|
+
await persistence.close?.();
|
|
13855
|
+
this._nativeDurableRecoveryReadyForReopen = true;
|
|
13856
|
+
return;
|
|
13857
|
+
}
|
|
11166
13858
|
await this.flushNativeBackboneCoordinateJournal();
|
|
11167
13859
|
await persistence.close?.();
|
|
11168
13860
|
}
|
|
@@ -12012,6 +14704,9 @@ let SharedLog = (() => {
|
|
|
12012
14704
|
async findFullReplicaLeaders(replicas, roleAge, peerFilter) {
|
|
12013
14705
|
const now = Date.now();
|
|
12014
14706
|
const leaders = new Map();
|
|
14707
|
+
// Strict-only peers are not global fallbacks, but may still own an entry's
|
|
14708
|
+
// coordinates. Remember them so a partial fallback cannot bypass sampling.
|
|
14709
|
+
const excludedStrictPeers = new Set();
|
|
12015
14710
|
const includeStrict = this._logProperties?.strictFullReplicaFallback !== false;
|
|
12016
14711
|
const iterator = this.replicationIndex.iterate({}, { shape: { hash: true, timestamp: true, mode: true } });
|
|
12017
14712
|
try {
|
|
@@ -12025,10 +14720,11 @@ let SharedLog = (() => {
|
|
|
12025
14720
|
if (peerFilter && !peerFilter.has(range.hash)) {
|
|
12026
14721
|
continue;
|
|
12027
14722
|
}
|
|
12028
|
-
if (
|
|
14723
|
+
if (range.mode === ReplicationIntent.Strict && !includeStrict) {
|
|
14724
|
+
excludedStrictPeers.add(range.hash);
|
|
12029
14725
|
continue;
|
|
12030
14726
|
}
|
|
12031
|
-
if (range
|
|
14727
|
+
if (!isMatured(range, now, roleAge)) {
|
|
12032
14728
|
continue;
|
|
12033
14729
|
}
|
|
12034
14730
|
leaders.set(range.hash, { intersecting: true });
|
|
@@ -12041,6 +14737,11 @@ let SharedLog = (() => {
|
|
|
12041
14737
|
finally {
|
|
12042
14738
|
await iterator.close();
|
|
12043
14739
|
}
|
|
14740
|
+
for (const hash of excludedStrictPeers) {
|
|
14741
|
+
if (!leaders.has(hash)) {
|
|
14742
|
+
return undefined;
|
|
14743
|
+
}
|
|
14744
|
+
}
|
|
12044
14745
|
return leaders.size > 0 ? leaders : undefined;
|
|
12045
14746
|
}
|
|
12046
14747
|
async findEntryReplicatedLeaderBatch(entries, options) {
|
|
@@ -13074,10 +15775,19 @@ let SharedLog = (() => {
|
|
|
13074
15775
|
if (!canReplicate) {
|
|
13075
15776
|
return false;
|
|
13076
15777
|
}
|
|
13077
|
-
|
|
13078
|
-
|
|
13079
|
-
|
|
13080
|
-
|
|
15778
|
+
try {
|
|
15779
|
+
await this.startAnnounceReplicating([dynamicRange], {
|
|
15780
|
+
checkDuplicates: false,
|
|
15781
|
+
reset: false,
|
|
15782
|
+
});
|
|
15783
|
+
}
|
|
15784
|
+
catch (error) {
|
|
15785
|
+
if (isTransientReplicationAnnouncementError(error) &&
|
|
15786
|
+
this._replicationAnnouncementRetryPending) {
|
|
15787
|
+
return false;
|
|
15788
|
+
}
|
|
15789
|
+
throw error;
|
|
15790
|
+
}
|
|
13081
15791
|
/* await this._updateRole(newRole, onRoleChange); */
|
|
13082
15792
|
this.rebalanceParticipationDebounced?.call();
|
|
13083
15793
|
return true;
|