@peerbit/shared-log 13.2.5 → 13.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/src/index.js CHANGED
@@ -40,7 +40,7 @@ import { AccessError, Ed25519Keypair, Ed25519PublicKey, PublicSignKey, Secp256k1
40
40
  import { And, ByteMatchQuery, NotStartedError as IndexNotStartedError, Or, Sort, StringMatch, toId, } from "@peerbit/indexer-interface";
41
41
  import { Entry, EntryType, LamportClock, Log, Meta, ShallowEntry, ShallowMeta, Timestamp, verifyEd25519Batch, verifyEntryV0Ed25519BatchFromEntries, } from "@peerbit/log";
42
42
  import { logger as loggerFn } from "@peerbit/logger";
43
- import { ClosedError, Program } from "@peerbit/program";
43
+ import { ClosedError, Program, TerminalOperationNotStartedError, } from "@peerbit/program";
44
44
  import { FanoutChannel, waitForSubscribers, } from "@peerbit/pubsub";
45
45
  import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
46
46
  import { RPC } from "@peerbit/rpc";
@@ -53,8 +53,8 @@ import { BlocksMessage } from "./blocks.js";
53
53
  import { CheckedPruneCoordinator, } from "./checked-prune.js";
54
54
  import { CPUUsageIntervalLag } from "./cpu.js";
55
55
  import { debouncedAccumulatorMap, } from "./debounce.js";
56
- import { NoPeersError } from "./errors.js";
57
- import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, ResponseIPrune, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
56
+ import { NativeDurableCommitError, NoPeersError } from "./errors.js";
57
+ import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, ResponseIPrune, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
58
58
  import { FanoutEnvelope } from "./fanout-envelope.js";
59
59
  import { MAX_U32, MAX_U64, createNumbers, } from "./integers.js";
60
60
  import { TransportMessage } from "./message.js";
@@ -105,9 +105,9 @@ const toLocalPublicSignKey = (key) => {
105
105
  export { createReplicationDomainHash, createReplicationDomainTime, };
106
106
  export { CPUUsageIntervalLag };
107
107
  export * from "./replication.js";
108
- export { ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, EntryReplicatedU32, EntryReplicatedU64, NoPeersError, };
108
+ export { ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, EntryReplicatedU32, EntryReplicatedU64, NativeDurableCommitError, NoPeersError, };
109
109
  export { MAX_U32, MAX_U64 };
110
- export { ExchangeHeadsMessage, RawExchangeHeadsMessage };
110
+ export { ExchangeHeadsMessage, RawExchangeHeadsMessage, StashBackedRawExchangeHeadsMessage, };
111
111
  export const logger = loggerFn("peerbit:shared-log");
112
112
  const warn = logger.newScope("warn");
113
113
  const traceLogger = logger.trace;
@@ -129,6 +129,21 @@ const canUseOptionalNativeModuleImports = () => {
129
129
  * to avoid doubled slashes.
130
130
  */
131
131
  const joinNativeCoordinateDirectory = (nodeDirectory, fsSafeLogId) => `${nodeDirectory.replace(/[/\\]+$/, "")}/coordinates/${fsSafeLogId}`;
132
+ const createNativeDurableBlockStore = async (storage) => new AnyBlockStore(await storage.sublevel("blocks", {
133
+ // Strict mirrors remain WAL-backed across close. The available snapshot
134
+ // rewrite is not a crash-atomic generation protocol, so thresholds must
135
+ // not re-enable it behind this acknowledgement boundary.
136
+ compactOnClose: false,
137
+ // A native append is acknowledged only after this mirror resolves. The
138
+ // Rust store's normal immutable fast path may resolve before its WAL write;
139
+ // strict mode waits for the journal write and sync, closing the SIGKILL gap.
140
+ durability: "strict",
141
+ }));
142
+ const createDefaultDurableBlockStore = async (storage) => new AnyBlockStore(await storage.sublevel("blocks", {
143
+ // State this default explicitly so a cached child created by the native
144
+ // path cannot silently carry its deferred-close policy into this path.
145
+ compactOnClose: true,
146
+ }));
132
147
  /**
133
148
  * Write-through block store bridging the native backbone's in-wasm-memory block
134
149
  * store to a durable per-program {@link AnyBlockStore}.
@@ -148,11 +163,12 @@ const joinNativeCoordinateDirectory = (nodeDirectory, fsSafeLogId) => `${nodeDir
148
163
  * METHOD SURFACE (see #1006): `RemoteBlocks` and the log feature-detect the
149
164
  * optional batch methods (`putMany`/`putKnown`/`putKnownMany`/
150
165
  * `putKnownManyColumns`/`rmMany`). To keep the receive-fusion / columnar fast
151
- * paths engaged this wrapper exposes EXACTLY the methods the native store
152
- * exposes — including `putKnownManyColumns`, which `AnyBlockStore` does not have
153
- * — and delegates each. It deliberately does not add methods the native store
154
- * lacks (`getBlockResponsePayload`/`getNativeLogBlockStoreHandle` stay absent so
155
- * the optional-chained probes in `RemoteBlocks` behave exactly as today).
166
+ * paths engaged this wrapper preserves the native store's optional write
167
+ * methods — including `putKnownManyColumns`, which `AnyBlockStore` does not have
168
+ * — and delegates each. It adds only local durability/trim coordination hooks
169
+ * consumed by `RemoteBlocks` and the log; protocol/native-handle capabilities
170
+ * such as `getBlockResponsePayload`/`getNativeLogBlockStoreHandle` stay absent,
171
+ * so their optional-chained probes keep the existing fallback behavior.
156
172
  */
157
173
  class NativeBackboneWriteThroughBlockStore {
158
174
  native;
@@ -170,8 +186,302 @@ class NativeBackboneWriteThroughBlockStore {
170
186
  // vanishing and leaving the block out of durable while native/log report
171
187
  // success. `.catch` also prevents unhandled-rejection noise.
172
188
  pendingDurableWrites = new Set();
173
- durableWriteError;
174
- trackDurable(result) {
189
+ nativeDurableCommitFailure;
190
+ stopCompleted = false;
191
+ nativeDeleteTombstones = new Map();
192
+ nativeDeleteEpoch = 0;
193
+ nativeBlockWriteGenerations = new Map();
194
+ pendingNativeDeleteCleanup = new Map();
195
+ stagedNativeDeleteCleanups = new Map();
196
+ nextNativeDeleteCleanupToken = 0;
197
+ nativeDeleteCleanupRunning;
198
+ nextNativeCommitOwnershipToken = 0;
199
+ nativeCommitOwnerships = new Map();
200
+ nativeCommitOwnershipsByCid = new Map();
201
+ getNativeDurableCommitFailure() {
202
+ return this.nativeDurableCommitFailure;
203
+ }
204
+ recordNativeDurableCommitFailure(cause, options) {
205
+ if (this.nativeDurableCommitFailure) {
206
+ this.nativeDurableCommitFailure.addCommitContext(options);
207
+ return this.nativeDurableCommitFailure;
208
+ }
209
+ this.nativeDurableCommitFailure =
210
+ cause instanceof NativeDurableCommitError
211
+ ? cause
212
+ : new NativeDurableCommitError(cause, options);
213
+ if (cause instanceof NativeDurableCommitError) {
214
+ cause.addCommitContext(options);
215
+ }
216
+ return this.nativeDurableCommitFailure;
217
+ }
218
+ throwIfNativeDurableCommitFailed() {
219
+ if (this.nativeDurableCommitFailure) {
220
+ throw this.nativeDurableCommitFailure;
221
+ }
222
+ }
223
+ throwIfDurableWritesFailed() {
224
+ this.throwIfNativeDurableCommitFailed();
225
+ }
226
+ async commitDurableMutation(operation, committedCids, failedCids) {
227
+ this.throwIfNativeDurableCommitFailed();
228
+ const committedCidList = [...committedCids];
229
+ const failedCidList = failedCids ? [...failedCids] : committedCidList;
230
+ let result;
231
+ const operationResult = Promise.resolve().then(operation);
232
+ this.trackAwaitedDurable(operationResult);
233
+ try {
234
+ result = await operationResult;
235
+ }
236
+ catch (error) {
237
+ throw this.recordNativeDurableCommitFailure(error, {
238
+ committedCids: committedCidList,
239
+ failedCids: failedCidList,
240
+ });
241
+ }
242
+ // A different concurrent native mutation may have poisoned the wrapper
243
+ // while this durable call was in flight. Include this operation among the
244
+ // native-applied facts, but not among durable calls that actually failed.
245
+ if (this.nativeDurableCommitFailure) {
246
+ this.nativeDurableCommitFailure.addCommitContext({
247
+ committedCids: committedCidList,
248
+ failedCids: [],
249
+ });
250
+ throw this.nativeDurableCommitFailure;
251
+ }
252
+ return result;
253
+ }
254
+ beginNativeDelete(cids) {
255
+ this.nativeDeleteEpoch++;
256
+ for (const cid of cids) {
257
+ this.nativeDeleteTombstones.set(cid, (this.nativeDeleteTombstones.get(cid) ?? 0) + 1);
258
+ }
259
+ }
260
+ endNativeDelete(cids) {
261
+ for (const cid of cids) {
262
+ const remaining = (this.nativeDeleteTombstones.get(cid) ?? 1) - 1;
263
+ if (remaining <= 0) {
264
+ this.nativeDeleteTombstones.delete(cid);
265
+ }
266
+ else {
267
+ this.nativeDeleteTombstones.set(cid, remaining);
268
+ }
269
+ }
270
+ }
271
+ isNativeDeletePending(cid) {
272
+ return this.nativeDeleteTombstones.has(cid);
273
+ }
274
+ // A CID can be legitimately re-added after a native trim (content addressing
275
+ // makes the bytes identical, but its liveness is new). Cancel any queued trim
276
+ // for that CID and advance its generation before the write is exposed. An
277
+ // already-running cleanup uses the generation/pending map to avoid deleting
278
+ // the new native value; synchronous columnar writes also chain their durable
279
+ // mirror behind that cleanup below.
280
+ noteNativeBlockWrite(cids) {
281
+ const generations = new Map();
282
+ for (const cid of new Set(cids)) {
283
+ const generation = (this.nativeBlockWriteGenerations.get(cid) ?? 0) + 1;
284
+ this.nativeBlockWriteGenerations.set(cid, generation);
285
+ generations.set(cid, generation);
286
+ if (this.pendingNativeDeleteCleanup.delete(cid)) {
287
+ this.endNativeDelete([cid]);
288
+ }
289
+ for (const staged of this.stagedNativeDeleteCleanups.values()) {
290
+ if (staged.delete(cid)) {
291
+ this.endNativeDelete([cid]);
292
+ }
293
+ }
294
+ }
295
+ return generations;
296
+ }
297
+ beginNativeCommitOwnership(generations) {
298
+ if (generations.size === 0) {
299
+ return undefined;
300
+ }
301
+ const token = {
302
+ id: ++this.nextNativeCommitOwnershipToken,
303
+ rows: new Map(),
304
+ };
305
+ for (const [cid, generation] of generations) {
306
+ const owners = this.nativeCommitOwnershipsByCid.get(cid) ?? new Set();
307
+ const shared = owners.size > 0;
308
+ for (const ownerId of owners) {
309
+ const owner = this.nativeCommitOwnerships.get(ownerId);
310
+ const row = owner?.rows.get(cid);
311
+ if (row)
312
+ row.shared = true;
313
+ }
314
+ owners.add(token.id);
315
+ this.nativeCommitOwnershipsByCid.set(cid, owners);
316
+ token.rows.set(cid, { generation, shared });
317
+ }
318
+ this.nativeCommitOwnerships.set(token.id, token);
319
+ return token;
320
+ }
321
+ releaseNativeCommitOwnership(token) {
322
+ if (!token ||
323
+ typeof token !== "object" ||
324
+ typeof token.id !== "number") {
325
+ return;
326
+ }
327
+ const owned = this.nativeCommitOwnerships.get(token.id);
328
+ if (owned !== token) {
329
+ return;
330
+ }
331
+ this.nativeCommitOwnerships.delete(owned.id);
332
+ for (const cid of owned.rows.keys()) {
333
+ const owners = this.nativeCommitOwnershipsByCid.get(cid);
334
+ owners?.delete(owned.id);
335
+ if (owners?.size === 0) {
336
+ this.nativeCommitOwnershipsByCid.delete(cid);
337
+ }
338
+ }
339
+ }
340
+ acknowledgeNativeCommitOwnership(token) {
341
+ this.releaseNativeCommitOwnership(token);
342
+ }
343
+ enqueueNativeDeleteCleanup(cids) {
344
+ for (const cid of new Set(cids)) {
345
+ if (!this.pendingNativeDeleteCleanup.has(cid)) {
346
+ this.pendingNativeDeleteCleanup.set(cid, this.nativeBlockWriteGenerations.get(cid) ?? 0);
347
+ this.beginNativeDelete([cid]);
348
+ }
349
+ }
350
+ }
351
+ releaseStagedNativeDeleteCleanup(token) {
352
+ const staged = this.stagedNativeDeleteCleanups.get(token);
353
+ if (!staged) {
354
+ return false;
355
+ }
356
+ this.stagedNativeDeleteCleanups.delete(token);
357
+ for (const [cid] of staged) {
358
+ this.endNativeDelete([cid]);
359
+ }
360
+ return true;
361
+ }
362
+ discardStagedNativeDeleteCleanups() {
363
+ for (const token of [...this.stagedNativeDeleteCleanups.keys()]) {
364
+ this.releaseStagedNativeDeleteCleanup(token);
365
+ }
366
+ }
367
+ // Native commit callbacks call this immediately after the native transaction
368
+ // returns, before awaiting the new block's durable mirror. This stages read
369
+ // tombstones only. Durable deletion is promoted later by the exact EntryIndex
370
+ // consume token, so an unacknowledged/failed append cannot delete the old
371
+ // durable head that the lower log still publishes.
372
+ beginNativeDeleteCleanup(cids) {
373
+ this.throwIfNativeDurableCommitFailed();
374
+ const uniqueCids = [...new Set(cids)];
375
+ if (uniqueCids.length === 0) {
376
+ return undefined;
377
+ }
378
+ const token = ++this.nextNativeDeleteCleanupToken;
379
+ const staged = new Map(uniqueCids.map((cid) => [
380
+ cid,
381
+ this.nativeBlockWriteGenerations.get(cid) ?? 0,
382
+ ]));
383
+ this.stagedNativeDeleteCleanups.set(token, staged);
384
+ this.beginNativeDelete(uniqueCids);
385
+ return token;
386
+ }
387
+ cancelNativeDeleteCleanup(cleanupToken) {
388
+ if (typeof cleanupToken === "number") {
389
+ this.releaseStagedNativeDeleteCleanup(cleanupToken);
390
+ }
391
+ }
392
+ async waitForNativeDeleteCleanup() {
393
+ while (this.nativeDeleteCleanupRunning) {
394
+ await this.nativeDeleteCleanupRunning;
395
+ }
396
+ }
397
+ async waitForTrackedDurableWrites() {
398
+ while (this.pendingDurableWrites.size > 0) {
399
+ await Promise.allSettled([...this.pendingDurableWrites]);
400
+ }
401
+ }
402
+ async retryNativeDeleteCleanup(options) {
403
+ await this.waitForNativeDeleteCleanup();
404
+ if (this.nativeDurableCommitFailure && !options?.allowPoisoned) {
405
+ throw this.nativeDurableCommitFailure;
406
+ }
407
+ // A synchronous columnar write may already have scheduled its durable
408
+ // mirror. Let it settle before deleting queued CIDs, but leave any recorded
409
+ // error for drainDurable() to surface to its owning operation/stop.
410
+ await this.waitForTrackedDurableWrites();
411
+ await this.waitForNativeDeleteCleanup();
412
+ // A tracked mirror or the cleanup we just waited for may have poisoned the
413
+ // wrapper. Do not begin another durable mutation on the ordinary path after
414
+ // that asynchronous boundary. stop() alone opts into one cleanup retry so a
415
+ // transient delete failure can still release its tombstones and resources.
416
+ if (this.nativeDurableCommitFailure && !options?.allowPoisoned) {
417
+ throw this.nativeDurableCommitFailure;
418
+ }
419
+ if (this.pendingNativeDeleteCleanup.size === 0) {
420
+ return;
421
+ }
422
+ const cleanupEntries = [...this.pendingNativeDeleteCleanup].filter(([cid, generation]) => (this.nativeBlockWriteGenerations.get(cid) ?? 0) === generation);
423
+ if (cleanupEntries.length === 0) {
424
+ return;
425
+ }
426
+ const cids = cleanupEntries.map(([cid]) => cid);
427
+ let cleanupFailure;
428
+ const running = (async () => {
429
+ let durableRemoved = false;
430
+ let nativeRemoved = false;
431
+ try {
432
+ await this.durable.rmMany(cids);
433
+ durableRemoved = true;
434
+ }
435
+ catch (error) {
436
+ cleanupFailure = error;
437
+ // The new entry blocks are already durable at this point and the native
438
+ // transaction has selected the new graph/index state. Old content-addressed
439
+ // blocks are therefore harmless unreachable orphans. Keep this cleanup as
440
+ // retryable debt instead of poisoning/rolling back a fully durable append;
441
+ // a partial rmMany is safe for the same reason.
442
+ warn(`Failed durable native-trim cleanup; retaining retry debt: ${String(error)}`);
443
+ }
444
+ finally {
445
+ // A read that began in the native-transaction -> cleanup-hook gap may
446
+ // have repopulated native. Always repeat the native removal, even when
447
+ // durable rm failed. Exclude CIDs re-added while durable IO was pending;
448
+ // their generation change cancelled the queued delete.
449
+ const stillDeleted = cids.filter((cid) => this.pendingNativeDeleteCleanup.has(cid));
450
+ try {
451
+ if (stillDeleted.length > 0) {
452
+ await this.native.rmMany(stillDeleted);
453
+ }
454
+ nativeRemoved = true;
455
+ }
456
+ catch (error) {
457
+ cleanupFailure ??= error;
458
+ warn(`Failed to repeat native-trim hot block removal: ${String(error)}`);
459
+ }
460
+ }
461
+ if (durableRemoved && nativeRemoved) {
462
+ for (const [cid, generation] of cleanupEntries) {
463
+ if (this.pendingNativeDeleteCleanup.get(cid) === generation) {
464
+ this.pendingNativeDeleteCleanup.delete(cid);
465
+ this.nativeBlockWriteGenerations.delete(cid);
466
+ this.endNativeDelete([cid]);
467
+ }
468
+ }
469
+ }
470
+ })();
471
+ this.nativeDeleteCleanupRunning = running;
472
+ try {
473
+ await running;
474
+ }
475
+ finally {
476
+ if (this.nativeDeleteCleanupRunning === running) {
477
+ this.nativeDeleteCleanupRunning = undefined;
478
+ }
479
+ }
480
+ if (options?.throwOnFailure && cleanupFailure !== undefined) {
481
+ throw cleanupFailure;
482
+ }
483
+ }
484
+ trackDurable(result, cids) {
175
485
  // The durable store may answer synchronously (an in-memory or already
176
486
  // resolved store); only a real pending promise needs tracking. A sync
177
487
  // success is already durable; a sync throw would have propagated already.
@@ -182,9 +492,22 @@ class NativeBackboneWriteThroughBlockStore {
182
492
  this.pendingDurableWrites.delete(tracked);
183
493
  }, (error) => {
184
494
  this.pendingDurableWrites.delete(tracked);
185
- if (this.durableWriteError === undefined) {
186
- this.durableWriteError = error;
187
- }
495
+ this.recordNativeDurableCommitFailure(error, {
496
+ committedCids: cids,
497
+ failedCids: cids,
498
+ });
499
+ });
500
+ this.pendingDurableWrites.add(tracked);
501
+ }
502
+ // Awaited mirror writes surface their own rejection to the append that
503
+ // created them. Track a non-rejecting settlement companion as well so stop()
504
+ // and later wrapper operations cannot close/use durable storage while that
505
+ // append barrier is still in flight.
506
+ trackAwaitedDurable(result) {
507
+ const tracked = result.then(() => {
508
+ this.pendingDurableWrites.delete(tracked);
509
+ }, () => {
510
+ this.pendingDurableWrites.delete(tracked);
188
511
  });
189
512
  this.pendingDurableWrites.add(tracked);
190
513
  }
@@ -192,14 +515,8 @@ class NativeBackboneWriteThroughBlockStore {
192
515
  // the first failure. Awaited methods call this so a prior columnar durable
193
516
  // failure propagates as back-pressure to the next caller.
194
517
  async drainDurable() {
195
- while (this.pendingDurableWrites.size > 0) {
196
- await Promise.allSettled([...this.pendingDurableWrites]);
197
- }
198
- if (this.durableWriteError !== undefined) {
199
- const error = this.durableWriteError;
200
- this.durableWriteError = undefined;
201
- throw error;
202
- }
518
+ await this.waitForTrackedDurableWrites();
519
+ this.throwIfNativeDurableCommitFailed();
203
520
  }
204
521
  // --- lifecycle -------------------------------------------------------
205
522
  // The native store's lifecycle hooks are no-ops; only the durable store
@@ -213,12 +530,55 @@ class NativeBackboneWriteThroughBlockStore {
213
530
  // persisted coordinate + signer facts without resolving the entry block.
214
531
  async start() {
215
532
  await this.durable.start();
533
+ this.stopCompleted = false;
216
534
  }
217
535
  async stop() {
536
+ if (this.stopCompleted) {
537
+ return;
538
+ }
218
539
  // Surface any tracked (sync-path) durable write failure and ensure all
219
- // mirror writes have settled before the durable store is torn down.
220
- await this.drainDurable();
221
- await this.durable.stop();
540
+ // mirror writes have settled before the durable store is torn down. Closing
541
+ // the durable store is unconditional: a prior columnar mirror failure must
542
+ // not leak the store lifecycle resource.
543
+ let firstError;
544
+ let shutdownError;
545
+ try {
546
+ await this.drainDurable();
547
+ }
548
+ catch (error) {
549
+ firstError = error;
550
+ }
551
+ // Tokens not consumed by EntryIndex belong to native prepares that never
552
+ // published their lower-log trim. Release their read tombstones, but never
553
+ // promote them to durable deletion during shutdown.
554
+ this.discardStagedNativeDeleteCleanups();
555
+ try {
556
+ await this.retryNativeDeleteCleanup({ allowPoisoned: true });
557
+ }
558
+ catch (error) {
559
+ firstError ??= error;
560
+ shutdownError ??= error;
561
+ }
562
+ try {
563
+ await this.durable.stop();
564
+ }
565
+ catch (error) {
566
+ firstError ??= error;
567
+ shutdownError ??= error;
568
+ }
569
+ if (shutdownError === undefined) {
570
+ // A durable poison belongs to the generation being closed. Report it once
571
+ // so the owning terminal call observes the failed append, but remember that
572
+ // every mandatory shutdown stage completed. Conservative content-addressed
573
+ // trim debt may remain in this retired wrapper after best-effort cleanup; a
574
+ // fresh generation gets a new wrapper and must not be wedged by that debt.
575
+ // The exact terminal retry may therefore finish parent bookkeeping without
576
+ // rethrowing the same latched poison forever.
577
+ this.stopCompleted = true;
578
+ }
579
+ if (firstError !== undefined) {
580
+ throw firstError;
581
+ }
222
582
  }
223
583
  status() {
224
584
  return this.durable.status();
@@ -226,6 +586,12 @@ class NativeBackboneWriteThroughBlockStore {
226
586
  waitFor() {
227
587
  return Promise.resolve([]);
228
588
  }
589
+ // Native commit APIs must keep their block-store callback synchronous. The
590
+ // lower log calls this barrier after that callback reports committed blocks
591
+ // and before publishing index/head facts.
592
+ waitForDurableWrites() {
593
+ return this.drainDurable();
594
+ }
229
595
  // Mirror a single already-committed block (present in the wasm map) to the
230
596
  // durable store ONLY. Used by the native commit-only append fast path: the
231
597
  // native prepare commits the entry block into the wasm map and returns no raw
@@ -233,11 +599,157 @@ class NativeBackboneWriteThroughBlockStore {
233
599
  // route the block through the log's finishBlocks/putKnown* (that would disturb
234
600
  // the commit-only append path the RCS optimization depends on). Instead the
235
601
  // caller reads the committed bytes back and calls this so the block lands in
236
- // durable directly. Like putKnownManyColumns' durable mirror, the write is
237
- // tracked (not fire-and-forget) so an IO/disk-full failure surfaces on the
238
- // next awaited wrapper method / stop() rather than being swallowed.
239
- mirrorToDurable(cid, bytes) {
240
- this.trackDurable(this.durable.putKnown(cid, bytes));
602
+ // durable directly. The caller awaits this method before acknowledging its
603
+ // append, so a failed write rejects the append that produced the block.
604
+ async mirrorToDurable(cid, bytes, options) {
605
+ this.throwIfNativeDurableCommitFailed();
606
+ // The native commit happened before this call. Mark the CID live before
607
+ // awaiting anything so a concurrent retry of an older trim cannot remove
608
+ // the newly committed hot block.
609
+ let ownership;
610
+ if (options?.nativeTrimmed !== true) {
611
+ ownership = this.beginNativeCommitOwnership(this.noteNativeBlockWrite([cid]));
612
+ await this.waitForNativeDeleteCleanup();
613
+ }
614
+ try {
615
+ await this.drainDurable();
616
+ if (ownership) {
617
+ const existed = await this.durable.hasMany([...ownership.rows.keys()]);
618
+ let index = 0;
619
+ for (const row of ownership.rows.values()) {
620
+ row.durableExistedBefore = existed[index++] === true;
621
+ }
622
+ }
623
+ const result = this.commitDurableMutation(() => this.durable.putKnown(cid, bytes), [cid]);
624
+ await result;
625
+ await this.retryNativeDeleteCleanup();
626
+ this.throwIfNativeDurableCommitFailed();
627
+ return ownership;
628
+ }
629
+ catch (error) {
630
+ // The caller never receives an ownership token for an indeterminate write.
631
+ // Release the in-memory claim and preserve any bytes the backend may have
632
+ // applied before rejecting.
633
+ this.releaseNativeCommitOwnership(ownership);
634
+ throw error;
635
+ }
636
+ }
637
+ async mirrorManyToDurable(blocks, options) {
638
+ this.throwIfNativeDurableCommitFailed();
639
+ const cids = blocks.map(([cid]) => cid);
640
+ // Rows trimmed later in the same native batch deliberately remain owned by
641
+ // that batch's staged cleanup; only mark surviving rows live.
642
+ const liveCids = cids.filter((cid) => !options?.nativeTrimmedCids?.has(cid));
643
+ const ownership = this.beginNativeCommitOwnership(this.noteNativeBlockWrite(liveCids));
644
+ if (liveCids.length > 0) {
645
+ await this.waitForNativeDeleteCleanup();
646
+ }
647
+ try {
648
+ await this.drainDurable();
649
+ if (ownership) {
650
+ const existed = await this.durable.hasMany([...ownership.rows.keys()]);
651
+ let index = 0;
652
+ for (const row of ownership.rows.values()) {
653
+ row.durableExistedBefore = existed[index++] === true;
654
+ }
655
+ }
656
+ if (blocks.length > 0) {
657
+ await this.commitDurableMutation(() => this.durable.putKnownMany(blocks), cids);
658
+ }
659
+ await this.retryNativeDeleteCleanup();
660
+ this.throwIfNativeDurableCommitFailed();
661
+ return ownership;
662
+ }
663
+ catch (error) {
664
+ this.releaseNativeCommitOwnership(ownership);
665
+ throw error;
666
+ }
667
+ }
668
+ async rollbackFailedNativeCommits(cids, restoreNativeCids = [], ownershipToken) {
669
+ // This is the sole mutation allowed after poison: it removes a native
670
+ // transaction that the lower log never published. Do not route through the
671
+ // guarded rmMany path, and settle the failed mirror before compensating it.
672
+ await this.waitForTrackedDurableWrites();
673
+ // A failed replacement never published its trim. Reassert the restored CIDs
674
+ // as live before any compensation IO so staged/pending delete intents cannot
675
+ // remove the last acknowledged blocks during stop or reopen.
676
+ const ownership = ownershipToken && typeof ownershipToken === "object"
677
+ ? this.nativeCommitOwnerships.get(ownershipToken.id)
678
+ : undefined;
679
+ const verifiedOwnership = ownership && ownership === ownershipToken ? ownership : undefined;
680
+ const restoreSet = new Set(restoreNativeCids);
681
+ const safeDurableDeletes = verifiedOwnership
682
+ ? [...new Set(cids)].filter((cid) => {
683
+ const row = verifiedOwnership.rows.get(cid);
684
+ const owners = this.nativeCommitOwnershipsByCid.get(cid);
685
+ return (!restoreSet.has(cid) &&
686
+ row?.durableExistedBefore === false &&
687
+ row.shared === false &&
688
+ (this.nativeBlockWriteGenerations.get(cid) ?? 0) ===
689
+ row.generation &&
690
+ owners?.size === 1 &&
691
+ owners.has(verifiedOwnership.id));
692
+ })
693
+ : [];
694
+ this.noteNativeBlockWrite(restoreNativeCids);
695
+ let firstError;
696
+ try {
697
+ if (safeDurableDeletes.length > 0) {
698
+ await this.durable.rmMany(safeDurableDeletes);
699
+ }
700
+ }
701
+ catch (error) {
702
+ firstError = error;
703
+ }
704
+ // A native prepare runs before ownership can observe the hot map, so it cannot
705
+ // prove that a CID was absent there before this operation. Keep native bytes as
706
+ // unreachable orphans rather than deleting acknowledged/shared/restored data.
707
+ if (restoreNativeCids.length > 0) {
708
+ try {
709
+ const values = await this.durable.getMany(restoreNativeCids);
710
+ const restore = [];
711
+ for (let index = 0; index < restoreNativeCids.length; index++) {
712
+ const value = values[index];
713
+ if (value)
714
+ restore.push([restoreNativeCids[index], value]);
715
+ }
716
+ if (restore.length > 0) {
717
+ this.native.putKnownMany(restore);
718
+ }
719
+ }
720
+ catch (error) {
721
+ firstError ??= error;
722
+ }
723
+ }
724
+ this.releaseNativeCommitOwnership(ownershipToken);
725
+ if (firstError !== undefined)
726
+ throw firstError;
727
+ }
728
+ /**
729
+ * Compensate a native prepare that failed before its durable mirror began.
730
+ * Durable presence proves a same-CID acknowledged owner; an active ownership
731
+ * token proves a concurrent mirror. Only an unowned, non-durable hot block is
732
+ * exclusively attributable to the failed prepare and safe to remove.
733
+ */
734
+ async rollbackUnmirroredNativeCommits(cids, restoreNativeCids = []) {
735
+ const unique = [...new Set(cids)];
736
+ // Native prepares bypass this wrapper. Any observed wrapper generation is
737
+ // therefore evidence of a generic/same-CID writer, not of the failed prepare.
738
+ // Snapshot before the first await and require both absence and stability so a
739
+ // write starting before or during durable.hasMany cannot lose its hot value to
740
+ // a stale `false` result.
741
+ const genericWriteGenerations = new Map(unique.map((cid) => [cid, this.nativeBlockWriteGenerations.get(cid)]));
742
+ await this.rollbackFailedNativeCommits(cids, restoreNativeCids);
743
+ const durablePresence = await this.durable.hasMany(unique);
744
+ const restore = new Set(restoreNativeCids);
745
+ const safeNativeDeletes = unique.filter((cid, index) => !restore.has(cid) &&
746
+ durablePresence[index] !== true &&
747
+ genericWriteGenerations.get(cid) === undefined &&
748
+ this.nativeBlockWriteGenerations.get(cid) === undefined &&
749
+ (this.nativeCommitOwnershipsByCid.get(cid)?.size ?? 0) === 0);
750
+ if (safeNativeDeletes.length > 0) {
751
+ await this.native.rmMany(safeNativeDeletes);
752
+ }
241
753
  }
242
754
  // --- writes (apply to BOTH: native first for the hot path, then durable) ---
243
755
  async put(data) {
@@ -250,7 +762,13 @@ class NativeBackboneWriteThroughBlockStore {
250
762
  const value = data instanceof Uint8Array
251
763
  ? data
252
764
  : data.block.bytes;
253
- await this.durable.putKnown(cid, value);
765
+ this.noteNativeBlockWrite([cid]);
766
+ await this.waitForNativeDeleteCleanup();
767
+ // put() may have yielded while calculating the CID. Restore the hot value
768
+ // after any older cleanup that was already in flight.
769
+ this.throwIfNativeDurableCommitFailed();
770
+ this.native.putKnown(cid, value);
771
+ await this.commitDurableMutation(() => this.durable.putKnown(cid, value), [cid]);
254
772
  return cid;
255
773
  }
256
774
  async putMany(blocks) {
@@ -263,7 +781,11 @@ class NativeBackboneWriteThroughBlockStore {
263
781
  : block.block.bytes;
264
782
  return [cid, value];
265
783
  });
266
- await this.durable.putKnownMany(durableBlocks);
784
+ this.noteNativeBlockWrite(cids);
785
+ await this.waitForNativeDeleteCleanup();
786
+ this.throwIfNativeDurableCommitFailed();
787
+ this.native.putKnownMany(durableBlocks);
788
+ await this.commitDurableMutation(() => this.durable.putKnownMany(durableBlocks), cids);
267
789
  return cids;
268
790
  }
269
791
  // Native put is synchronous (the authoritative hot store); the durable mirror
@@ -273,18 +795,30 @@ class NativeBackboneWriteThroughBlockStore {
273
795
  // both await this method, so returning a promise is compatible.
274
796
  async putKnown(cid, bytes) {
275
797
  await this.drainDurable();
798
+ await this.waitForNativeDeleteCleanup();
799
+ this.throwIfNativeDurableCommitFailed();
276
800
  const stored = this.native.putKnown(cid, bytes);
277
- await this.durable.putKnown(cid, bytes);
801
+ this.noteNativeBlockWrite([cid]);
802
+ await this.commitDurableMutation(() => this.durable.putKnown(cid, bytes), [cid]);
278
803
  return stored;
279
804
  }
280
805
  async putKnownMany(blocks) {
281
806
  await this.drainDurable();
807
+ await this.waitForNativeDeleteCleanup();
808
+ this.throwIfNativeDurableCommitFailed();
282
809
  const cids = this.native.putKnownMany(blocks);
283
- await this.durable.putKnownMany(blocks);
810
+ this.noteNativeBlockWrite(cids);
811
+ await this.commitDurableMutation(() => this.durable.putKnownMany(blocks), cids);
284
812
  return cids;
285
813
  }
286
814
  putKnownManyColumns(cids, bytes) {
815
+ this.throwIfNativeDurableCommitFailed();
816
+ if (cids.length !== bytes.length) {
817
+ throw new Error("Expected equal block column lengths");
818
+ }
819
+ const cleanupBarrier = this.nativeDeleteCleanupRunning;
287
820
  const stored = this.native.putKnownManyColumns(cids, bytes);
821
+ this.noteNativeBlockWrite(cids);
288
822
  // AnyBlockStore has no columnar method; mirror via putKnownMany, which
289
823
  // takes [cid, bytes] tuples and hits the same batched store path.
290
824
  // This method must return a synchronous string[] (RemoteBlocks.putKnownManyColumns
@@ -292,7 +826,19 @@ class NativeBackboneWriteThroughBlockStore {
292
826
  // inline. Track it instead of `void`ing it so a durable rejection surfaces
293
827
  // on the next awaited wrapper method / stop() rather than being swallowed.
294
828
  const durableBlocks = cids.map((cid, index) => [cid, bytes[index]]);
295
- this.trackDurable(this.durable.putKnownMany(durableBlocks));
829
+ let durableResult;
830
+ try {
831
+ durableResult = cleanupBarrier
832
+ ? cleanupBarrier.then(() => this.durable.putKnownMany(durableBlocks))
833
+ : this.durable.putKnownMany(durableBlocks);
834
+ }
835
+ catch (error) {
836
+ throw this.recordNativeDurableCommitFailure(error, {
837
+ committedCids: cids,
838
+ failedCids: cids,
839
+ });
840
+ }
841
+ this.trackDurable(durableResult, cids);
296
842
  return stored;
297
843
  }
298
844
  // --- reads (native/wasm first; on miss, durable fallback + repopulate) ---
@@ -302,11 +848,28 @@ class NativeBackboneWriteThroughBlockStore {
302
848
  // avoids a spurious miss (which would otherwise fall through to a remote
303
849
  // read) for a block that is present on disk.
304
850
  async get(cid, _options) {
851
+ if (this.nativeDurableCommitFailure) {
852
+ // After poison, durable storage is the last acknowledged authority. Do not
853
+ // repopulate or otherwise mutate the native store until reopen.
854
+ return this.isNativeDeletePending(cid)
855
+ ? undefined
856
+ : this.durable.get(cid);
857
+ }
858
+ const deleteEpoch = this.nativeDeleteEpoch;
859
+ if (this.isNativeDeletePending(cid)) {
860
+ return undefined;
861
+ }
305
862
  const local = this.native.get(cid);
306
863
  if (local != null) {
307
- return local;
864
+ return deleteEpoch === this.nativeDeleteEpoch ? local : this.get(cid);
308
865
  }
309
866
  const durableValue = await this.durable.get(cid);
867
+ if (this.nativeDurableCommitFailure) {
868
+ return this.isNativeDeletePending(cid) ? undefined : durableValue;
869
+ }
870
+ if (deleteEpoch !== this.nativeDeleteEpoch) {
871
+ return this.get(cid);
872
+ }
310
873
  if (durableValue != null) {
311
874
  // Repopulate the native map so the native graph reads it next time.
312
875
  this.native.putKnownManyColumns([cid], [durableValue]);
@@ -315,25 +878,52 @@ class NativeBackboneWriteThroughBlockStore {
315
878
  return undefined;
316
879
  }
317
880
  async getMany(cids) {
881
+ if (this.nativeDurableCommitFailure) {
882
+ const values = await this.durable.getMany(cids);
883
+ for (let index = 0; index < cids.length; index++) {
884
+ if (this.isNativeDeletePending(cids[index])) {
885
+ values[index] = undefined;
886
+ }
887
+ }
888
+ return values;
889
+ }
890
+ const deleteEpoch = this.nativeDeleteEpoch;
318
891
  const results = await this.native.getMany(cids);
892
+ if (deleteEpoch !== this.nativeDeleteEpoch) {
893
+ return this.getMany(cids);
894
+ }
319
895
  const missing = [];
896
+ const missingIndexes = [];
320
897
  for (let i = 0; i < results.length; i++) {
321
- if (results[i] == null) {
898
+ if (this.isNativeDeletePending(cids[i])) {
899
+ results[i] = undefined;
900
+ }
901
+ else if (results[i] == null) {
322
902
  missing.push(cids[i]);
903
+ missingIndexes.push(i);
323
904
  }
324
905
  }
325
906
  if (missing.length === 0) {
326
907
  return results;
327
908
  }
328
909
  const durableValues = await this.durable.getMany(missing);
910
+ if (this.nativeDurableCommitFailure) {
911
+ const values = await this.durable.getMany(cids);
912
+ for (let index = 0; index < cids.length; index++) {
913
+ if (this.isNativeDeletePending(cids[index])) {
914
+ values[index] = undefined;
915
+ }
916
+ }
917
+ return values;
918
+ }
919
+ if (deleteEpoch !== this.nativeDeleteEpoch) {
920
+ return this.getMany(cids);
921
+ }
329
922
  const repopulateCids = [];
330
923
  const repopulateBytes = [];
331
- let missingIndex = 0;
332
- for (let i = 0; i < results.length; i++) {
333
- if (results[i] != null) {
334
- continue;
335
- }
336
- const value = durableValues[missingIndex++];
924
+ for (let missingIndex = 0; missingIndex < missingIndexes.length; missingIndex++) {
925
+ const i = missingIndexes[missingIndex];
926
+ const value = durableValues[missingIndex];
337
927
  if (value != null) {
338
928
  results[i] = value;
339
929
  repopulateCids.push(cids[i]);
@@ -347,8 +937,15 @@ class NativeBackboneWriteThroughBlockStore {
347
937
  return results;
348
938
  }
349
939
  async has(cid) {
940
+ if (this.nativeDurableCommitFailure) {
941
+ return this.isNativeDeletePending(cid) ? false : this.durable.has(cid);
942
+ }
943
+ const deleteEpoch = this.nativeDeleteEpoch;
944
+ if (this.isNativeDeletePending(cid)) {
945
+ return false;
946
+ }
350
947
  if (this.native.has(cid)) {
351
- return true;
948
+ return deleteEpoch === this.nativeDeleteEpoch ? true : this.has(cid);
352
949
  }
353
950
  // Mirror getMany/hasMany: a block absent from the native wasm map may still
354
951
  // be present in the durable store (e.g. persisted on disk but not yet
@@ -356,25 +953,44 @@ class NativeBackboneWriteThroughBlockStore {
356
953
  // checks agree with the resolves that getMany/hasMany already durable-fall
357
954
  // back on. `Blocks.has` is declared `MaybePromise<boolean>`, so returning a
358
955
  // promise here is contract-compatible.
359
- return this.durable.has(cid);
956
+ const durableHas = await this.durable.has(cid);
957
+ return deleteEpoch === this.nativeDeleteEpoch ? durableHas : this.has(cid);
360
958
  }
361
959
  async hasMany(cids) {
960
+ if (this.nativeDurableCommitFailure) {
961
+ const values = await this.durable.hasMany(cids);
962
+ for (let index = 0; index < cids.length; index++) {
963
+ if (this.isNativeDeletePending(cids[index])) {
964
+ values[index] = false;
965
+ }
966
+ }
967
+ return values;
968
+ }
969
+ const deleteEpoch = this.nativeDeleteEpoch;
362
970
  const nativeHas = await this.native.hasMany(cids);
971
+ if (deleteEpoch !== this.nativeDeleteEpoch) {
972
+ return this.hasMany(cids);
973
+ }
363
974
  const missing = [];
975
+ const missingIndexes = [];
364
976
  for (let i = 0; i < nativeHas.length; i++) {
365
- if (!nativeHas[i]) {
977
+ if (this.isNativeDeletePending(cids[i])) {
978
+ nativeHas[i] = false;
979
+ }
980
+ else if (!nativeHas[i]) {
366
981
  missing.push(cids[i]);
982
+ missingIndexes.push(i);
367
983
  }
368
984
  }
369
985
  if (missing.length === 0) {
370
986
  return nativeHas;
371
987
  }
372
988
  const durableHas = await this.durable.hasMany(missing);
373
- let missingIndex = 0;
374
- for (let i = 0; i < nativeHas.length; i++) {
375
- if (!nativeHas[i]) {
376
- nativeHas[i] = durableHas[missingIndex++];
377
- }
989
+ if (deleteEpoch !== this.nativeDeleteEpoch) {
990
+ return this.hasMany(cids);
991
+ }
992
+ for (let missingIndex = 0; missingIndex < missingIndexes.length; missingIndex++) {
993
+ nativeHas[missingIndexes[missingIndex]] = durableHas[missingIndex];
378
994
  }
379
995
  return nativeHas;
380
996
  }
@@ -383,25 +999,128 @@ class NativeBackboneWriteThroughBlockStore {
383
999
  // resolves only after both succeed and a durable failure rejects here rather
384
1000
  // than being swallowed. All rm callers (RemoteBlocks.rm, the log) await it.
385
1001
  async rm(cid) {
386
- await this.drainDurable();
387
- this.native.rm(cid);
388
- await this.durable.rm(cid);
1002
+ this.throwIfNativeDurableCommitFailed();
1003
+ const writeGeneration = this.nativeBlockWriteGenerations.get(cid);
1004
+ this.beginNativeDelete([cid]);
1005
+ try {
1006
+ await this.drainDurable();
1007
+ this.native.rm(cid);
1008
+ await this.commitDurableMutation(() => this.durable.rm(cid), [cid]);
1009
+ // A durable read that began before the tombstone may have repopulated
1010
+ // native while durable rm was pending. Remove it idempotently again.
1011
+ this.native.rm(cid);
1012
+ if (this.nativeBlockWriteGenerations.get(cid) === writeGeneration &&
1013
+ !this.pendingNativeDeleteCleanup.has(cid)) {
1014
+ this.nativeBlockWriteGenerations.delete(cid);
1015
+ }
1016
+ }
1017
+ finally {
1018
+ this.endNativeDelete([cid]);
1019
+ }
389
1020
  }
390
1021
  del(cid) {
391
- void this.rm(cid);
1022
+ return this.rm(cid);
392
1023
  }
393
1024
  async rmMany(cids) {
394
- await this.drainDurable();
395
- const removed = await this.native.rmMany(cids);
396
- await this.durable.rmMany(cids);
397
- return removed;
1025
+ this.throwIfNativeDurableCommitFailed();
1026
+ const writeGenerations = new Map(cids.map((cid) => [cid, this.nativeBlockWriteGenerations.get(cid)]));
1027
+ this.beginNativeDelete(cids);
1028
+ try {
1029
+ await this.drainDurable();
1030
+ const removed = await this.native.rmMany(cids);
1031
+ await this.commitDurableMutation(() => this.durable.rmMany(cids), cids);
1032
+ await this.native.rmMany(cids);
1033
+ for (const cid of cids) {
1034
+ if (this.nativeBlockWriteGenerations.get(cid) ===
1035
+ writeGenerations.get(cid) &&
1036
+ !this.pendingNativeDeleteCleanup.has(cid)) {
1037
+ this.nativeBlockWriteGenerations.delete(cid);
1038
+ }
1039
+ }
1040
+ return removed;
1041
+ }
1042
+ finally {
1043
+ this.endNativeDelete(cids);
1044
+ }
1045
+ }
1046
+ // Native trim may already have removed the hot wasm blocks. Queue the durable
1047
+ // copy for cleanup, retaining read tombstones until removal succeeds; a
1048
+ // cleanup failure is retried and never fed into ordinary append rollback.
1049
+ // EntryIndex feature-detects this hook.
1050
+ async rmManyAfterNativeDelete(cids, cleanupToken) {
1051
+ if (this.nativeDurableCommitFailure) {
1052
+ this.cancelNativeDeleteCleanup(cleanupToken);
1053
+ throw this.nativeDurableCommitFailure;
1054
+ }
1055
+ let preannounced = false;
1056
+ if (typeof cleanupToken === "number") {
1057
+ const staged = this.stagedNativeDeleteCleanups.get(cleanupToken);
1058
+ if (staged) {
1059
+ preannounced = true;
1060
+ this.stagedNativeDeleteCleanups.delete(cleanupToken);
1061
+ for (const [cid, generation] of staged) {
1062
+ if ((this.nativeBlockWriteGenerations.get(cid) ?? 0) !== generation) {
1063
+ this.endNativeDelete([cid]);
1064
+ continue;
1065
+ }
1066
+ if (this.pendingNativeDeleteCleanup.has(cid)) {
1067
+ // Another delete already owns a tombstone for this CID.
1068
+ this.endNativeDelete([cid]);
1069
+ }
1070
+ else {
1071
+ // Transfer the staged tombstone to the now-published cleanup.
1072
+ this.pendingNativeDeleteCleanup.set(cid, generation);
1073
+ }
1074
+ }
1075
+ }
1076
+ }
1077
+ if (!preannounced) {
1078
+ this.enqueueNativeDeleteCleanup(cids);
1079
+ }
1080
+ await this.retryNativeDeleteCleanup();
1081
+ }
1082
+ /** Finish committed trim GC before its durable recovery intent is retired. */
1083
+ async completeCommittedNativeDeleteCleanup(cids, options) {
1084
+ this.throwIfNativeDurableCommitFailed();
1085
+ const uniqueCids = [...new Set(cids.filter(Boolean))];
1086
+ if (uniqueCids.length === 0) {
1087
+ return;
1088
+ }
1089
+ // Only restart recovery reconstructs missing debt. On the live path, an
1090
+ // absent row may mean the CID was legitimately re-added after its original
1091
+ // generation-owned trim completed or was cancelled; re-enqueueing it here
1092
+ // would capture the new generation and delete live content.
1093
+ if (options?.reconstructMissing) {
1094
+ this.enqueueNativeDeleteCleanup(uniqueCids);
1095
+ }
1096
+ if (!uniqueCids.some((cid) => this.pendingNativeDeleteCleanup.has(cid))) {
1097
+ return;
1098
+ }
1099
+ await this.retryNativeDeleteCleanup({ throwOnFailure: true });
1100
+ const remaining = uniqueCids.filter((cid) => this.pendingNativeDeleteCleanup.has(cid));
1101
+ if (remaining.length > 0) {
1102
+ throw new Error(`Committed native trim cleanup remains incomplete: ${remaining.join(", ")}`);
1103
+ }
398
1104
  }
399
1105
  // --- misc ------------------------------------------------------------
400
1106
  async *iterator() {
1107
+ if (this.nativeDurableCommitFailure) {
1108
+ for await (const block of this.durable.iterator()) {
1109
+ if (!this.isNativeDeletePending(block[0])) {
1110
+ yield block;
1111
+ }
1112
+ }
1113
+ return;
1114
+ }
401
1115
  yield* this.native.iterator();
402
1116
  }
403
- size() {
404
- return this.native.size();
1117
+ async size() {
1118
+ // The hot wasm map is intentionally cold after reopen, so it cannot be the
1119
+ // storage-budget authority. Settle synchronous columnar mirrors first, then
1120
+ // report durable bytes; pending trim cleanup remains conservatively counted
1121
+ // until its durable deletion succeeds.
1122
+ await this.drainDurable();
1123
+ return this.durable.size();
405
1124
  }
406
1125
  persisted() {
407
1126
  // The blocks are now mirrored to a durable store, so report persisted so
@@ -473,6 +1192,29 @@ const canConfirmNativeRequestPruneBatch = (hints, hashCount) => {
473
1192
  }
474
1193
  return true;
475
1194
  };
1195
+ const NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILE = "strict-durable-transaction-intent.json";
1196
+ const NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_BACKUP_FILE = "strict-durable-transaction-intent.backup.json";
1197
+ const NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILES = [
1198
+ NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILE,
1199
+ NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_BACKUP_FILE,
1200
+ ];
1201
+ const NATIVE_STRICT_DURABLE_TRANSACTION_JOURNAL_FORMAT = "peerbit-native-strict-durable-transaction";
1202
+ const nativeStrictDurableTransactionJournalBody = (sequence, intent) => ({
1203
+ format: NATIVE_STRICT_DURABLE_TRANSACTION_JOURNAL_FORMAT,
1204
+ version: 1,
1205
+ sequence,
1206
+ state: intent ? "intent" : "cleared",
1207
+ intent: intent ?? null,
1208
+ });
1209
+ const nativeStrictDurableTransactionJournalBodyBytes = (body) => new TextEncoder().encode(JSON.stringify(body));
1210
+ const nativeStrictDurableTransactionJournalRecordBytes = (sequence, intent) => {
1211
+ const body = nativeStrictDurableTransactionJournalBody(sequence, intent);
1212
+ const record = {
1213
+ ...body,
1214
+ checksum: toHexString(sha256Sync(nativeStrictDurableTransactionJournalBodyBytes(body))),
1215
+ };
1216
+ return new TextEncoder().encode(JSON.stringify(record));
1217
+ };
476
1218
  const getLatestEntry = (entries) => {
477
1219
  let latest = undefined;
478
1220
  for (const element of entries) {
@@ -602,6 +1344,43 @@ const isNotStartedError = (e) => {
602
1344
  }
603
1345
  return false;
604
1346
  };
1347
+ /**
1348
+ * Replication announcements are best-effort convergence messages. A detached
1349
+ * fanout shard can time out even though the shared log itself remains open.
1350
+ * Keep retries deliberately limited to concrete TimeoutErrors: abort/close and
1351
+ * unexpected programming/data errors must retain their existing semantics.
1352
+ *
1353
+ * Exact constructor/name checks complement `instanceof` for errors crossing
1354
+ * worker or duplicate-package boundaries in browsers.
1355
+ */
1356
+ const isTransientReplicationAnnouncementError = (error, seen = new Set()) => {
1357
+ if (error != null &&
1358
+ (typeof error === "object" || typeof error === "function")) {
1359
+ if (seen.has(error)) {
1360
+ return false;
1361
+ }
1362
+ seen.add(error);
1363
+ }
1364
+ if (error instanceof TimeoutError) {
1365
+ return true;
1366
+ }
1367
+ const nested = error?.errors;
1368
+ if (Array.isArray(nested) && nested.length > 0) {
1369
+ return nested.every((item) => isTransientReplicationAnnouncementError(item, new Set(seen)));
1370
+ }
1371
+ const cause = error?.cause;
1372
+ if (cause != null && isTransientReplicationAnnouncementError(cause, seen)) {
1373
+ return true;
1374
+ }
1375
+ const constructorName = typeof error?.constructor
1376
+ ?.name === "string"
1377
+ ? error.constructor.name
1378
+ : "";
1379
+ const name = typeof error?.name === "string"
1380
+ ? error.name
1381
+ : "";
1382
+ return constructorName === "TimeoutError" || name === "TimeoutError";
1383
+ };
605
1384
  const EMPTY_HASHES = [];
606
1385
  const normalizedHashValues = (hashes) => {
607
1386
  if (Array.isArray(hashes)) {
@@ -698,6 +1477,7 @@ const CHECKED_PRUNE_RETRY_MAX_ATTEMPTS = 3;
698
1477
  const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
699
1478
  // DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
700
1479
  const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
1480
+ const REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL = 1000;
701
1481
  const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
702
1482
  const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
703
1483
  const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
@@ -876,11 +1656,23 @@ let SharedLog = (() => {
876
1656
  _nativeRangePlanner;
877
1657
  _nativeSharedLogState;
878
1658
  _nativeBackbone;
1659
+ _nativeDurableCommitFailure;
1660
+ _nativeDurableRecoveryReadyForReopen = false;
1661
+ _nativeDurableRecoveryCids = new Set();
879
1662
  _wireSyncSession;
880
1663
  _nativeBackboneCoordinatePersistence;
1664
+ _nativeBackboneCoordinatePersistenceStore;
1665
+ _nativeBackboneDropStarted = false;
881
1666
  _nativeBackboneCoordinateJournalLastFlushMs = 0;
1667
+ _nativeStrictDurableTransactionTail;
1668
+ _nativeStrictDurableTransactions;
1669
+ _nativeStrictDurableTransactionJournalState;
1670
+ _nativeStrictDurableDocumentRecoveryDeferred = false;
1671
+ _nativeStrictDurableTransactionsClosing = false;
1672
+ _nativeStrictDurableTransactionFailure;
882
1673
  _defaultAppendReplicaMetadataCache;
883
1674
  _residentEntryCoordinatesByHash;
1675
+ _nativeCoordinateMutationGenerations;
884
1676
  coordinateToHash;
885
1677
  recentlyRebalanced;
886
1678
  uniqueReplicators;
@@ -923,77 +1715,816 @@ let SharedLog = (() => {
923
1715
  _replicatorLivenessFailures;
924
1716
  _replicatorLastActivityAt;
925
1717
  remoteBlocks;
926
- openTime;
927
- oldestOpenTime;
928
- keep;
929
- // A fn that we can call many times that recalculates the participation role
930
- rebalanceParticipationDebounced;
931
- // A fn for debouncing the calls for pruning
932
- pruneDebouncedFn;
933
- responseToPruneDebouncedFn;
934
- get _requestIPruneSent() {
935
- return this._checkedPrune.requestIPruneSent;
1718
+ throwIfNativeDurableCommitFailed() {
1719
+ if (this._nativeStrictDurableTransactionFailure !== undefined) {
1720
+ throw new Error("Native durable transaction recovery is required before another mutation", { cause: this._nativeStrictDurableTransactionFailure });
1721
+ }
1722
+ const wrapperFailure = this.remoteBlocks?.localStore?.getNativeDurableCommitFailure?.();
1723
+ if (wrapperFailure) {
1724
+ this._nativeDurableCommitFailure ??= wrapperFailure;
1725
+ }
1726
+ if (this._nativeDurableCommitFailure) {
1727
+ throw this._nativeDurableCommitFailure;
1728
+ }
936
1729
  }
937
- get _requestIPruneResponseReplicatorSet() {
938
- return this._checkedPrune.responseReplicatorSet;
1730
+ poisonNativeStrictDurableTransaction(cause) {
1731
+ this._nativeStrictDurableTransactionFailure ??= cause;
1732
+ this.log?.entryIndex?.poisonNativeDurableTransactionMutations(this._nativeStrictDurableTransactionFailure);
939
1733
  }
940
- get _checkedPruneRetries() {
941
- return this._checkedPrune.retries;
1734
+ clearNativeStrictDurableTransactionFailure() {
1735
+ this._nativeStrictDurableTransactionFailure = undefined;
1736
+ this.log?.entryIndex?.clearNativeDurableTransactionMutationFailure();
942
1737
  }
943
- replicationChangeDebounceFn;
944
- _repairRetryTimers;
945
- _recentRepairDispatch;
946
- _repairSweepRunning;
947
- _repairSweepPendingModes;
948
- _repairSweepPendingPeersByMode;
949
- _repairFrontierByMode;
950
- _repairFrontierActiveTargetsByMode;
951
- _repairFrontierBypassKnownPeersByMode;
952
- _repairSweepOptimisticGidPeersPending;
953
- _entryKnownPeers;
954
- _entryKnownPeerObservedAt;
955
- _joinAuthoritativeRepairTimersByDelay;
956
- _joinAuthoritativeRepairPeersByDelay;
957
- _assumeSyncedRepairSuppressedUntil;
958
- _appendBackfillTimer;
959
- _appendBackfillPendingByTarget;
960
- _repairMetrics;
961
- _topicSubscribersCache;
962
- _leaderSelectionContextCache;
963
- // Sync capability bits advertised by peers (SyncCapabilitiesMessage), keyed
964
- // by public key hash. Entries are dropped on unsubscribe/disconnect.
965
- _peerSyncCapabilities;
966
- // Pending live raw exchange-head gossip, coalesced per recipient set and
967
- // flushed at the end of the current event-loop turn (or when a batch cap
968
- // is hit). Only used when every recipient advertised raw capability.
969
- _liveRawGossipBatches;
970
- _liveRawGossipFlushScheduled;
971
- // regular distribution checks
972
- distributeQueue;
973
- syncronizer;
974
- replicas;
975
- cpuUsage;
976
- _lastLocalAppendAt;
977
- adaptiveRebalanceIdleMs;
978
- timeUntilRoleMaturity;
979
- waitForReplicatorTimeout;
980
- waitForReplicatorRequestIntervalMs;
981
- waitForReplicatorRequestMaxAttempts;
982
- waitForPruneDelay;
983
- distributionDebounceTime;
984
- repairSweepTargetBufferSize;
985
- replicationController;
986
- history;
987
- domain;
988
- indexableDomain;
989
- interval;
990
- constructor(properties) {
991
- super();
992
- this.log = new Log(properties);
993
- this.rpc = new RPC();
994
- this._checkedPrune = new CheckedPruneCoordinator();
995
- this._pendingIHave = new Map();
996
- this.latestReplicationInfoMessage = new Map();
1738
+ failNativeDurableCommit(cause, options) {
1739
+ this.ensureNativeDurabilityRuntimeState();
1740
+ for (const cid of options?.committedCids ?? []) {
1741
+ this._nativeDurableRecoveryCids.add(cid);
1742
+ }
1743
+ if (cause instanceof NativeDurableCommitError) {
1744
+ cause.addCommitContext(options, { preferIncomingOrder: true });
1745
+ }
1746
+ this._nativeDurableCommitFailure ??=
1747
+ cause instanceof NativeDurableCommitError
1748
+ ? cause
1749
+ : new NativeDurableCommitError(cause, options);
1750
+ this._nativeDurableCommitFailure.addCommitContext(options, {
1751
+ preferIncomingOrder: true,
1752
+ });
1753
+ throw this._nativeDurableCommitFailure;
1754
+ }
1755
+ snapshotNativeBackboneDocument(input) {
1756
+ const backbone = this._nativeBackbone;
1757
+ if (!backbone || !input)
1758
+ return undefined;
1759
+ const value = backbone.documentValueBytes(input.key);
1760
+ return {
1761
+ key: input.key,
1762
+ value: value ? new Uint8Array(value) : undefined,
1763
+ byteElementIndexLimit: input.byteElementIndexLimit ?? 0,
1764
+ };
1765
+ }
1766
+ restoreNativeBackboneDocument(rollback) {
1767
+ const backbone = this._nativeBackbone;
1768
+ if (!backbone)
1769
+ return;
1770
+ backbone.deleteDocument(rollback.key);
1771
+ if (rollback.value) {
1772
+ // documentValueBytes returns the complete stored encoding, so it can be
1773
+ // restored as one prefix with an empty suffix.
1774
+ backbone.putDocumentEncodedPartsStored(rollback.key, rollback.value, new Uint8Array(), rollback.byteElementIndexLimit);
1775
+ }
1776
+ }
1777
+ parseNativeStrictDurableTransactionJournalRecord(bytes, slot) {
1778
+ if (bytes === undefined) {
1779
+ return undefined;
1780
+ }
1781
+ // The pre-journal implementation represented a cleared intent as an empty
1782
+ // primary file. Treat it as generation zero so the first framed update is
1783
+ // written to the other slot and can never destroy the only valid state.
1784
+ if (bytes.byteLength === 0) {
1785
+ return { sequence: 0, slot };
1786
+ }
1787
+ let parsed;
1788
+ try {
1789
+ parsed = JSON.parse(new TextDecoder().decode(bytes));
1790
+ }
1791
+ catch (error) {
1792
+ throw new Error("Invalid native durable transaction journal JSON", {
1793
+ cause: error,
1794
+ });
1795
+ }
1796
+ if (!parsed || typeof parsed !== "object") {
1797
+ throw new Error("Invalid native durable transaction journal record");
1798
+ }
1799
+ const candidate = parsed;
1800
+ if (candidate.format === NATIVE_STRICT_DURABLE_TRANSACTION_JOURNAL_FORMAT) {
1801
+ if (candidate.version !== 1 ||
1802
+ !Number.isSafeInteger(candidate.sequence) ||
1803
+ (candidate.sequence ?? -1) < 1 ||
1804
+ (candidate.state !== "intent" && candidate.state !== "cleared") ||
1805
+ typeof candidate.checksum !== "string") {
1806
+ throw new Error("Invalid native durable transaction journal frame");
1807
+ }
1808
+ const intent = candidate.intent ?? null;
1809
+ if ((candidate.state === "intent" && intent?.version !== 1) ||
1810
+ (candidate.state === "cleared" && intent !== null)) {
1811
+ throw new Error("Invalid native durable transaction journal state");
1812
+ }
1813
+ const body = nativeStrictDurableTransactionJournalBody(candidate.sequence, intent ?? undefined);
1814
+ const checksum = toHexString(sha256Sync(nativeStrictDurableTransactionJournalBodyBytes(body)));
1815
+ if (checksum !== candidate.checksum) {
1816
+ throw new Error("Native durable transaction journal checksum mismatch");
1817
+ }
1818
+ return {
1819
+ sequence: candidate.sequence,
1820
+ slot,
1821
+ intent: intent ?? undefined,
1822
+ };
1823
+ }
1824
+ // Backward compatibility with the original single raw-JSON intent. A
1825
+ // framed generation always sorts after this synthetic generation zero.
1826
+ const legacy = parsed;
1827
+ if (legacy.version !== 1) {
1828
+ throw new Error("Unsupported native durable transaction recovery intent");
1829
+ }
1830
+ return {
1831
+ sequence: 0,
1832
+ slot,
1833
+ intent: legacy,
1834
+ };
1835
+ }
1836
+ async loadNativeStrictDurableTransactionJournalState() {
1837
+ if (this._nativeStrictDurableTransactionJournalState) {
1838
+ return this._nativeStrictDurableTransactionJournalState;
1839
+ }
1840
+ const store = this._nativeBackboneCoordinatePersistenceStore;
1841
+ if (!store) {
1842
+ return (this._nativeStrictDurableTransactionJournalState = {
1843
+ sequence: 0,
1844
+ slot: 0,
1845
+ });
1846
+ }
1847
+ const bytes = await Promise.all(NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILES.map((name) => store.read(name)));
1848
+ const valid = [];
1849
+ const errors = [];
1850
+ for (let index = 0; index < bytes.length; index++) {
1851
+ try {
1852
+ const state = this.parseNativeStrictDurableTransactionJournalRecord(bytes[index], index);
1853
+ if (state)
1854
+ valid.push(state);
1855
+ }
1856
+ catch (error) {
1857
+ errors.push(error);
1858
+ }
1859
+ }
1860
+ if (valid.length === 0) {
1861
+ if (errors.length > 0) {
1862
+ throw new AggregateError(errors, "No valid native durable transaction journal generation remains");
1863
+ }
1864
+ // A completely new store has an implicit cleared generation. Before the
1865
+ // first intent is written we materialize this baseline in one slot, so a
1866
+ // corrupt sole slot can never be confused with a safe first-write tear (or
1867
+ // with a torn legacy single-file intent).
1868
+ return (this._nativeStrictDurableTransactionJournalState = {
1869
+ sequence: 0,
1870
+ slot: 0,
1871
+ implicit: true,
1872
+ });
1873
+ }
1874
+ valid.sort((left, right) => left.sequence - right.sequence || left.slot - right.slot);
1875
+ return (this._nativeStrictDurableTransactionJournalState = valid.at(-1));
1876
+ }
1877
+ async writeNativeStrictDurableTransactionIntent(intent) {
1878
+ const store = this._nativeBackboneCoordinatePersistenceStore;
1879
+ if (!store) {
1880
+ return;
1881
+ }
1882
+ let previous = await this.loadNativeStrictDurableTransactionJournalState();
1883
+ if (previous.implicit) {
1884
+ const baselineSequence = previous.sequence + 1;
1885
+ const baselineSlot = (previous.slot === 0 ? 1 : 0);
1886
+ const baselineFile = NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILES[baselineSlot];
1887
+ await store.write(baselineFile, nativeStrictDurableTransactionJournalRecordBytes(baselineSequence, undefined));
1888
+ await this.barrierNativeStrictDurableStore(store, baselineFile);
1889
+ previous = {
1890
+ sequence: baselineSequence,
1891
+ slot: baselineSlot,
1892
+ };
1893
+ this._nativeStrictDurableTransactionJournalState = previous;
1894
+ }
1895
+ const sequence = previous.sequence + 1;
1896
+ const slot = (previous.slot === 0 ? 1 : 0);
1897
+ const bytes = nativeStrictDurableTransactionJournalRecordBytes(sequence, intent);
1898
+ const file = NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILES[slot];
1899
+ // Alternate slots. If this write is interrupted or torn, the previous
1900
+ // checksummed generation remains untouched and recovery ignores the invalid
1901
+ // newer slot. A durable shared-log generation requires an explicit physical
1902
+ // barrier before this generation can become recovery-authoritative.
1903
+ await store.write(file, bytes);
1904
+ await this.barrierNativeStrictDurableStore(store, file);
1905
+ this._nativeStrictDurableTransactionJournalState = {
1906
+ sequence,
1907
+ slot,
1908
+ intent,
1909
+ };
1910
+ }
1911
+ async barrierNativeStrictDurableStore(store, file) {
1912
+ if (this.node.directory != null) {
1913
+ if (typeof store.durableBarrier !== "function") {
1914
+ throw new Error("Durable native coordinate persistence does not expose a physical durability barrier");
1915
+ }
1916
+ await store.durableBarrier(file);
1917
+ return;
1918
+ }
1919
+ // Memory-only operation has no durable lower marker. Preserve compatibility
1920
+ // for transient adapters while still using a real barrier when they expose it.
1921
+ if (store.durableBarrier) {
1922
+ await store.durableBarrier(file);
1923
+ }
1924
+ else {
1925
+ await store.flush?.(file);
1926
+ }
1927
+ }
1928
+ async beginNativeStrictDurableTransaction(documents) {
1929
+ this.throwIfNativeDurableCommitFailed();
1930
+ if (this._nativeStrictDurableTransactionsClosing) {
1931
+ throw new Error("Shared log is closing");
1932
+ }
1933
+ const previous = this._nativeStrictDurableTransactionTail ?? Promise.resolve();
1934
+ let release;
1935
+ const held = new Promise((resolve) => {
1936
+ release = resolve;
1937
+ });
1938
+ this._nativeStrictDurableTransactionTail = previous.then(() => held);
1939
+ await previous;
1940
+ try {
1941
+ this.throwIfNativeDurableCommitFailed();
1942
+ if (this._nativeStrictDurableTransactionsClosing) {
1943
+ throw new Error("Shared log is closing");
1944
+ }
1945
+ }
1946
+ catch (error) {
1947
+ release();
1948
+ throw error;
1949
+ }
1950
+ const handle = {
1951
+ intent: {
1952
+ version: 1,
1953
+ lowerMarkerCommitted: false,
1954
+ appendHashes: [],
1955
+ trimHashes: [],
1956
+ coordinateDeleteHashes: [],
1957
+ lowerIndexRows: [],
1958
+ coordinates: [],
1959
+ documents: documents.map((document) => ({
1960
+ key: document.key,
1961
+ value: document.value ? [...document.value] : undefined,
1962
+ byteElementIndexLimit: document.byteElementIndexLimit,
1963
+ })),
1964
+ },
1965
+ release,
1966
+ released: false,
1967
+ };
1968
+ (this._nativeStrictDurableTransactions ??= new Set()).add(handle);
1969
+ try {
1970
+ await this.writeNativeStrictDurableTransactionIntent(handle.intent);
1971
+ return handle;
1972
+ }
1973
+ catch (error) {
1974
+ handle.released = true;
1975
+ this._nativeStrictDurableTransactions.delete(handle);
1976
+ handle.release();
1977
+ throw error;
1978
+ }
1979
+ }
1980
+ async setNativeStrictDurableTransactionOperation(handle, appendHashes, trimHashes, coordinateRollback, coordinateDeleteHashes = []) {
1981
+ if (!handle) {
1982
+ return;
1983
+ }
1984
+ handle.intent.appendHashes = [...new Set(appendHashes.filter(Boolean))];
1985
+ handle.intent.trimHashes = [...new Set(trimHashes.filter(Boolean))];
1986
+ handle.intent.coordinateDeleteHashes = [
1987
+ ...new Set(coordinateDeleteHashes.filter(Boolean)),
1988
+ ];
1989
+ const lowerHashes = [
1990
+ ...new Set([
1991
+ ...handle.intent.appendHashes,
1992
+ ...handle.intent.trimHashes,
1993
+ ...(coordinateRollback?.hashes ?? []),
1994
+ ...handle.intent.coordinateDeleteHashes,
1995
+ ]),
1996
+ ];
1997
+ if (handle.lowerHashMutationLockOwner) {
1998
+ throw new Error("Native durable transaction operation is already locked");
1999
+ }
2000
+ // This lease is shared with the lower native transaction and every ordinary
2001
+ // EntryIndex mutation. Acquire it before reading any before-image, then hold
2002
+ // it through marker acknowledgement or exact compensation.
2003
+ handle.lowerHashMutationLockOwner =
2004
+ await this.log.entryIndex.acquireHashMutationLocks(lowerHashes);
2005
+ handle.intent.lowerIndexRows = await Promise.all(lowerHashes.map(async (hash) => {
2006
+ // EntryIndex publication may still live only in its pending generation.
2007
+ // Snapshot the exact logical row that the marker phase will consume,
2008
+ // rather than only the durable index, so a pre-marker crash can restore a
2009
+ // pending external-next head instead of treating it as previously absent.
2010
+ const previous = (await this.log.entryIndex.getShallow(hash))?.value;
2011
+ return {
2012
+ hash,
2013
+ before: previous ? [...serialize(previous)] : undefined,
2014
+ };
2015
+ }));
2016
+ handle.intent.coordinates = coordinateRollback
2017
+ ? [...coordinateRollback.hashes].map((hash) => {
2018
+ const previous = coordinateRollback.entries.get(hash);
2019
+ if (!previous) {
2020
+ return { hash };
2021
+ }
2022
+ const materialized = this.materializeResidentCoordinateEntry(previous);
2023
+ return {
2024
+ hash,
2025
+ value: {
2026
+ hashNumber: materialized.hashNumber.toString(),
2027
+ gid: materialized.gid,
2028
+ coordinates: materialized.coordinates.map((value) => value.toString()),
2029
+ wallTime: materialized.wallTime.toString(),
2030
+ assignedToRangeBoundary: materialized.assignedToRangeBoundary,
2031
+ metaBytes: [...materialized.getMetaBytes()],
2032
+ },
2033
+ };
2034
+ })
2035
+ : [];
2036
+ await this.writeNativeStrictDurableTransactionIntent(handle.intent);
2037
+ }
2038
+ async setNativeStrictDurableTransactionExpectedRows(handle, rows) {
2039
+ if (!handle) {
2040
+ return;
2041
+ }
2042
+ if (!handle.lowerHashMutationLockOwner) {
2043
+ throw new Error("Native durable transaction has no lower hash lock owner");
2044
+ }
2045
+ this.log.entryIndex.assertHashMutationLocks(handle.lowerHashMutationLockOwner, rows.flatMap((row) => [row.hash, ...row.meta.next]));
2046
+ const rowsByHash = new Map(handle.intent.lowerIndexRows.map((row) => [row.hash, row]));
2047
+ for (const row of rows) {
2048
+ const intentRow = rowsByHash.get(row.hash);
2049
+ if (intentRow) {
2050
+ intentRow.after = [...serialize(row)];
2051
+ }
2052
+ }
2053
+ for (const nextHash of new Set(rows.flatMap((row) => row.meta.next))) {
2054
+ const existingIntentRow = rowsByHash.get(nextHash);
2055
+ if (existingIntentRow) {
2056
+ if (!existingIntentRow.after && existingIntentRow.before) {
2057
+ const after = deserialize(Uint8Array.from(existingIntentRow.before), ShallowEntry);
2058
+ after.head = false;
2059
+ existingIntentRow.after = [...serialize(after)];
2060
+ }
2061
+ continue;
2062
+ }
2063
+ const previous = (await this.log.entryIndex.getShallow(nextHash))?.value;
2064
+ if (!previous) {
2065
+ continue;
2066
+ }
2067
+ const after = deserialize(serialize(previous), ShallowEntry);
2068
+ after.head = false;
2069
+ const intentRow = {
2070
+ hash: nextHash,
2071
+ before: [...serialize(previous)],
2072
+ after: [...serialize(after)],
2073
+ };
2074
+ handle.intent.lowerIndexRows.push(intentRow);
2075
+ rowsByHash.set(nextHash, intentRow);
2076
+ }
2077
+ handle.intent.coordinateDeleteHashes = [
2078
+ ...new Set([
2079
+ ...(handle.intent.coordinateDeleteHashes ?? []),
2080
+ ...handle.intent.trimHashes,
2081
+ ...rows.flatMap((row) => row.meta.next),
2082
+ ]),
2083
+ ];
2084
+ await this.writeNativeStrictDurableTransactionIntent(handle.intent);
2085
+ }
2086
+ async markNativeStrictDurableTransactionLowerMarker(handle) {
2087
+ if (!handle || handle.released) {
2088
+ return;
2089
+ }
2090
+ handle.intent.lowerMarkerCommitted = true;
2091
+ await this.writeNativeStrictDurableTransactionIntent(handle.intent);
2092
+ }
2093
+ async markNativeStrictDurableTransactionRollback(handle) {
2094
+ if (!handle || handle.released) {
2095
+ return;
2096
+ }
2097
+ const previousMarker = handle.intent.lowerMarkerCommitted;
2098
+ handle.intent.lowerMarkerCommitted = false;
2099
+ try {
2100
+ await this.writeNativeStrictDurableTransactionIntent(handle.intent);
2101
+ }
2102
+ catch (error) {
2103
+ // The last valid generation may still contain a true marker. Preserve
2104
+ // that in-memory knowledge and keep the handle held until the caller has
2105
+ // retained the lower finalizer. Releasing first would let concurrent close
2106
+ // compensate lower facts while recovery still sees a committed marker.
2107
+ handle.intent.lowerMarkerCommitted = previousMarker;
2108
+ this.poisonNativeStrictDurableTransaction(error);
2109
+ throw error;
2110
+ }
2111
+ }
2112
+ async completeNativeStrictDurableTrimCleanup(intent, committed = intent.lowerMarkerCommitted === true, reconstructMissing = false) {
2113
+ if (!committed || intent.trimHashes.length === 0) {
2114
+ return;
2115
+ }
2116
+ const localStore = this.remoteBlocks?.localStore;
2117
+ if (typeof localStore?.completeCommittedNativeDeleteCleanup === "function") {
2118
+ await localStore.completeCommittedNativeDeleteCleanup(intent.trimHashes, {
2119
+ reconstructMissing,
2120
+ });
2121
+ }
2122
+ }
2123
+ async completeNativeStrictDurableCoordinateCleanup(intent, committed = intent.lowerMarkerCommitted === true) {
2124
+ const hashes = intent.coordinateDeleteHashes ?? [];
2125
+ if (!committed || hashes.length === 0) {
2126
+ return;
2127
+ }
2128
+ await this.deleteCoordinatesForHashes(hashes);
2129
+ const flushed = this.flushNativeBackboneCoordinateJournal();
2130
+ if (isPromiseLike(flushed)) {
2131
+ await flushed;
2132
+ }
2133
+ }
2134
+ async completeNativeStrictDurableTransaction(handle) {
2135
+ if (!handle || handle.released) {
2136
+ return;
2137
+ }
2138
+ try {
2139
+ await this.completeNativeStrictDurableCoordinateCleanup(handle.intent);
2140
+ await this.completeNativeStrictDurableTrimCleanup(handle.intent);
2141
+ await this.writeNativeStrictDurableTransactionIntent(undefined);
2142
+ this.clearNativeStrictDurableTransactionFailure();
2143
+ }
2144
+ catch (error) {
2145
+ // The lower marker may already be acknowledged. Retain the intent and
2146
+ // reject every later mutation until reopen can finish recovery; allowing a
2147
+ // new transaction to overwrite this generation would make rollback/GC debt
2148
+ // ambiguous and can erase acknowledged data.
2149
+ this.poisonNativeStrictDurableTransaction(error);
2150
+ throw error;
2151
+ }
2152
+ finally {
2153
+ if (handle.lowerHashMutationLockOwner) {
2154
+ this.log.entryIndex.releaseHashMutationLocks(handle.lowerHashMutationLockOwner);
2155
+ handle.lowerHashMutationLockOwner = undefined;
2156
+ }
2157
+ handle.released = true;
2158
+ this._nativeStrictDurableTransactions?.delete(handle);
2159
+ handle.release();
2160
+ }
2161
+ }
2162
+ releaseNativeStrictDurableTransaction(handle, cause = new Error("Native durable transaction intent was retained for recovery")) {
2163
+ if (!handle || handle.released) {
2164
+ return;
2165
+ }
2166
+ this.poisonNativeStrictDurableTransaction(cause);
2167
+ if (handle.lowerHashMutationLockOwner) {
2168
+ this.log.entryIndex.releaseHashMutationLocks(handle.lowerHashMutationLockOwner);
2169
+ handle.lowerHashMutationLockOwner = undefined;
2170
+ }
2171
+ handle.released = true;
2172
+ this._nativeStrictDurableTransactions?.delete(handle);
2173
+ handle.release();
2174
+ }
2175
+ retainNativeStrictDurableTransactionAfterMarkerFailure(handle, finalizer, cause) {
2176
+ const failures = [cause];
2177
+ try {
2178
+ finalizer?.retainForRecovery();
2179
+ }
2180
+ catch (error) {
2181
+ failures.push(error);
2182
+ }
2183
+ finally {
2184
+ // retainForRecovery finalizes its lower transaction even when one of its
2185
+ // internal cleanup steps reports an error. Only release the strict handle
2186
+ // after that synchronous state transition has been attempted.
2187
+ this.releaseNativeStrictDurableTransaction(handle, cause);
2188
+ }
2189
+ return failures;
2190
+ }
2191
+ async settleNativeStrictDurableTransactionsForClose() {
2192
+ while ((this._nativeStrictDurableTransactions?.size ?? 0) > 0) {
2193
+ const tail = this._nativeStrictDurableTransactionTail;
2194
+ if (!tail) {
2195
+ throw new Error("Native strict durable transaction has no settlement tail");
2196
+ }
2197
+ // A close racing an acknowledged lower marker must not release the strict
2198
+ // handle and let Log.close() compensate while the on-disk intent still says
2199
+ // committed. Wait until the owner either retires the intent or deliberately
2200
+ // retains it for recovery before closing the lower log or persistence stores.
2201
+ await tail;
2202
+ }
2203
+ }
2204
+ async recoverNativeStrictDurableTransactionIntent(documentIndexReady = false) {
2205
+ const store = this._nativeBackboneCoordinatePersistenceStore;
2206
+ if (!store || !this._nativeBackbone) {
2207
+ this._nativeStrictDurableDocumentRecoveryDeferred = false;
2208
+ return true;
2209
+ }
2210
+ const journalState = await this.loadNativeStrictDurableTransactionJournalState();
2211
+ const intent = journalState.intent;
2212
+ if (!intent) {
2213
+ this._nativeStrictDurableDocumentRecoveryDeferred = false;
2214
+ this.clearNativeStrictDurableTransactionFailure();
2215
+ return true;
2216
+ }
2217
+ if (intent.version !== 1) {
2218
+ throw new Error("Unsupported native durable transaction recovery intent");
2219
+ }
2220
+ intent.trimHashes ??= [];
2221
+ intent.coordinateDeleteHashes ??= [];
2222
+ intent.lowerIndexRows ??= [];
2223
+ intent.coordinates ??= [];
2224
+ const bytesEqual = (left, right) => {
2225
+ if (!left || !right) {
2226
+ return left === undefined && right === undefined;
2227
+ }
2228
+ if (left.byteLength !== right.length) {
2229
+ return false;
2230
+ }
2231
+ for (let index = 0; index < left.byteLength; index++) {
2232
+ if (left[index] !== right[index]) {
2233
+ return false;
2234
+ }
2235
+ }
2236
+ return true;
2237
+ };
2238
+ const immutableRowEquals = (current, expected) => {
2239
+ if (!current || !expected) {
2240
+ return current === undefined && expected === undefined;
2241
+ }
2242
+ const currentRow = deserialize(current, ShallowEntry);
2243
+ const expectedRow = deserialize(Uint8Array.from(expected), ShallowEntry);
2244
+ // `head` is a mutable graph projection. Hash, payload size, and metadata
2245
+ // are content-addressed append identity and are safe marker evidence even
2246
+ // when a later acknowledged entry has demoted this row.
2247
+ currentRow.head = false;
2248
+ expectedRow.head = false;
2249
+ return bytesEqual(serialize(currentRow), [...serialize(expectedRow)]);
2250
+ };
2251
+ const currentLowerRows = new Map();
2252
+ for (const row of intent.lowerIndexRows) {
2253
+ const current = (await this.log.entryIndex.properties.index.get(toId(row.hash)))?.value;
2254
+ currentLowerRows.set(row.hash, current ? serialize(current) : undefined);
2255
+ }
2256
+ const trimHashes = new Set(intent.trimHashes);
2257
+ const retainedMarkerRows = intent.lowerIndexRows.filter((row) => intent.appendHashes.includes(row.hash) &&
2258
+ !trimHashes.has(row.hash) &&
2259
+ row.after !== undefined);
2260
+ // Only a row known absent in the before-image is an unambiguous lower commit
2261
+ // marker. An existing content-addressed row can equal the after-image once
2262
+ // mutable `head` is ignored even before this transaction mutated anything.
2263
+ const expectedMarkerRows = retainedMarkerRows.filter((row) => row.before === undefined);
2264
+ let lowerMarkerCommitted = intent.lowerMarkerCommitted === true ||
2265
+ (expectedMarkerRows.length > 0 &&
2266
+ expectedMarkerRows.every((row) => immutableRowEquals(currentLowerRows.get(row.hash), row.after)));
2267
+ if (!lowerMarkerCommitted &&
2268
+ !documentIndexReady &&
2269
+ intent.documents.some((document) => document.value !== undefined)) {
2270
+ // SharedLog opens before Documents can attach its schema-aware native
2271
+ // index. Restoring an encoded before-image requires that schema. Keep the
2272
+ // intent authoritative and mutations poisoned until Documents has attached
2273
+ // the index and explicitly resumes recovery.
2274
+ this._nativeStrictDurableDocumentRecoveryDeferred = true;
2275
+ this.poisonNativeStrictDurableTransaction(new Error("Native strict durable document recovery is waiting for its document index"));
2276
+ return false;
2277
+ }
2278
+ const lowerIndex = this.log.entryIndex.properties
2279
+ .index;
2280
+ const deleteLowerIndexHash = async (hash) => {
2281
+ if (lowerIndex.delIds) {
2282
+ await lowerIndex.delIds([hash]);
2283
+ }
2284
+ else if (lowerIndex.delIdsNoReturn) {
2285
+ await lowerIndex.delIdsNoReturn([hash]);
2286
+ }
2287
+ else {
2288
+ await lowerIndex.del({ query: { hash } });
2289
+ }
2290
+ };
2291
+ let lowerIndexChanged = false;
2292
+ if (lowerMarkerCommitted) {
2293
+ for (const row of intent.lowerIndexRows) {
2294
+ if (trimHashes.has(row.hash) || !row.after) {
2295
+ continue;
2296
+ }
2297
+ const current = currentLowerRows.get(row.hash);
2298
+ if (immutableRowEquals(current, row.after)) {
2299
+ // Preserve the current mutable head projection. It may include a later
2300
+ // acknowledged Y -> X demotion that must survive recovery.
2301
+ continue;
2302
+ }
2303
+ if (!intent.appendHashes.includes(row.hash) || current !== undefined) {
2304
+ // External-next rows are not resurrected over a later delete, and a
2305
+ // conflicting present content-addressed row is never overwritten.
2306
+ continue;
2307
+ }
2308
+ await lowerIndex.put(deserialize(Uint8Array.from(row.after), ShallowEntry));
2309
+ lowerIndexChanged = true;
2310
+ }
2311
+ for (const hash of intent.trimHashes) {
2312
+ const current = await lowerIndex.get(toId(hash));
2313
+ const intentRow = intent.lowerIndexRows.find((row) => row.hash === hash);
2314
+ if (current &&
2315
+ intentRow?.before &&
2316
+ bytesEqual(serialize(current.value), intentRow.before)) {
2317
+ await deleteLowerIndexHash(hash);
2318
+ lowerIndexChanged = true;
2319
+ }
2320
+ }
2321
+ }
2322
+ else {
2323
+ for (const row of intent.lowerIndexRows) {
2324
+ const current = currentLowerRows.get(row.hash);
2325
+ if (bytesEqual(current, row.before)) {
2326
+ continue;
2327
+ }
2328
+ // Exact after-image CAS: a later mutation (including only a `head`
2329
+ // change) owns the row and must not be erased or overwritten by recovery.
2330
+ if (!bytesEqual(current, row.after)) {
2331
+ continue;
2332
+ }
2333
+ if (row.before) {
2334
+ await lowerIndex.put(deserialize(Uint8Array.from(row.before), ShallowEntry));
2335
+ }
2336
+ else {
2337
+ await deleteLowerIndexHash(row.hash);
2338
+ }
2339
+ lowerIndexChanged = true;
2340
+ }
2341
+ }
2342
+ if (lowerIndexChanged) {
2343
+ await this.log.entryIndex.init();
2344
+ }
2345
+ if (!lowerMarkerCommitted) {
2346
+ if (intent.coordinates.length > 0) {
2347
+ const mutationGenerations = (this._nativeCoordinateMutationGenerations ??= new Map());
2348
+ const rollback = {
2349
+ hashes: new Set(),
2350
+ entries: new Map(),
2351
+ generations: new Map(),
2352
+ };
2353
+ for (const coordinate of intent.coordinates) {
2354
+ rollback.hashes.add(coordinate.hash);
2355
+ const generation = (mutationGenerations.get(coordinate.hash) ?? 0) + 1;
2356
+ mutationGenerations.set(coordinate.hash, generation);
2357
+ rollback.generations.set(coordinate.hash, generation);
2358
+ if (coordinate.value) {
2359
+ const number = (value) => (this.domain.resolution === "u32"
2360
+ ? Number(value)
2361
+ : BigInt(value));
2362
+ rollback.entries.set(coordinate.hash, new this.indexableDomain.constructorEntry({
2363
+ hash: coordinate.hash,
2364
+ hashNumber: number(coordinate.value.hashNumber),
2365
+ gid: coordinate.value.gid,
2366
+ coordinates: coordinate.value.coordinates.map(number),
2367
+ wallTime: BigInt(coordinate.value.wallTime),
2368
+ assignedToRangeBoundary: coordinate.value.assignedToRangeBoundary,
2369
+ metaBytes: Uint8Array.from(coordinate.value.metaBytes),
2370
+ }));
2371
+ }
2372
+ }
2373
+ await this.rollbackNativeBackboneCoordinateAppendDurably("", rollback);
2374
+ }
2375
+ for (const document of intent.documents) {
2376
+ this.restoreNativeBackboneDocument({
2377
+ key: document.key,
2378
+ value: document.value ? Uint8Array.from(document.value) : undefined,
2379
+ byteElementIndexLimit: document.byteElementIndexLimit,
2380
+ });
2381
+ }
2382
+ const flushed = this.flushNativeBackboneCoordinateJournal();
2383
+ if (isPromiseLike(flushed)) {
2384
+ await flushed;
2385
+ }
2386
+ }
2387
+ if (lowerMarkerCommitted) {
2388
+ await this.completeNativeStrictDurableCoordinateCleanup(intent, true);
2389
+ await this.completeNativeStrictDurableTrimCleanup(intent, true, true);
2390
+ }
2391
+ await this.writeNativeStrictDurableTransactionIntent(undefined);
2392
+ this._nativeStrictDurableDocumentRecoveryDeferred = false;
2393
+ this.clearNativeStrictDurableTransactionFailure();
2394
+ return true;
2395
+ }
2396
+ /** @internal Complete a deferred rollback after Documents attaches its schema. */
2397
+ async finishNativeStrictDurableDocumentRecovery() {
2398
+ if (!this._nativeStrictDurableDocumentRecoveryDeferred) {
2399
+ return;
2400
+ }
2401
+ const completed = await this.recoverNativeStrictDurableTransactionIntent(true);
2402
+ if (!completed) {
2403
+ throw new Error("Native strict durable document recovery did not complete");
2404
+ }
2405
+ await this.reconcileNativeCoordinatesWithLowerCommitMarkers();
2406
+ }
2407
+ async rollbackFailedNativeBackboneTransaction(properties) {
2408
+ const backbone = this._nativeBackbone;
2409
+ if (!backbone)
2410
+ return;
2411
+ for (let index = properties.committedHashes.length - 1; index >= 0; index--) {
2412
+ const hash = properties.committedHashes[index];
2413
+ backbone.graph.delete(hash);
2414
+ this.rollbackNativeBackboneCoordinateAppend(hash, properties.coordinateEntries);
2415
+ }
2416
+ if (properties.restoreGraphFromIndex) {
2417
+ await this.log.entryIndex.restoreNativeGraphFromIndex();
2418
+ }
2419
+ else {
2420
+ if (properties.trimmedEntries?.length) {
2421
+ backbone.graph.putBatch(properties.trimmedEntries);
2422
+ }
2423
+ }
2424
+ for (const document of properties.documents ?? []) {
2425
+ this.restoreNativeBackboneDocument(document);
2426
+ }
2427
+ const flushed = this.flushNativeBackboneCoordinateJournal();
2428
+ if (isPromiseLike(flushed)) {
2429
+ await flushed;
2430
+ }
2431
+ if (properties.skipBlockCompensation) {
2432
+ return;
2433
+ }
2434
+ let compensated = false;
2435
+ try {
2436
+ if (properties.unmirroredBlockCompensation &&
2437
+ properties.durableWrapper?.rollbackUnmirroredNativeCommits) {
2438
+ await properties.durableWrapper.rollbackUnmirroredNativeCommits(properties.committedHashes, properties.trimmedEntries?.map((entry) => entry.hash));
2439
+ }
2440
+ else if (properties.durableWrapper?.rollbackFailedNativeCommits) {
2441
+ await properties.durableWrapper.rollbackFailedNativeCommits(properties.committedHashes, properties.trimmedEntries?.map((entry) => entry.hash));
2442
+ }
2443
+ else {
2444
+ await backbone.blocks.rmMany(properties.committedHashes);
2445
+ }
2446
+ compensated = true;
2447
+ }
2448
+ finally {
2449
+ this._nativeDurableRecoveryReadyForReopen = compensated;
2450
+ }
2451
+ }
2452
+ openTime;
2453
+ oldestOpenTime;
2454
+ keep;
2455
+ // A fn that we can call many times that recalculates the participation role
2456
+ rebalanceParticipationDebounced;
2457
+ replicationAnnouncementRetryDebounced;
2458
+ _replicationAnnouncementRetryPending;
2459
+ _replicationAnnouncementRetryGeneration;
2460
+ _replicationAnnouncementRetryController;
2461
+ // A fn for debouncing the calls for pruning
2462
+ pruneDebouncedFn;
2463
+ responseToPruneDebouncedFn;
2464
+ get _requestIPruneSent() {
2465
+ return this._checkedPrune.requestIPruneSent;
2466
+ }
2467
+ get _requestIPruneResponseReplicatorSet() {
2468
+ return this._checkedPrune.responseReplicatorSet;
2469
+ }
2470
+ get _checkedPruneRetries() {
2471
+ return this._checkedPrune.retries;
2472
+ }
2473
+ replicationChangeDebounceFn;
2474
+ _repairRetryTimers;
2475
+ _recentRepairDispatch;
2476
+ _repairSweepRunning;
2477
+ _repairSweepPendingModes;
2478
+ _repairSweepPendingPeersByMode;
2479
+ _repairFrontierByMode;
2480
+ _repairFrontierActiveTargetsByMode;
2481
+ _repairFrontierBypassKnownPeersByMode;
2482
+ _repairSweepOptimisticGidPeersPending;
2483
+ _entryKnownPeers;
2484
+ _entryKnownPeerObservedAt;
2485
+ _joinAuthoritativeRepairTimersByDelay;
2486
+ _joinAuthoritativeRepairPeersByDelay;
2487
+ _assumeSyncedRepairSuppressedUntil;
2488
+ _appendBackfillTimer;
2489
+ _appendBackfillPendingByTarget;
2490
+ _repairMetrics;
2491
+ _topicSubscribersCache;
2492
+ _leaderSelectionContextCache;
2493
+ // Sync capability bits advertised by peers (SyncCapabilitiesMessage), keyed
2494
+ // by public key hash. Entries are dropped on unsubscribe/disconnect.
2495
+ _peerSyncCapabilities;
2496
+ // Pending live raw exchange-head gossip, coalesced per recipient set and
2497
+ // flushed at the end of the current event-loop turn (or when a batch cap
2498
+ // is hit). Only used when every recipient advertised raw capability.
2499
+ _liveRawGossipBatches;
2500
+ _liveRawGossipFlushScheduled;
2501
+ // regular distribution checks
2502
+ distributeQueue;
2503
+ syncronizer;
2504
+ replicas;
2505
+ cpuUsage;
2506
+ _lastLocalAppendAt;
2507
+ adaptiveRebalanceIdleMs;
2508
+ timeUntilRoleMaturity;
2509
+ waitForReplicatorTimeout;
2510
+ waitForReplicatorRequestIntervalMs;
2511
+ waitForReplicatorRequestMaxAttempts;
2512
+ waitForPruneDelay;
2513
+ distributionDebounceTime;
2514
+ repairSweepTargetBufferSize;
2515
+ replicationController;
2516
+ history;
2517
+ domain;
2518
+ indexableDomain;
2519
+ interval;
2520
+ constructor(properties) {
2521
+ super();
2522
+ this.ensureNativeDurabilityRuntimeState();
2523
+ this.log = new Log(properties);
2524
+ this.rpc = new RPC();
2525
+ this._checkedPrune = new CheckedPruneCoordinator();
2526
+ this._pendingIHave = new Map();
2527
+ this.latestReplicationInfoMessage = new Map();
997
2528
  this._replicationInfoBlockedPeers = new Set();
998
2529
  this._replicationInfoRequestByPeer = new Map();
999
2530
  this._replicationInfoApplyQueueByPeer = new Map();
@@ -1025,9 +2556,23 @@ let SharedLog = (() => {
1025
2556
  this._replicatorLivenessCursor = 0;
1026
2557
  this._replicatorLivenessFailures = new Map();
1027
2558
  this._replicatorLastActivityAt = new Map();
2559
+ this._replicationAnnouncementRetryPending = false;
2560
+ this._replicationAnnouncementRetryGeneration = 0;
2561
+ this._replicationAnnouncementRetryController = new AbortController();
1028
2562
  this.pendingMaturity = new Map();
1029
2563
  this._closeController = new AbortController();
1030
2564
  }
2565
+ ensureNativeDurabilityRuntimeState() {
2566
+ // Program clones are borsh-created without running class field initializers.
2567
+ // Keep recovery state from an existing generation, while supplying fresh
2568
+ // defaults only when the runtime-only fields are absent.
2569
+ this._nativeDurableRecoveryReadyForReopen ??= false;
2570
+ this._nativeDurableRecoveryCids ??= new Set();
2571
+ this._nativeBackboneDropStarted ??= false;
2572
+ this._nativeBackboneCoordinateJournalLastFlushMs ??= 0;
2573
+ this._nativeStrictDurableDocumentRecoveryDeferred ??= false;
2574
+ this._nativeStrictDurableTransactionsClosing ??= false;
2575
+ }
1031
2576
  get compatibility() {
1032
2577
  return this._logProperties?.compatibility;
1033
2578
  }
@@ -2031,6 +3576,7 @@ let SharedLog = (() => {
2031
3576
  return undefined;
2032
3577
  }
2033
3578
  setupRebalanceDebounceFunction(interval = RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL) {
3579
+ this.rebalanceParticipationDebounced?.close();
2034
3580
  this.rebalanceParticipationDebounced = undefined;
2035
3581
  this.rebalanceParticipationDebounced = debounceFixedInterval(() => this.rebalanceParticipation(),
2036
3582
  /* Math.max(
@@ -2040,7 +3586,133 @@ let SharedLog = (() => {
2040
3586
  REBALANCE_DEBOUNCE_INTERVAL
2041
3587
  )
2042
3588
  ) */
2043
- interval);
3589
+ interval, // TODO make this dynamic on the number of replicators
3590
+ {
3591
+ onError: (error) => this.onRebalanceParticipationError(error),
3592
+ });
3593
+ }
3594
+ queueCurrentReplicationStateAnnouncementRetry(error) {
3595
+ if (this.closed ||
3596
+ this._closeController.signal.aborted ||
3597
+ this._replicationAnnouncementRetryController.signal.aborted ||
3598
+ !isTransientReplicationAnnouncementError(error)) {
3599
+ return false;
3600
+ }
3601
+ this._replicationAnnouncementRetryPending = true;
3602
+ void this.replicationAnnouncementRetryDebounced?.call();
3603
+ return true;
3604
+ }
3605
+ onRebalanceParticipationError(error) {
3606
+ if (this.closed ||
3607
+ isNotStartedError(error) ||
3608
+ (isTransientReplicationAnnouncementError(error) &&
3609
+ this._replicationAnnouncementRetryPending)) {
3610
+ return;
3611
+ }
3612
+ // Debounced invocations run from an un-awaited timer. Throwing here would
3613
+ // create an unhandled rejection (and a browser pageerror), so surface
3614
+ // unexpected failures through the logger instead.
3615
+ logger.error(error);
3616
+ }
3617
+ setupReplicationAnnouncementRetryFunction(interval = REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL) {
3618
+ this.replicationAnnouncementRetryDebounced?.close();
3619
+ this._replicationAnnouncementRetryController?.abort();
3620
+ this._replicationAnnouncementRetryController = new AbortController();
3621
+ this.replicationAnnouncementRetryDebounced = debounceFixedInterval(() => this.retryCurrentReplicationStateAnnouncement(), interval, {
3622
+ leading: false,
3623
+ onError: (error) => {
3624
+ if (this.closed ||
3625
+ this._closeController.signal.aborted ||
3626
+ isNotStartedError(error)) {
3627
+ return;
3628
+ }
3629
+ logger.error(error);
3630
+ },
3631
+ });
3632
+ }
3633
+ cancelCurrentReplicationStateAnnouncementRetry() {
3634
+ this._replicationAnnouncementRetryGeneration += 1;
3635
+ this._replicationAnnouncementRetryPending = false;
3636
+ this._replicationAnnouncementRetryController?.abort();
3637
+ this.replicationAnnouncementRetryDebounced?.close();
3638
+ }
3639
+ async sendReplicationAnnouncement(message) {
3640
+ // Advance before every post-mutation send, including successful ones. An
3641
+ // authoritative retry already in flight may have captured the previous
3642
+ // local state; the generation mismatch forces one more current snapshot
3643
+ // after that stale send settles.
3644
+ this._replicationAnnouncementRetryGeneration += 1;
3645
+ try {
3646
+ await this.rpc.send(message, {
3647
+ priority: CONVERGENCE_MESSAGE_PRIORITY,
3648
+ });
3649
+ }
3650
+ catch (error) {
3651
+ // The local replication-index mutation precedes all calls to this
3652
+ // wrapper. Preserve the explicit caller's rejection, but independently
3653
+ // schedule an authoritative snapshot so peers eventually observe the
3654
+ // already-committed local state.
3655
+ this.queueCurrentReplicationStateAnnouncementRetry(error);
3656
+ throw error;
3657
+ }
3658
+ }
3659
+ async retryCurrentReplicationStateAnnouncement() {
3660
+ const generation = this._replicationAnnouncementRetryGeneration;
3661
+ const controller = this._replicationAnnouncementRetryController;
3662
+ try {
3663
+ const segments = (await this.getMyReplicationSegments()).map((range) => range.toReplicationRange());
3664
+ if (this.closed ||
3665
+ this._closeController.signal.aborted ||
3666
+ controller.signal.aborted) {
3667
+ return;
3668
+ }
3669
+ if (generation !== this._replicationAnnouncementRetryGeneration) {
3670
+ void this.replicationAnnouncementRetryDebounced?.call();
3671
+ return;
3672
+ }
3673
+ await this.rpc.send(new AllReplicatingSegmentsMessage({ segments }), {
3674
+ priority: CONVERGENCE_MESSAGE_PRIORITY,
3675
+ signal: controller.signal,
3676
+ });
3677
+ }
3678
+ catch (error) {
3679
+ if (this.closed ||
3680
+ this._closeController.signal.aborted ||
3681
+ controller.signal.aborted) {
3682
+ return;
3683
+ }
3684
+ if (this.queueCurrentReplicationStateAnnouncementRetry(error)) {
3685
+ return;
3686
+ }
3687
+ if (generation === this._replicationAnnouncementRetryGeneration) {
3688
+ this._replicationAnnouncementRetryPending = false;
3689
+ }
3690
+ else {
3691
+ void this.replicationAnnouncementRetryDebounced?.call();
3692
+ }
3693
+ throw error;
3694
+ }
3695
+ if (this.closed ||
3696
+ this._closeController.signal.aborted ||
3697
+ controller.signal.aborted) {
3698
+ return;
3699
+ }
3700
+ // A newer mutation announcement may have started while this snapshot was
3701
+ // in flight. In that case keep the repair pending so the newer current
3702
+ // state is also announced in full, regardless of whether its incremental
3703
+ // send succeeded or failed.
3704
+ if (generation === this._replicationAnnouncementRetryGeneration) {
3705
+ this._replicationAnnouncementRetryPending = false;
3706
+ if (!this.closed &&
3707
+ !this._closeController.signal.aborted &&
3708
+ !controller.signal.aborted &&
3709
+ this._isAdaptiveReplicating) {
3710
+ void this.rebalanceParticipationDebounced?.call();
3711
+ }
3712
+ }
3713
+ else {
3714
+ void this.replicationAnnouncementRetryDebounced?.call();
3715
+ }
2044
3716
  }
2045
3717
  markLocalAppendActivity(timestamp = Date.now()) {
2046
3718
  this._lastLocalAppendAt = Math.max(this._lastLocalAppendAt ?? 0, timestamp);
@@ -2278,11 +3950,9 @@ let SharedLog = (() => {
2278
3950
  rebalance,
2279
3951
  });
2280
3952
  if (rangesToUnreplicate.length > 0) {
2281
- await this.rpc.send(new StoppedReplicating({
3953
+ await this.sendReplicationAnnouncement(new StoppedReplicating({
2282
3954
  segmentIds: rangesToUnreplicate.map((x) => x.id),
2283
- }), {
2284
- priority: CONVERGENCE_MESSAGE_PRIORITY,
2285
- });
3955
+ }));
2286
3956
  }
2287
3957
  return rangesToReplicate;
2288
3958
  }
@@ -2406,9 +4076,7 @@ let SharedLog = (() => {
2406
4076
  }
2407
4077
  const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(segmentIds, this.node.identity.publicKey);
2408
4078
  await this.removeReplicationRanges(rangesToRemove, this.node.identity.publicKey);
2409
- await this.rpc.send(new StoppedReplicating({ segmentIds }), {
2410
- priority: CONVERGENCE_MESSAGE_PRIORITY,
2411
- });
4079
+ await this.sendReplicationAnnouncement(new StoppedReplicating({ segmentIds }));
2412
4080
  }
2413
4081
  async removeReplicator(key, options) {
2414
4082
  const keyHash = typeof key === "string" ? key : key.hashcode();
@@ -2427,9 +4095,7 @@ let SharedLog = (() => {
2427
4095
  const isMe = this.node.identity.publicKey.hashcode() === keyHash;
2428
4096
  if (isMe) {
2429
4097
  // announce that we are no longer replicating
2430
- await this.rpc.send(new AllReplicatingSegmentsMessage({ segments: [] }), {
2431
- priority: CONVERGENCE_MESSAGE_PRIORITY,
2432
- });
4098
+ await this.sendReplicationAnnouncement(new AllReplicatingSegmentsMessage({ segments: [] }));
2433
4099
  }
2434
4100
  if (options?.noEvent !== true) {
2435
4101
  const publicKey = toLocalPublicSignKey(key);
@@ -2803,9 +4469,7 @@ let SharedLog = (() => {
2803
4469
  return options.announce(message);
2804
4470
  }
2805
4471
  else {
2806
- await this.rpc.send(message, {
2807
- priority: CONVERGENCE_MESSAGE_PRIORITY,
2808
- });
4472
+ await this.sendReplicationAnnouncement(message);
2809
4473
  }
2810
4474
  }
2811
4475
  }
@@ -4055,6 +5719,7 @@ let SharedLog = (() => {
4055
5719
  }
4056
5720
  }
4057
5721
  async append(data, options) {
5722
+ this.throwIfNativeDurableCommitFailed();
4058
5723
  if (this._isAdaptiveReplicating) {
4059
5724
  this.markLocalAppendActivity();
4060
5725
  }
@@ -4067,6 +5732,7 @@ let SharedLog = (() => {
4067
5732
  }
4068
5733
  // Trusted local append path for callers that already validated the entry.
4069
5734
  async appendLocallyValidated(data, options) {
5735
+ this.throwIfNativeDurableCommitFailed();
4070
5736
  if (options?.canAppend || options?.onChange) {
4071
5737
  throw new Error("appendLocallyValidated does not accept canAppend or onChange hooks");
4072
5738
  }
@@ -4084,6 +5750,7 @@ let SharedLog = (() => {
4084
5750
  }
4085
5751
  // Trusted local append path that lets the shared log own change application.
4086
5752
  async appendLocallyPrepared(data, options, properties) {
5753
+ this.throwIfNativeDurableCommitFailed();
4087
5754
  if (options?.canAppend || options?.onChange) {
4088
5755
  throw new Error("appendLocallyPrepared does not accept canAppend or onChange hooks");
4089
5756
  }
@@ -4221,6 +5888,7 @@ let SharedLog = (() => {
4221
5888
  }
4222
5889
  // Trusted local payload append path that keeps the public Entry lazy.
4223
5890
  appendLocallyPreparedPayloadCommitOnly(payloadData, options, properties) {
5891
+ this.throwIfNativeDurableCommitFailed();
4224
5892
  if (options?.canAppend || options?.onChange) {
4225
5893
  throw new Error("appendLocallyPreparedPayloadCommitOnly does not accept canAppend or onChange hooks");
4226
5894
  }
@@ -4250,6 +5918,7 @@ let SharedLog = (() => {
4250
5918
  }
4251
5919
  // Strict native document path. Never falls back to compatibility append.
4252
5920
  appendStrictNativeDocumentPayloadCommitOnly(payloadData, options, properties) {
5921
+ this.throwIfNativeDurableCommitFailed();
4253
5922
  if (options?.canAppend || options?.onChange) {
4254
5923
  throw new Error("appendStrictNativeDocumentPayloadCommitOnly does not accept canAppend or onChange hooks");
4255
5924
  }
@@ -4316,116 +5985,300 @@ let SharedLog = (() => {
4316
5985
  // commit-only block can be mirrored to durable WITHOUT routing it through
4317
5986
  // the log's finishBlocks/putKnown* (which would disturb the strict-native
4318
5987
  // resident-coordinate append path). `mirrorToDurable` writes to the durable
4319
- // side only and tracks the write for error-surfacing.
5988
+ // side only; the lower-log result is held behind that durability barrier.
4320
5989
  const durableWrapper = durableWrapperActive
4321
5990
  ? this.remoteBlocks?.localStore
4322
5991
  : undefined;
4323
5992
  let nativeBackboneDocumentIndexCommitted = false;
4324
- let committedNativeBackboneDocumentIndex;
5993
+ let nativeDeleteCleanupToken;
5994
+ let nativeDocumentRollback;
5995
+ let nativeStrictTransaction;
5996
+ let lowerPublicationRollback;
4325
5997
  const nativeCommitProperties = {
4326
5998
  payloadData,
4327
5999
  resolveTrimmedEntries: properties?.resolveTrimmedEntries,
4328
6000
  };
4329
- nativeCommitProperties.skipMissingNextJoin =
4330
- properties?.skipMissingNextJoin;
4331
- nativeCommitProperties.retainMaterializationBytes =
4332
- this._logProperties?.trim != null;
4333
- const result = asTrustedLowerLog(this.log).appendLocallyPreparedNativeNoNextCommitOnly(undefined, appendOptions, nativeCommitProperties, (input) => {
4334
- const next = "next" in input && Array.isArray(input.next) ? input.next : [];
4335
- const nativeBackboneDocumentIndex = properties?.nativeBackboneDocumentIndex ??
4336
- properties?.prepareNativeBackboneDocumentIndex?.({
4337
- wallTime: input.wallTime,
4338
- gid: input.gid,
4339
- payloadSize: input.payloadData.byteLength,
4340
- });
4341
- const nativeBackboneDocumentIndexForAppend = nativeBackboneDocumentIndex &&
4342
- input.trimLengthTo == null &&
4343
- nativeBackboneDocumentIndex.deleteTrimmedHeads === true
4344
- ? {
4345
- ...nativeBackboneDocumentIndex,
4346
- deleteTrimmedHeads: false,
4347
- }
4348
- : nativeBackboneDocumentIndex;
4349
- if (nativeBackboneDocumentIndex) {
4350
- nativeBackboneDocumentIndexCommitted = true;
4351
- committedNativeBackboneDocumentIndex = nativeBackboneDocumentIndex;
4352
- }
4353
- const useLatestDocumentContext = properties?.useNativeExistingDocumentContext === true;
4354
- const prepared = backbone.graph.prepareEntryV0PlainEntryCommit({
4355
- ...input,
4356
- next,
4357
- includeMaterializationBytes: false,
4358
- includeAppendFactsBytes: true,
4359
- trimLengthTo: input.trimLengthTo,
4360
- ...(nativeBackboneDocumentIndexForAppend
6001
+ nativeCommitProperties.skipMissingNextJoin =
6002
+ properties?.skipMissingNextJoin;
6003
+ nativeCommitProperties.retainMaterializationBytes =
6004
+ this._logProperties?.trim != null;
6005
+ nativeCommitProperties.deferNativeTransactionAcknowledgement = true;
6006
+ const rollbackLowerPublication = async (error) => {
6007
+ durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6008
+ const rollbackFailures = [];
6009
+ if (lowerPublicationRollback &&
6010
+ !(error instanceof NativeDurableCommitError)) {
6011
+ try {
6012
+ const lowerPublicationStarted = lowerPublicationRollback.lowerPublicationStarted;
6013
+ await this.rollbackFailedNativeBackboneTransaction({
6014
+ committedHashes: lowerPublicationRollback.committedHashes,
6015
+ trimmedEntries: lowerPublicationRollback.trimmedEntries,
6016
+ coordinateEntries: lowerPublicationRollback.coordinateEntries,
6017
+ documents: lowerPublicationRollback.documents,
6018
+ durableWrapper: lowerPublicationStarted
6019
+ ? undefined
6020
+ : lowerPublicationRollback.durableWrapper,
6021
+ skipBlockCompensation: lowerPublicationStarted,
6022
+ unmirroredBlockCompensation: !lowerPublicationStarted,
6023
+ restoreGraphFromIndex: true,
6024
+ });
6025
+ }
6026
+ catch (rollbackError) {
6027
+ rollbackFailures.push(rollbackError);
6028
+ }
6029
+ }
6030
+ else if (nativeDocumentRollback &&
6031
+ !(error instanceof NativeDurableCommitError)) {
6032
+ try {
6033
+ this.restoreNativeBackboneDocument(nativeDocumentRollback);
6034
+ const flushed = this.flushNativeBackboneCoordinateJournal();
6035
+ if (isPromiseLike(flushed)) {
6036
+ await flushed;
6037
+ }
6038
+ }
6039
+ catch (rollbackError) {
6040
+ rollbackFailures.push(rollbackError);
6041
+ }
6042
+ }
6043
+ if (rollbackFailures.length > 0) {
6044
+ this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
6045
+ throw new AggregateError([error, ...rollbackFailures], "Lower-log publication and native compensation both failed");
6046
+ }
6047
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
6048
+ throw error;
6049
+ };
6050
+ let result;
6051
+ try {
6052
+ result = asTrustedLowerLog(this.log).appendLocallyPreparedNativeNoNextCommitOnly(undefined, appendOptions, nativeCommitProperties, async (input) => {
6053
+ const next = "next" in input && Array.isArray(input.next) ? input.next : [];
6054
+ const nativeBackboneDocumentIndex = properties?.nativeBackboneDocumentIndex ??
6055
+ properties?.prepareNativeBackboneDocumentIndex?.({
6056
+ wallTime: input.wallTime,
6057
+ gid: input.gid,
6058
+ payloadSize: input.payloadData.byteLength,
6059
+ });
6060
+ const nativeBackboneDocumentIndexForAppend = nativeBackboneDocumentIndex &&
6061
+ input.trimLengthTo == null &&
6062
+ nativeBackboneDocumentIndex.deleteTrimmedHeads === true
4361
6063
  ? {
4362
- documentIndex: {
4363
- ...nativeBackboneDocumentIndexForAppend,
4364
- ...(useLatestDocumentContext
4365
- ? { useLatestContext: true }
4366
- : {}),
4367
- },
6064
+ ...nativeBackboneDocumentIndex,
6065
+ deleteTrimmedHeads: false,
4368
6066
  }
4369
- : {}),
4370
- }, backbone.blocks);
4371
- if (prepared && !prepared.bytes) {
4372
- if (durableWrapper) {
4373
- // The durable write-through wrapper is active, so the block store
4374
- // the log writes through (this.remoteBlocks.localStore) is the
4375
- // wrapper, NOT the raw wasm block map. prepareEntryV0PlainEntryCommit
4376
- // committed the block into the wasm map ONLY and returned no raw
4377
- // bytes, so on its own the block would never reach durable (log's
4378
- // finishBlocks only calls putKnown* when prepared.bytes is set) and
4379
- // a non-replicating native node would lose it on restart.
4380
- //
4381
- // Mirror the just-committed block (read back from the wasm store)
4382
- // straight to the DURABLE side of the wrapper. Crucially we do NOT
4383
- // attach prepared.bytes: doing so would make the log's finishBlocks
4384
- // call putKnown*, which changes the commit-only append path and
4385
- // breaks the strict-native resident-coordinate optimization (the
4386
- // reopen tests assert the append stays native and resolves no entry
4387
- // block). Instead the prepared result is returned exactly as the
4388
- // memory-only branch below (getBytes only, no bytes), so the log's
4389
- // finishBlocks path is UNCHANGED, and the block is made durable
4390
- // out-of-band here. mirrorToDurable tracks the write so an IO/
4391
- // disk-full failure surfaces on a later awaited wrapper method.
6067
+ : nativeBackboneDocumentIndex;
6068
+ if (nativeBackboneDocumentIndex) {
6069
+ nativeBackboneDocumentIndexCommitted = true;
6070
+ }
6071
+ const useLatestDocumentContext = properties?.useNativeExistingDocumentContext === true;
6072
+ nativeDocumentRollback = this.snapshotNativeBackboneDocument(nativeBackboneDocumentIndexForAppend);
6073
+ nativeStrictTransaction =
6074
+ await this.beginNativeStrictDurableTransaction(nativeDocumentRollback ? [nativeDocumentRollback] : []);
6075
+ const prepared = backbone.graph.prepareEntryV0PlainEntryCommit({
6076
+ ...input,
6077
+ next,
6078
+ includeMaterializationBytes: false,
6079
+ includeAppendFactsBytes: true,
6080
+ trimLengthTo: input.trimLengthTo,
6081
+ ...(nativeBackboneDocumentIndexForAppend
6082
+ ? {
6083
+ documentIndex: {
6084
+ ...nativeBackboneDocumentIndexForAppend,
6085
+ ...(useLatestDocumentContext
6086
+ ? { useLatestContext: true }
6087
+ : {}),
6088
+ },
6089
+ }
6090
+ : {}),
6091
+ }, backbone.blocks);
6092
+ if (prepared) {
4392
6093
  const preparedHash = prepared.cid ?? prepared.hash;
4393
- const committedBytes = preparedHash
4394
- ? backbone.blocks.get(preparedHash)
4395
- : undefined;
4396
- if (committedBytes && durableWrapper.mirrorToDurable) {
4397
- durableWrapper.mirrorToDurable(preparedHash, committedBytes);
6094
+ const preparedNext = prepared.next ?? next;
6095
+ const nativeTrimmedHashes = prepared.trimmedEntryHashes ??
6096
+ prepared.trimmedEntries?.flatMap((entry) => (entry.hash ? [entry.hash] : [])) ??
6097
+ [];
6098
+ const coordinateRollback = this.snapshotResidentCoordinateEntries([
6099
+ ...(preparedHash ? [preparedHash] : []),
6100
+ ...preparedNext,
6101
+ ...nativeTrimmedHashes,
6102
+ ]);
6103
+ lowerPublicationRollback = {
6104
+ committedHashes: preparedHash ? [preparedHash] : [],
6105
+ trimmedEntries: prepared.trimmedEntries,
6106
+ coordinateEntries: coordinateRollback,
6107
+ documents: nativeDocumentRollback
6108
+ ? [nativeDocumentRollback]
6109
+ : undefined,
6110
+ durableWrapper,
6111
+ lowerPublicationStarted: false,
6112
+ };
6113
+ await this.setNativeStrictDurableTransactionOperation(nativeStrictTransaction, preparedHash ? [preparedHash] : [], nativeTrimmedHashes, coordinateRollback, combineCoordinateDeleteHashes(preparedNext, nativeTrimmedHashes));
6114
+ if (prepared.bytes) {
6115
+ lowerPublicationRollback.lowerPublicationStarted = true;
6116
+ return {
6117
+ ...prepared,
6118
+ nativeIndexMutationLockOwner: nativeStrictTransaction?.lowerHashMutationLockOwner,
6119
+ };
6120
+ }
6121
+ const rollbackCommitted = async (cause, committedCids) => {
6122
+ durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6123
+ let compensated = false;
6124
+ try {
6125
+ await this.rollbackFailedNativeBackboneTransaction({
6126
+ committedHashes: committedCids,
6127
+ trimmedEntries: prepared.trimmedEntries,
6128
+ coordinateEntries: coordinateRollback,
6129
+ documents: nativeDocumentRollback
6130
+ ? [nativeDocumentRollback]
6131
+ : undefined,
6132
+ durableWrapper,
6133
+ });
6134
+ compensated = true;
6135
+ }
6136
+ catch {
6137
+ // Keep recovery marked incomplete; close will discard pending native
6138
+ // journals. Reopen preserves uncertain content-addressed bytes and
6139
+ // recovers liveness from the authoritative lower-log facts.
6140
+ }
6141
+ if (compensated) {
6142
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
6143
+ }
6144
+ else {
6145
+ this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
6146
+ }
6147
+ return this.failNativeDurableCommit(cause, {
6148
+ committedCids,
6149
+ failedCids: committedCids,
6150
+ });
6151
+ };
6152
+ if (durableWrapper &&
6153
+ nativeTrimmedHashes.length > 0 &&
6154
+ !durableWrapper.beginNativeDeleteCleanup) {
6155
+ return rollbackCommitted(new Error("Native durable block wrapper cannot preannounce trim cleanup"), preparedHash ? [preparedHash] : []);
6156
+ }
6157
+ nativeDeleteCleanupToken =
6158
+ durableWrapper?.beginNativeDeleteCleanup?.(nativeTrimmedHashes);
6159
+ const preparedResult = {
6160
+ ...prepared,
6161
+ nativeIndexMutationLockOwner: nativeStrictTransaction?.lowerHashMutationLockOwner,
6162
+ getBytes: (hash) => backbone.blocks.get(hash),
6163
+ nativeBlocksDeleted: true,
6164
+ nativeDeleteCleanupToken,
6165
+ };
6166
+ if (durableWrapper) {
6167
+ // The durable write-through wrapper is active, so the block store
6168
+ // the log writes through (this.remoteBlocks.localStore) is the
6169
+ // wrapper, NOT the raw wasm block map. prepareEntryV0PlainEntryCommit
6170
+ // committed the block into the wasm map ONLY and returned no raw
6171
+ // bytes, so on its own the block would never reach durable (log's
6172
+ // finishBlocks only calls putKnown* when prepared.bytes is set) and
6173
+ // a non-replicating native node would lose it on restart.
6174
+ //
6175
+ // Mirror the just-committed block (read back from the wasm store)
6176
+ // straight to the DURABLE side of the wrapper. Crucially we do NOT
6177
+ // attach prepared.bytes: doing so would make the log's finishBlocks
6178
+ // call putKnown*, which changes the commit-only append path and
6179
+ // breaks the strict-native resident-coordinate optimization (the
6180
+ // reopen tests assert the append stays native and resolves no entry
6181
+ // block). Instead the prepared result is returned exactly as the
6182
+ // memory-only branch below (getBytes only, no bytes), so the log's
6183
+ // finishBlocks path is UNCHANGED. Returning the mirror promise from
6184
+ // this prepare callback holds lower-log index/head/trim publication
6185
+ // until durable succeeds.
6186
+ if (!preparedHash) {
6187
+ durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6188
+ return rollbackCommitted(new Error("Native commit returned no entry CID to mirror"), []);
6189
+ }
6190
+ if (!durableWrapper.mirrorToDurable) {
6191
+ durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6192
+ return rollbackCommitted(new Error("Native durable block wrapper has no mirror method"), [preparedHash]);
6193
+ }
6194
+ const committedBytes = backbone.blocks.get(preparedHash);
6195
+ if (!committedBytes) {
6196
+ durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6197
+ return rollbackCommitted(new Error(`Native committed block ${preparedHash} is missing from the hot store`), [preparedHash]);
6198
+ }
6199
+ return durableWrapper
6200
+ .mirrorToDurable(preparedHash, committedBytes, {
6201
+ nativeTrimmed: nativeTrimmedHashes.includes(preparedHash),
6202
+ })
6203
+ .then((nativeCommitOwnershipToken) => {
6204
+ lowerPublicationRollback.lowerPublicationStarted = true;
6205
+ return {
6206
+ ...preparedResult,
6207
+ nativeCommitOwnershipToken,
6208
+ };
6209
+ }, (error) => {
6210
+ durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6211
+ return rollbackCommitted(error, [preparedHash]);
6212
+ });
4398
6213
  }
6214
+ lowerPublicationRollback.lowerPublicationStarted = true;
6215
+ return preparedResult;
4399
6216
  }
4400
- // The block lives in the wasm map; expose getBytes for materialization
4401
- // (identical to the memory-only native node path). When the durable
4402
- // wrapper is active the durable copy was mirrored just above.
4403
- return {
4404
- ...prepared,
4405
- getBytes: (hash) => backbone.blocks.get(hash),
4406
- };
6217
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
6218
+ return prepared;
6219
+ });
6220
+ if (isPromiseLike(result)) {
6221
+ result = result.catch(rollbackLowerPublication);
4407
6222
  }
4408
- return prepared;
4409
- });
6223
+ }
6224
+ catch (error) {
6225
+ return rollbackLowerPublication(error);
6226
+ }
4410
6227
  if (!result) {
4411
- return undefined;
6228
+ return this.completeNativeStrictDurableTransaction(nativeStrictTransaction).then(() => undefined);
4412
6229
  }
4413
- return mapMaybePromise(result, (prepared) => {
6230
+ return mapMaybePromise(result, async (prepared) => {
4414
6231
  if (!prepared) {
6232
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
4415
6233
  return undefined;
4416
6234
  }
4417
- const rollback = (error) => {
4418
- if (committedNativeBackboneDocumentIndex) {
4419
- backbone.deleteDocument(committedNativeBackboneDocumentIndex.key);
6235
+ const rollback = async (error) => {
6236
+ const rollbackFailures = [];
6237
+ try {
6238
+ await this.markNativeStrictDurableTransactionRollback(nativeStrictTransaction);
6239
+ }
6240
+ catch (rollbackError) {
6241
+ const retentionFailures = this.retainNativeStrictDurableTransactionAfterMarkerFailure(nativeStrictTransaction, prepared.nativeCommittedAppendFinalizer, rollbackError);
6242
+ throw new AggregateError([error, ...retentionFailures], "Native rollback marker could not be persisted; recovery is required");
6243
+ }
6244
+ try {
6245
+ await prepared.nativeCommittedAppendFinalizer?.rollback();
6246
+ }
6247
+ catch (rollbackError) {
6248
+ rollbackFailures.push(rollbackError);
6249
+ }
6250
+ try {
6251
+ await this.rollbackNativeBackboneCoordinateAppendDurably(prepared.appendFacts.hash, lowerPublicationRollback?.coordinateEntries);
6252
+ for (const document of lowerPublicationRollback?.documents ?? []) {
6253
+ this.restoreNativeBackboneDocument(document);
6254
+ }
6255
+ const flushed = this.flushNativeBackboneCoordinateJournal();
6256
+ if (isPromiseLike(flushed)) {
6257
+ await flushed;
6258
+ }
6259
+ }
6260
+ catch (rollbackError) {
6261
+ rollbackFailures.push(rollbackError);
6262
+ }
6263
+ if (rollbackFailures.length === 0) {
6264
+ try {
6265
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
6266
+ }
6267
+ catch (rollbackError) {
6268
+ rollbackFailures.push(rollbackError);
6269
+ }
6270
+ }
6271
+ else {
6272
+ this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
6273
+ }
6274
+ if (rollbackFailures.length > 0) {
6275
+ throw new AggregateError([error, ...rollbackFailures], "Shared-log append and compensation both failed");
4420
6276
  }
4421
6277
  throw error;
4422
6278
  };
6279
+ let finishResult;
4423
6280
  try {
4424
- const deferredCoordinateDeleteHashes = this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, { removedHashes: prepared.removedHashes });
4425
- const deleteHashes = deferredCoordinateDeleteHashes &&
4426
- deferredCoordinateDeleteHashes.length > 0
4427
- ? deferredCoordinateDeleteHashes
4428
- : [];
6281
+ await this.setNativeStrictDurableTransactionExpectedRows(nativeStrictTransaction, [prepared.shallowEntry]);
4429
6282
  const finish = () => {
4430
6283
  const appendCommit = this.createPreparedLocalAppendCommitFromFacts(prepared.appendFacts);
4431
6284
  if (nativeBackboneDocumentIndexCommitted) {
@@ -4442,16 +6295,27 @@ let SharedLog = (() => {
4442
6295
  appendCommit,
4443
6296
  };
4444
6297
  };
4445
- const completed = deleteHashes.length === 0
4446
- ? finish()
4447
- : mapMaybePromise(this.deleteCoordinatesForHashes(deleteHashes), finish);
4448
- return isPromiseLike(completed)
4449
- ? completed.catch((error) => rollback(error))
4450
- : completed;
6298
+ if (!prepared.nativeCommittedAppendFinalizer) {
6299
+ throw new Error("Missing deferred native append finalizer");
6300
+ }
6301
+ // Strict success cannot honor batching thresholds: native
6302
+ // coordinate/document/signer facts must be physically durable before
6303
+ // the lower commit marker is acknowledged and its intent is retired.
6304
+ await this.flushNativeBackboneCoordinateJournal();
6305
+ await prepared.nativeCommittedAppendFinalizer.acknowledge(() => this.markNativeStrictDurableTransactionLowerMarker(nativeStrictTransaction));
6306
+ finishResult = finish();
4451
6307
  }
4452
6308
  catch (error) {
4453
6309
  return rollback(error);
4454
6310
  }
6311
+ this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, { removedHashes: prepared.removedHashes });
6312
+ try {
6313
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
6314
+ }
6315
+ catch (error) {
6316
+ warn(`Failed to retire committed native intent: ${String(error)}`);
6317
+ }
6318
+ return finishResult;
4455
6319
  });
4456
6320
  }
4457
6321
  appendLocallyPreparedPayloadNativeBackboneStorageTransaction(payloadData, appendOptions, properties, minReplicasValue, runtimeOnlyCoordinates) {
@@ -4484,8 +6348,11 @@ let SharedLog = (() => {
4484
6348
  durableWrapperActive;
4485
6349
  let nativeBackboneDocumentIndexCommitted = false;
4486
6350
  let nativeBackboneDocumentDeleteCommitted = false;
4487
- let committedNativeBackboneDocumentIndex;
4488
- const prepareBackboneAppend = (input) => {
6351
+ let nativeDocumentRollback;
6352
+ let nativeCoordinateRollback;
6353
+ let nativeDeleteCleanupToken;
6354
+ let nativeStrictTransaction;
6355
+ const prepareBackboneAppend = async (input) => {
4489
6356
  const next = input.next ?? [];
4490
6357
  const appendInput = {
4491
6358
  wallTime: input.wallTime,
@@ -4536,6 +6403,12 @@ let SharedLog = (() => {
4536
6403
  documentDeleteKey: nativeBackboneDocumentDeleteKey,
4537
6404
  }
4538
6405
  : appendInput;
6406
+ nativeDocumentRollback = this.snapshotNativeBackboneDocument(nativeBackboneDocumentIndexForAppend ??
6407
+ (nativeBackboneDocumentDeleteKey
6408
+ ? { key: nativeBackboneDocumentDeleteKey }
6409
+ : undefined));
6410
+ nativeStrictTransaction =
6411
+ await this.beginNativeStrictDurableTransaction(nativeDocumentRollback ? [nativeDocumentRollback] : []);
4539
6412
  if (next.length === 0) {
4540
6413
  if (commitBlocksInBackbone) {
4541
6414
  if (nativeBackboneDocumentIndex &&
@@ -4568,29 +6441,61 @@ let SharedLog = (() => {
4568
6441
  }
4569
6442
  if (nativeBackboneDocumentIndex) {
4570
6443
  nativeBackboneDocumentIndexCommitted = true;
4571
- committedNativeBackboneDocumentIndex = nativeBackboneDocumentIndex;
4572
6444
  }
4573
6445
  nativeBackboneDocumentDeleteCommitted =
4574
6446
  !!nativeBackboneDocumentDeleteKey;
4575
6447
  const useTrimmedHashesOnly = properties?.resolveTrimmedEntries === false;
4576
- if (durableWrapper?.mirrorToDurable) {
4577
- // The block was committed into the wasm map (commitBlocksInBackbone is
4578
- // true) but not into durable, because the log's finishBlocks path is
4579
- // left UNCHANGED for strict-native mode (getBytes only, no bytes, so
4580
- // no putKnown* through the wrapper). Mirror the just-committed block
4581
- // straight to the durable side so a non-replicating native node keeps
4582
- // it across a restart. mirrorToDurable tracks the write for
4583
- // error-surfacing.
4584
- const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
4585
- const committedBytes = committedHash
4586
- ? backbone.blocks.get(committedHash)
4587
- : undefined;
4588
- if (committedBytes) {
4589
- durableWrapper.mirrorToDurable(committedHash, committedBytes);
6448
+ const nativeTrimmedHashes = backboneAppend.trimmedHashes ??
6449
+ backboneAppend.trimmed.map((entry) => entry.hash);
6450
+ const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
6451
+ const committedNext = backboneAppend.entry.next ?? next;
6452
+ nativeCoordinateRollback = this.snapshotResidentCoordinateEntries([
6453
+ ...(committedHash ? [committedHash] : []),
6454
+ ...committedNext,
6455
+ ...nativeTrimmedHashes,
6456
+ ]);
6457
+ await this.setNativeStrictDurableTransactionOperation(nativeStrictTransaction, committedHash ? [committedHash] : [], nativeTrimmedHashes, nativeCoordinateRollback, combineCoordinateDeleteHashes(committedNext, nativeTrimmedHashes));
6458
+ const rollbackCommitted = async (cause, committedCids) => {
6459
+ durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6460
+ let compensated = false;
6461
+ try {
6462
+ await this.rollbackFailedNativeBackboneTransaction({
6463
+ committedHashes: committedCids,
6464
+ trimmedEntries: backboneAppend?.trimmed,
6465
+ coordinateEntries: nativeCoordinateRollback,
6466
+ documents: nativeDocumentRollback
6467
+ ? [nativeDocumentRollback]
6468
+ : undefined,
6469
+ durableWrapper,
6470
+ });
6471
+ compensated = true;
4590
6472
  }
4591
- }
4592
- return {
6473
+ catch {
6474
+ // close/reopen completes recovery if durable compensation failed
6475
+ }
6476
+ if (compensated) {
6477
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
6478
+ }
6479
+ else {
6480
+ this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
6481
+ }
6482
+ return this.failNativeDurableCommit(cause, {
6483
+ committedCids,
6484
+ failedCids: committedCids,
6485
+ });
6486
+ };
6487
+ if (durableWrapper &&
6488
+ commitBlocksInBackbone &&
6489
+ nativeTrimmedHashes.length > 0 &&
6490
+ !durableWrapper.beginNativeDeleteCleanup) {
6491
+ return rollbackCommitted(new Error("Native durable block wrapper cannot preannounce trim cleanup"), committedHash ? [committedHash] : []);
6492
+ }
6493
+ nativeDeleteCleanupToken = commitBlocksInBackbone
6494
+ ? durableWrapper?.beginNativeDeleteCleanup?.(nativeTrimmedHashes)
6495
+ : undefined;
6496
+ const preparedResult = {
4593
6497
  ...backboneAppend.entry,
6498
+ nativeIndexMutationLockOwner: nativeStrictTransaction?.lowerHashMutationLockOwner,
4594
6499
  gid: backboneAppend.coordinate.gid,
4595
6500
  getBytes: commitBlocksInBackbone
4596
6501
  ? (hash) => backbone.blocks.get(hash)
@@ -4601,8 +6506,42 @@ let SharedLog = (() => {
4601
6506
  trimmedEntryHashes: useTrimmedHashesOnly
4602
6507
  ? backboneAppend.trimmedHashes
4603
6508
  : undefined,
6509
+ nativeBlocksDeleted: commitBlocksInBackbone,
6510
+ nativeDeleteCleanupToken,
4604
6511
  documentPreviousContext: backboneAppend.documentPreviousContext,
4605
6512
  };
6513
+ if (durableWrapper?.mirrorToDurable) {
6514
+ // The block was committed into the wasm map (commitBlocksInBackbone is
6515
+ // true) but not into durable, because the log's finishBlocks path is
6516
+ // left UNCHANGED for strict-native mode (getBytes only, no bytes, so
6517
+ // no putKnown* through the wrapper). Returning the promise here prevents
6518
+ // lower-log index/head/trim publication until the mirror settles.
6519
+ if (!committedHash) {
6520
+ durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6521
+ return rollbackCommitted(new Error("Native commit returned no entry CID to mirror"), []);
6522
+ }
6523
+ const committedBytes = backboneAppend.entry.bytes ?? backbone.blocks.get(committedHash);
6524
+ if (!committedBytes) {
6525
+ durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6526
+ return rollbackCommitted(new Error(`Native committed block ${committedHash} is missing from the hot store`), [committedHash]);
6527
+ }
6528
+ return durableWrapper
6529
+ .mirrorToDurable(committedHash, committedBytes, {
6530
+ nativeTrimmed: nativeTrimmedHashes.includes(committedHash),
6531
+ })
6532
+ .then((nativeCommitOwnershipToken) => ({
6533
+ ...preparedResult,
6534
+ nativeCommitOwnershipToken,
6535
+ }), (error) => {
6536
+ durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6537
+ return rollbackCommitted(error, [committedHash]);
6538
+ });
6539
+ }
6540
+ if (durableWrapper) {
6541
+ durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6542
+ return rollbackCommitted(new Error("Native durable block wrapper has no mirror method"), committedHash ? [committedHash] : []);
6543
+ }
6544
+ return preparedResult;
4606
6545
  };
4607
6546
  const hasKnownNoNext = appendOptions.meta?.next != null &&
4608
6547
  appendOptions.meta.next.length === 0;
@@ -4611,19 +6550,62 @@ let SharedLog = (() => {
4611
6550
  resolveTrimmedEntries: properties?.resolveTrimmedEntries,
4612
6551
  skipMissingNextJoin: properties?.skipMissingNextJoin,
4613
6552
  retainMaterializationBytes: this._logProperties?.trim != null,
6553
+ deferNativeTransactionAcknowledgement: true,
4614
6554
  }, prepareBackboneAppend);
4615
- const directNoNextResult = hasKnownNoNext
4616
- ? asTrustedLowerLog(this.log).appendLocallyPreparedNativeKnownNoNextCommitOnly(undefined, appendOptions, {
4617
- payloadData,
4618
- resolveTrimmedEntries: properties?.resolveTrimmedEntries,
4619
- retainMaterializationBytes: this._logProperties?.trim != null,
4620
- }, prepareBackboneAppend)
4621
- : undefined;
4622
- const result = directNoNextResult === undefined
4623
- ? appendGenericNativeCommit()
4624
- : directNoNextResult;
4625
- return mapMaybePromise(result, (prepared) => {
6555
+ const rollbackLowerPublication = async (error) => {
6556
+ durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
6557
+ let compensated = !backboneAppend;
6558
+ if (backboneAppend && !(error instanceof NativeDurableCommitError)) {
6559
+ const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
6560
+ try {
6561
+ await this.rollbackFailedNativeBackboneTransaction({
6562
+ committedHashes: committedHash ? [committedHash] : [],
6563
+ coordinateEntries: nativeCoordinateRollback,
6564
+ documents: nativeDocumentRollback
6565
+ ? [nativeDocumentRollback]
6566
+ : undefined,
6567
+ skipBlockCompensation: true,
6568
+ restoreGraphFromIndex: true,
6569
+ });
6570
+ compensated = true;
6571
+ }
6572
+ catch {
6573
+ // Lower-log compensation already handled durable/native blocks.
6574
+ // Preserve the index publication error for this caller.
6575
+ }
6576
+ }
6577
+ if (compensated) {
6578
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
6579
+ }
6580
+ else {
6581
+ this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
6582
+ }
6583
+ throw error;
6584
+ };
6585
+ let result;
6586
+ try {
6587
+ const directNoNextResult = hasKnownNoNext
6588
+ ? asTrustedLowerLog(this.log).appendLocallyPreparedNativeKnownNoNextCommitOnly(undefined, appendOptions, {
6589
+ payloadData,
6590
+ resolveTrimmedEntries: properties?.resolveTrimmedEntries,
6591
+ retainMaterializationBytes: this._logProperties?.trim != null,
6592
+ deferNativeTransactionAcknowledgement: true,
6593
+ }, prepareBackboneAppend)
6594
+ : undefined;
6595
+ result =
6596
+ directNoNextResult === undefined
6597
+ ? appendGenericNativeCommit()
6598
+ : directNoNextResult;
6599
+ if (isPromiseLike(result)) {
6600
+ result = result.catch(rollbackLowerPublication);
6601
+ }
6602
+ }
6603
+ catch (error) {
6604
+ return rollbackLowerPublication(error);
6605
+ }
6606
+ return mapMaybePromise(result, async (prepared) => {
4626
6607
  if (!prepared || !backboneAppend) {
6608
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
4627
6609
  return undefined;
4628
6610
  }
4629
6611
  const coordinateFields = this.createCoordinateFieldsFromNativePlanFacts({
@@ -4639,11 +6621,11 @@ let SharedLog = (() => {
4639
6621
  fields: coordinateFields,
4640
6622
  });
4641
6623
  const plannedCoordinateDeleteHashes = combineCoordinateDeleteHashes(prepared.appendFacts.next, prepared.removedHashes ?? prepared.removed.map((entry) => entry.hash));
4642
- const rollbackCoordinateEntries = this.snapshotResidentCoordinateEntries(plannedCoordinateDeleteHashes);
4643
- const deferredCoordinateDeleteHashes = this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, {
4644
- forgetNativeCoordinates: false,
4645
- removedHashes: prepared.removedHashes,
4646
- });
6624
+ const rollbackCoordinateEntries = nativeCoordinateRollback ??
6625
+ this.snapshotResidentCoordinateEntries([
6626
+ prepared.appendFacts.hash,
6627
+ ...plannedCoordinateDeleteHashes,
6628
+ ]);
4647
6629
  const finish = () => {
4648
6630
  const appendCommit = this.createPreparedLocalAppendCommitFromFacts(prepared.appendFacts, {
4649
6631
  hashNumber: backboneAppend.coordinate
@@ -4671,14 +6653,48 @@ let SharedLog = (() => {
4671
6653
  };
4672
6654
  };
4673
6655
  const coordinateIndex = this.entryCoordinatesIndex;
4674
- const rollback = (error) => {
4675
- this.rollbackNativeBackboneCoordinateAppend(prepared.appendFacts.hash, rollbackCoordinateEntries);
4676
- if (committedNativeBackboneDocumentIndex) {
4677
- backbone.deleteDocument(committedNativeBackboneDocumentIndex.key);
6656
+ const rollback = async (error) => {
6657
+ const rollbackFailures = [];
6658
+ try {
6659
+ await this.markNativeStrictDurableTransactionRollback(nativeStrictTransaction);
6660
+ }
6661
+ catch (rollbackError) {
6662
+ const retentionFailures = this.retainNativeStrictDurableTransactionAfterMarkerFailure(nativeStrictTransaction, prepared.nativeCommittedAppendFinalizer, rollbackError);
6663
+ throw new AggregateError([error, ...retentionFailures], "Native rollback marker could not be persisted; recovery is required");
6664
+ }
6665
+ try {
6666
+ await prepared.nativeCommittedAppendFinalizer?.rollback();
6667
+ }
6668
+ catch (rollbackError) {
6669
+ rollbackFailures.push(rollbackError);
6670
+ }
6671
+ try {
6672
+ await this.rollbackNativeBackboneCoordinateAppendDurably(prepared.appendFacts.hash, rollbackCoordinateEntries);
6673
+ }
6674
+ catch (rollbackError) {
6675
+ rollbackFailures.push(rollbackError);
6676
+ }
6677
+ if (nativeDocumentRollback) {
6678
+ try {
6679
+ this.restoreNativeBackboneDocument(nativeDocumentRollback);
6680
+ const flushed = this.flushNativeBackboneCoordinateJournal();
6681
+ if (isPromiseLike(flushed)) {
6682
+ await flushed;
6683
+ }
6684
+ }
6685
+ catch (rollbackError) {
6686
+ rollbackFailures.push(rollbackError);
6687
+ }
4678
6688
  }
6689
+ if (rollbackFailures.length > 0) {
6690
+ this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
6691
+ throw new AggregateError([error, ...rollbackFailures], "Shared-log append and compensation both failed");
6692
+ }
6693
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
4679
6694
  throw error;
4680
6695
  };
4681
6696
  try {
6697
+ await this.setNativeStrictDurableTransactionExpectedRows(nativeStrictTransaction, [prepared.shallowEntry]);
4682
6698
  const hasNativeCoordinatePut = this.canUseBackboneOnlyCoordinatePersistence() ||
4683
6699
  coordinateIndex.putSharedLogCoordinateFieldsEncodedAndDeleteHashesNoReturn ||
4684
6700
  coordinateIndex.putSharedLogCoordinateFieldsAndDeleteHashesNoReturn;
@@ -4687,7 +6703,7 @@ let SharedLog = (() => {
4687
6703
  coordinateIndex,
4688
6704
  fields: coordinateFields,
4689
6705
  hash: prepared.appendFacts.hash,
4690
- deleteHashes: plannedCoordinateDeleteHashes,
6706
+ deleteHashes: [],
4691
6707
  coordinates: backboneAppend.coordinate
4692
6708
  .coordinates,
4693
6709
  skipGenericTransientCoordinateIndex: runtimeOnlyCoordinates,
@@ -4695,43 +6711,56 @@ let SharedLog = (() => {
4695
6711
  : this.persistPreparedCoordinate({
4696
6712
  prepared: getPreparedCoordinate(),
4697
6713
  hash: prepared.appendFacts.hash,
4698
- nextHashes: prepared.appendFacts.next,
4699
- deleteHashes: deferredCoordinateDeleteHashes,
6714
+ nextHashes: [],
6715
+ deleteHashes: [],
4700
6716
  coordinates: backboneAppend.coordinate
4701
6717
  .coordinates,
4702
6718
  replicas: backboneAppend.coordinate.coordinates.length,
4703
6719
  commitNative: true,
4704
6720
  commitNativeBackbone: false,
4705
6721
  });
4706
- const completed = mapMaybePromise(persisted, () => {
4707
- if (commitBlocksInBackbone &&
4708
- !runtimeOnlyCoordinates &&
4709
- this.remoteBlocks.hasNotifyStoredHook()) {
4710
- this.remoteBlocks.notifyStoredDeferred(prepared.appendFacts.hash);
4711
- }
4712
- const delayAdaptiveRebalance = this.shouldDelayAdaptiveRebalance();
4713
- if (!backboneAppend.isLeader && !delayAdaptiveRebalance) {
4714
- const leaders = backboneAppend.leaders;
4715
- if (leaders) {
4716
- const pruneEntry = this.materializePreparedCoordinateEntry(getPreparedCoordinate());
4717
- this.pruneDebouncedFnAddIfNotKeeping({
4718
- key: pruneEntry.hash,
4719
- value: { entry: pruneEntry, leaders },
4720
- });
4721
- }
4722
- }
4723
- if (!delayAdaptiveRebalance) {
4724
- this.rebalanceParticipationDebounced?.call();
4725
- }
4726
- return finish();
4727
- });
4728
- return isPromiseLike(completed)
4729
- ? completed.catch((error) => rollback(error))
4730
- : completed;
6722
+ if (isPromiseLike(persisted)) {
6723
+ await persisted;
6724
+ }
6725
+ if (!prepared.nativeCommittedAppendFinalizer) {
6726
+ throw new Error("Missing deferred native append finalizer");
6727
+ }
6728
+ await this.flushNativeBackboneCoordinateJournal();
6729
+ await prepared.nativeCommittedAppendFinalizer.acknowledge(() => this.markNativeStrictDurableTransactionLowerMarker(nativeStrictTransaction));
4731
6730
  }
4732
6731
  catch (error) {
4733
6732
  return rollback(error);
4734
6733
  }
6734
+ this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, {
6735
+ forgetNativeCoordinates: false,
6736
+ removedHashes: prepared.removedHashes,
6737
+ });
6738
+ try {
6739
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
6740
+ }
6741
+ catch (error) {
6742
+ warn(`Failed to retire committed native intent: ${String(error)}`);
6743
+ }
6744
+ if (commitBlocksInBackbone &&
6745
+ !runtimeOnlyCoordinates &&
6746
+ this.remoteBlocks.hasNotifyStoredHook()) {
6747
+ this.remoteBlocks.notifyStoredDeferred(prepared.appendFacts.hash);
6748
+ }
6749
+ const delayAdaptiveRebalance = this.shouldDelayAdaptiveRebalance();
6750
+ if (!backboneAppend.isLeader && !delayAdaptiveRebalance) {
6751
+ const leaders = backboneAppend.leaders;
6752
+ if (leaders) {
6753
+ const pruneEntry = this.materializePreparedCoordinateEntry(getPreparedCoordinate());
6754
+ this.pruneDebouncedFnAddIfNotKeeping({
6755
+ key: pruneEntry.hash,
6756
+ value: { entry: pruneEntry, leaders },
6757
+ });
6758
+ }
6759
+ }
6760
+ if (!delayAdaptiveRebalance) {
6761
+ this.rebalanceParticipationDebounced?.call();
6762
+ }
6763
+ return finish();
4735
6764
  });
4736
6765
  });
4737
6766
  }
@@ -4985,16 +7014,27 @@ let SharedLog = (() => {
4985
7014
  const context = await this.createLeaderSelectionContext();
4986
7015
  const nativeLeaderOptions = this.createNativeLeaderOptions(context);
4987
7016
  let backboneAppends;
4988
- const appended = await asTrustedLowerLog(this.log).appendLocallyPreparedNativeKnownNoNextCommitOnlyBatch(data, appendOptions, {
4989
- payloadDatas,
4990
- resolveTrimmedEntries: properties?.resolveTrimmedEntries,
4991
- allowPreparedNexts: usesLatestDocumentContext,
4992
- retainMaterializationBytes: properties?.retainMaterializationBytes === true ||
4993
- this._logProperties?.trim != null,
4994
- }, (inputs) => {
4995
- const documentDeleteTrimmedHeadsForAppend = deleteTrimmedHeads && inputs[0]?.trimLengthTo != null;
4996
- backboneAppends =
4997
- usesLatestDocumentContext
7017
+ let batchDocumentRollbacks = [];
7018
+ let batchCoordinateRollback;
7019
+ let nativeDeleteCleanupToken;
7020
+ let nativeStrictTransaction;
7021
+ let appended;
7022
+ try {
7023
+ appended = await asTrustedLowerLog(this.log).appendLocallyPreparedNativeKnownNoNextCommitOnlyBatch(data, appendOptions, {
7024
+ payloadDatas,
7025
+ resolveTrimmedEntries: properties?.resolveTrimmedEntries,
7026
+ allowPreparedNexts: usesLatestDocumentContext,
7027
+ retainMaterializationBytes: properties?.retainMaterializationBytes === true ||
7028
+ this._logProperties?.trim != null,
7029
+ deferNativeTransactionAcknowledgement: true,
7030
+ }, async (inputs) => {
7031
+ batchDocumentRollbacks = documentIndexes
7032
+ .map((index) => this.snapshotNativeBackboneDocument(index))
7033
+ .filter((value) => !!value);
7034
+ nativeStrictTransaction =
7035
+ await this.beginNativeStrictDurableTransaction(batchDocumentRollbacks);
7036
+ const documentDeleteTrimmedHeadsForAppend = deleteTrimmedHeads && inputs[0]?.trimLengthTo != null;
7037
+ backboneAppends = usesLatestDocumentContext
4998
7038
  ? backbone.preparePlainCommittedStorageAppendDocumentIndexLatestBatchTransaction({
4999
7039
  entries: inputs.map((input, index) => ({
5000
7040
  wallTime: input.wallTime,
@@ -5034,72 +7074,294 @@ let SharedLog = (() => {
5034
7074
  documentDeleteTrimmedHeads: documentDeleteTrimmedHeadsForAppend,
5035
7075
  trimLengthTo: inputs[0]?.trimLengthTo,
5036
7076
  });
5037
- return backboneAppends?.map((append) => ({
5038
- cid: append.entry.hash,
5039
- hash: append.entry.hash,
5040
- gid: append.coordinate.gid,
5041
- next: append.entry.next,
5042
- bytes: append.entry.bytes,
5043
- byteLength: append.entry.byteLength,
5044
- metaBytes: append.entry.metaBytes,
5045
- hashDigestBytes: append.entry.hashDigestBytes,
5046
- getBytes: (hash) => backbone.blocks.get(hash),
5047
- trimmedEntryHashes: append.trimmedHashes,
5048
- documentTrimmedHeadsProcessed: append.documentTrimmedHeadsProcessed,
5049
- documentPreviousContext: append.documentPreviousContext,
5050
- }));
5051
- });
7077
+ if (!backboneAppends) {
7078
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
7079
+ return undefined;
7080
+ }
7081
+ const committedAppends = backboneAppends;
7082
+ const committedCids = committedAppends
7083
+ .map((append) => append.entry.cid ?? append.entry.hash)
7084
+ .filter((cid) => !!cid);
7085
+ const nativeTrimmedHashSet = new Set(committedAppends.flatMap((append) => append.trimmedHashes ??
7086
+ append.trimmed.map((entry) => entry.hash)));
7087
+ const nativeTrimmedHashes = [...nativeTrimmedHashSet];
7088
+ batchCoordinateRollback = this.snapshotResidentCoordinateEntries(committedAppends.flatMap((append) => [
7089
+ ...((append.entry.cid ?? append.entry.hash)
7090
+ ? [append.entry.cid ?? append.entry.hash]
7091
+ : []),
7092
+ ...append.entry.next,
7093
+ ...(append.trimmedHashes ??
7094
+ append.trimmed.map((entry) => entry.hash)),
7095
+ ]));
7096
+ await this.setNativeStrictDurableTransactionOperation(nativeStrictTransaction, committedCids, nativeTrimmedHashes, batchCoordinateRollback, committedAppends.flatMap((append) => [
7097
+ ...append.entry.next,
7098
+ ...(append.trimmedHashes ??
7099
+ append.trimmed.map((entry) => entry.hash)),
7100
+ ]));
7101
+ const rollbackCommitted = async (cause) => {
7102
+ durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
7103
+ let compensated = false;
7104
+ try {
7105
+ await this.rollbackFailedNativeBackboneTransaction({
7106
+ committedHashes: committedCids,
7107
+ trimmedEntries: committedAppends.flatMap((append) => append.trimmed),
7108
+ coordinateEntries: batchCoordinateRollback,
7109
+ documents: batchDocumentRollbacks,
7110
+ durableWrapper,
7111
+ });
7112
+ compensated = true;
7113
+ }
7114
+ catch {
7115
+ // close/reopen completes recovery if durable compensation failed
7116
+ }
7117
+ if (compensated) {
7118
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
7119
+ }
7120
+ else {
7121
+ this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
7122
+ }
7123
+ return this.failNativeDurableCommit(cause, {
7124
+ committedCids,
7125
+ failedCids: committedCids,
7126
+ });
7127
+ };
7128
+ if (durableWrapper &&
7129
+ nativeTrimmedHashes.length > 0 &&
7130
+ !durableWrapper.beginNativeDeleteCleanup) {
7131
+ return rollbackCommitted(new Error("Native durable block wrapper cannot preannounce trim cleanup"));
7132
+ }
7133
+ nativeDeleteCleanupToken =
7134
+ durableWrapper?.beginNativeDeleteCleanup?.(nativeTrimmedHashes);
7135
+ const preparedRows = committedAppends.map((append) => ({
7136
+ cid: append.entry.hash,
7137
+ hash: append.entry.hash,
7138
+ gid: append.coordinate.gid,
7139
+ next: append.entry.next,
7140
+ bytes: append.entry.bytes,
7141
+ byteLength: append.entry.byteLength,
7142
+ metaBytes: append.entry.metaBytes,
7143
+ hashDigestBytes: append.entry.hashDigestBytes,
7144
+ getBytes: (hash) => backbone.blocks.get(hash),
7145
+ nativeIndexMutationLockOwner: nativeStrictTransaction?.lowerHashMutationLockOwner,
7146
+ trimmedEntryHashes: append.trimmedHashes,
7147
+ nativeBlocksDeleted: true,
7148
+ nativeDeleteCleanupToken,
7149
+ documentTrimmedHeadsProcessed: append.documentTrimmedHeadsProcessed,
7150
+ documentPreviousContext: append.documentPreviousContext,
7151
+ }));
7152
+ if (!durableWrapper) {
7153
+ return preparedRows;
7154
+ }
7155
+ if (!durableWrapper.mirrorManyToDurable) {
7156
+ durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
7157
+ return rollbackCommitted(new Error("Native durable block wrapper has no batch mirror method"));
7158
+ }
7159
+ const durableMirrorBlocks = [];
7160
+ const missingCommittedCids = [];
7161
+ let missingCommittedHash = false;
7162
+ for (const backboneAppend of committedAppends) {
7163
+ const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
7164
+ if (!committedHash) {
7165
+ missingCommittedHash = true;
7166
+ continue;
7167
+ }
7168
+ // Earlier rows can be trimmed by later rows in this one native batch.
7169
+ // The native result retains their bytes even though the final hot map
7170
+ // no longer does; mirror those bytes, then let the explicit trim cleanup
7171
+ // remove the durable copy.
7172
+ const committedBytes = backboneAppend.entry.bytes ?? backbone.blocks.get(committedHash);
7173
+ if (!committedBytes) {
7174
+ missingCommittedCids.push(committedHash);
7175
+ continue;
7176
+ }
7177
+ durableMirrorBlocks.push([committedHash, committedBytes]);
7178
+ }
7179
+ // One strict putKnownMany WAL mutation gives the whole native batch one
7180
+ // durability barrier instead of issuing and fsyncing one record per row.
7181
+ const durableMirror = durableMirrorBlocks.length > 0
7182
+ ? durableWrapper.mirrorManyToDurable(durableMirrorBlocks, {
7183
+ nativeTrimmedCids: nativeTrimmedHashSet,
7184
+ })
7185
+ : Promise.resolve();
7186
+ return Promise.allSettled([durableMirror]).then(async (settled) => {
7187
+ const rejected = settled[0]?.status === "rejected"
7188
+ ? settled[0].reason
7189
+ : undefined;
7190
+ if (missingCommittedHash ||
7191
+ missingCommittedCids.length > 0 ||
7192
+ rejected !== undefined) {
7193
+ durableWrapper.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
7194
+ const cause = rejected === undefined
7195
+ ? new Error(missingCommittedHash
7196
+ ? "Native batch commit returned an entry with no CID to mirror"
7197
+ : `Native committed blocks are missing from the hot store: ${missingCommittedCids.join(", ")}`)
7198
+ : rejected;
7199
+ const rejectedCids = cause instanceof NativeDurableCommitError
7200
+ ? cause.failedCids.filter((cid) => committedCids.includes(cid))
7201
+ : durableMirrorBlocks.map(([cid]) => cid);
7202
+ if (cause instanceof NativeDurableCommitError) {
7203
+ cause.addCommitContext({
7204
+ committedCids,
7205
+ failedCids: [...missingCommittedCids, ...rejectedCids],
7206
+ });
7207
+ }
7208
+ return rollbackCommitted(cause);
7209
+ }
7210
+ const nativeCommitOwnershipToken = settled[0]?.status === "fulfilled" ? settled[0].value : undefined;
7211
+ return preparedRows.map((row) => ({
7212
+ ...row,
7213
+ nativeCommitOwnershipToken,
7214
+ }));
7215
+ });
7216
+ });
7217
+ }
7218
+ catch (error) {
7219
+ durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
7220
+ let compensated = !backboneAppends;
7221
+ if (backboneAppends && !(error instanceof NativeDurableCommitError)) {
7222
+ try {
7223
+ await this.rollbackFailedNativeBackboneTransaction({
7224
+ committedHashes: backboneAppends
7225
+ .map((append) => append.entry.cid ?? append.entry.hash)
7226
+ .filter((hash) => !!hash),
7227
+ coordinateEntries: batchCoordinateRollback,
7228
+ documents: batchDocumentRollbacks,
7229
+ skipBlockCompensation: true,
7230
+ restoreGraphFromIndex: true,
7231
+ });
7232
+ compensated = true;
7233
+ }
7234
+ catch {
7235
+ // Preserve the lower index publication failure.
7236
+ }
7237
+ }
7238
+ if (!(error instanceof NativeDurableCommitError)) {
7239
+ if (compensated) {
7240
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
7241
+ }
7242
+ else {
7243
+ this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
7244
+ }
7245
+ }
7246
+ throw error;
7247
+ }
5052
7248
  if (!appended || !backboneAppends) {
7249
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
5053
7250
  return undefined;
5054
7251
  }
5055
- const appendCommits = [];
5056
7252
  const runtimeOnlyCoordinates = options?.replicate === false;
5057
- for (let i = 0; i < appended.appendFacts.length; i++) {
5058
- const facts = appended.appendFacts[i];
5059
- const backboneAppend = backboneAppends[i];
5060
- if (durableWrapper?.mirrorToDurable) {
5061
- // The batch prepare committed each block into the wasm map; the log's
5062
- // finishBlocks path is left unchanged (getBytes only) to preserve
5063
- // strict-native mode, so mirror the committed block straight to durable
5064
- // so a non-replicating native node keeps it across a restart. Tracked
5065
- // for error-surfacing by mirrorToDurable.
5066
- const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
5067
- const committedBytes = committedHash
5068
- ? backbone.blocks.get(committedHash)
5069
- : undefined;
5070
- if (committedBytes) {
5071
- durableWrapper.mirrorToDurable(committedHash, committedBytes);
7253
+ const rollbackBatch = async (error) => {
7254
+ const rollbackFailures = [];
7255
+ try {
7256
+ await this.markNativeStrictDurableTransactionRollback(nativeStrictTransaction);
7257
+ }
7258
+ catch (rollbackError) {
7259
+ const retentionFailures = this.retainNativeStrictDurableTransactionAfterMarkerFailure(nativeStrictTransaction, appended.nativeCommittedAppendFinalizer, rollbackError);
7260
+ throw new AggregateError([error, ...retentionFailures], "Native rollback marker could not be persisted; recovery is required");
7261
+ }
7262
+ try {
7263
+ await appended.nativeCommittedAppendFinalizer?.rollback();
7264
+ }
7265
+ catch (rollbackError) {
7266
+ rollbackFailures.push(rollbackError);
7267
+ }
7268
+ try {
7269
+ await this.rollbackNativeBackboneCoordinateAppendDurably(appended.appendFacts[0]?.hash ?? "", batchCoordinateRollback);
7270
+ }
7271
+ catch (rollbackError) {
7272
+ rollbackFailures.push(rollbackError);
7273
+ }
7274
+ try {
7275
+ for (const document of batchDocumentRollbacks) {
7276
+ this.restoreNativeBackboneDocument(document);
7277
+ }
7278
+ const flushed = this.flushNativeBackboneCoordinateJournal();
7279
+ if (isPromiseLike(flushed)) {
7280
+ await flushed;
7281
+ }
7282
+ }
7283
+ catch (rollbackError) {
7284
+ rollbackFailures.push(rollbackError);
7285
+ }
7286
+ if (rollbackFailures.length > 0) {
7287
+ this.releaseNativeStrictDurableTransaction(nativeStrictTransaction);
7288
+ throw new AggregateError([error, ...rollbackFailures], "Shared-log append batch and compensation both failed");
7289
+ }
7290
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
7291
+ throw error;
7292
+ };
7293
+ const coordinateRows = [];
7294
+ try {
7295
+ const batchExternalNextHashes = new Set(appended.appendFacts.flatMap((facts) => facts.next));
7296
+ await this.setNativeStrictDurableTransactionExpectedRows(nativeStrictTransaction, appended.appendFacts.map((facts) => new ShallowEntry({
7297
+ hash: facts.hash,
7298
+ payloadSize: facts.payloadSize,
7299
+ head: !batchExternalNextHashes.has(facts.hash),
7300
+ meta: new ShallowMeta({
7301
+ gid: facts.gid,
7302
+ clock: new LamportClock({
7303
+ id: facts.clockId ?? this.node.identity.publicKey.bytes,
7304
+ timestamp: new Timestamp({
7305
+ wallTime: facts.wallTime,
7306
+ logical: facts.logical,
7307
+ }),
7308
+ }),
7309
+ data: facts.metaData,
7310
+ next: facts.next,
7311
+ type: facts.type ?? EntryType.APPEND,
7312
+ }),
7313
+ })));
7314
+ for (let i = 0; i < appended.appendFacts.length; i++) {
7315
+ const facts = appended.appendFacts[i];
7316
+ const backboneAppend = backboneAppends[i];
7317
+ const coordinateFields = this.createCoordinateFieldsFromNativePlanFacts({
7318
+ appendFacts: facts,
7319
+ plan: backboneAppend.coordinate,
7320
+ });
7321
+ if (!coordinateFields) {
7322
+ throw new Error("Native backbone batch append transaction returned mismatched coordinate facts");
7323
+ }
7324
+ const plannedCoordinateDeleteHashes = combineCoordinateDeleteHashes(facts.next, backboneAppend.trimmedHashes ?? []);
7325
+ coordinateRows.push({
7326
+ facts,
7327
+ backboneAppend,
7328
+ coordinateFields,
7329
+ plannedCoordinateDeleteHashes,
7330
+ });
7331
+ const persisted = this.persistBackboneCoordinateFieldsNativeTransaction({
7332
+ coordinateIndex: this.entryCoordinatesIndex,
7333
+ fields: coordinateFields,
7334
+ hash: facts.hash,
7335
+ deleteHashes: [],
7336
+ coordinates: backboneAppend.coordinate
7337
+ .coordinates,
7338
+ skipGenericTransientCoordinateIndex: runtimeOnlyCoordinates,
7339
+ });
7340
+ if (isPromiseLike(persisted)) {
7341
+ await persisted;
5072
7342
  }
5073
7343
  }
5074
- const coordinateFields = this.createCoordinateFieldsFromNativePlanFacts({
5075
- appendFacts: facts,
5076
- plan: backboneAppend.coordinate,
5077
- });
5078
- if (!coordinateFields) {
5079
- throw new Error("Native backbone batch append transaction returned mismatched coordinate facts");
7344
+ if (!appended.nativeCommittedAppendFinalizer) {
7345
+ throw new Error("Missing deferred native append batch finalizer");
5080
7346
  }
5081
- const plannedCoordinateDeleteHashes = combineCoordinateDeleteHashes(facts.next, backboneAppend.trimmedHashes ?? []);
7347
+ await this.flushNativeBackboneCoordinateJournal();
7348
+ await appended.nativeCommittedAppendFinalizer.acknowledge(() => this.markNativeStrictDurableTransactionLowerMarker(nativeStrictTransaction));
7349
+ }
7350
+ catch (error) {
7351
+ return rollbackBatch(error);
7352
+ }
7353
+ const appendCommits = [];
7354
+ for (let i = 0; i < coordinateRows.length; i++) {
7355
+ const { facts, backboneAppend, coordinateFields, plannedCoordinateDeleteHashes, } = coordinateRows[i];
5082
7356
  this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(facts, [], appended.materializeEntries[i], {
5083
7357
  forgetNativeCoordinates: false,
5084
7358
  removedHashes: plannedCoordinateDeleteHashes,
5085
7359
  });
5086
- const persisted = this.persistBackboneCoordinateFieldsNativeTransaction({
5087
- coordinateIndex: this.entryCoordinatesIndex,
5088
- fields: coordinateFields,
5089
- hash: facts.hash,
5090
- deleteHashes: plannedCoordinateDeleteHashes,
5091
- coordinates: backboneAppend.coordinate.coordinates,
5092
- skipGenericTransientCoordinateIndex: runtimeOnlyCoordinates,
5093
- });
5094
- if (isPromiseLike(persisted)) {
5095
- await persisted;
5096
- }
5097
7360
  if (!runtimeOnlyCoordinates && this.remoteBlocks.hasNotifyStoredHook()) {
5098
7361
  this.remoteBlocks.notifyStoredDeferred(facts.hash);
5099
7362
  }
5100
7363
  const appendCommit = this.createPreparedLocalAppendCommitFromFacts(facts, {
5101
- hashNumber: backboneAppend.coordinate
5102
- .hashNumber,
7364
+ hashNumber: backboneAppend.coordinate.hashNumber,
5103
7365
  coordinateFields,
5104
7366
  });
5105
7367
  appendCommit.nativeBackboneDocumentIndexCommitted = true;
@@ -5109,6 +7371,12 @@ let SharedLog = (() => {
5109
7371
  backboneAppend.documentPreviousContext;
5110
7372
  appendCommits.push(appendCommit);
5111
7373
  }
7374
+ try {
7375
+ await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
7376
+ }
7377
+ catch (error) {
7378
+ warn(`Failed to retire committed native intent: ${String(error)}`);
7379
+ }
5112
7380
  const delayAdaptiveRebalance = this.shouldDelayAdaptiveRebalance();
5113
7381
  if (!delayAdaptiveRebalance) {
5114
7382
  this.rebalanceParticipationDebounced?.call();
@@ -5123,6 +7391,7 @@ let SharedLog = (() => {
5123
7391
  };
5124
7392
  }
5125
7393
  async appendLocallyPreparedManyIndependent(data, options, properties) {
7394
+ this.throwIfNativeDurableCommitFailed();
5126
7395
  if (data.length === 0) {
5127
7396
  return { entries: [], removed: [], appendCommits: [] };
5128
7397
  }
@@ -5204,6 +7473,7 @@ let SharedLog = (() => {
5204
7473
  });
5205
7474
  }
5206
7475
  async appendMany(data, options) {
7476
+ this.throwIfNativeDurableCommitFailed();
5207
7477
  if (data.length === 0) {
5208
7478
  return { entries: [], removed: [] };
5209
7479
  }
@@ -5838,6 +8108,9 @@ let SharedLog = (() => {
5838
8108
  return this.createPreparedLocalAppendCommits(entries, nativeAppendPlans);
5839
8109
  }
5840
8110
  async open(options) {
8111
+ this.ensureNativeDurabilityRuntimeState();
8112
+ this._nativeStrictDurableTransactionsClosing = false;
8113
+ const recoveringNativeDurableFailure = this._nativeDurableCommitFailure !== undefined;
5841
8114
  options = applySharedLogNativeDefaults(options, this.node
5842
8115
  .sharedLogNativeDefaults);
5843
8116
  this.replicas = {
@@ -5900,6 +8173,8 @@ let SharedLog = (() => {
5900
8173
  this._replicatorLivenessFailures = new Map();
5901
8174
  this._replicatorLastActivityAt = new Map();
5902
8175
  this._lastLocalAppendAt = 0;
8176
+ this._replicationAnnouncementRetryPending = false;
8177
+ this._replicationAnnouncementRetryGeneration = 0;
5903
8178
  const adaptiveReplicateOptions = options?.replicate && isAdaptiveReplicatorOption(options.replicate)
5904
8179
  ? options.replicate
5905
8180
  : undefined;
@@ -5937,6 +8212,7 @@ let SharedLog = (() => {
5937
8212
  throw new Error("waitForReplicatorRequestMaxAttempts must be a positive number");
5938
8213
  }
5939
8214
  this._closeController = new AbortController();
8215
+ this.setupReplicationAnnouncementRetryFunction();
5940
8216
  this._closeController.signal.addEventListener("abort", () => {
5941
8217
  for (const [_peer, state] of this._replicationInfoRequestByPeer) {
5942
8218
  if (state.timer)
@@ -6002,7 +8278,7 @@ let SharedLog = (() => {
6002
8278
  let localBlocks;
6003
8279
  if (this._nativeBackbone) {
6004
8280
  if (this.node.directory != null) {
6005
- const durable = new AnyBlockStore(await storage.sublevel("blocks"));
8281
+ const durable = await createNativeDurableBlockStore(storage);
6006
8282
  localBlocks = new NativeBackboneWriteThroughBlockStore(this._nativeBackbone.blocks, durable);
6007
8283
  }
6008
8284
  else {
@@ -6010,7 +8286,7 @@ let SharedLog = (() => {
6010
8286
  }
6011
8287
  }
6012
8288
  else {
6013
- localBlocks = new AnyBlockStore(await storage.sublevel("blocks"));
8289
+ localBlocks = await createDefaultDurableBlockStore(storage);
6014
8290
  }
6015
8291
  this.remoteBlocks = new RemoteBlocks({
6016
8292
  local: localBlocks,
@@ -6160,6 +8436,10 @@ let SharedLog = (() => {
6160
8436
  };
6161
8437
  }, PRUNE_DEBOUNCE_INTERVAL);
6162
8438
  await remoteBlocksStartPromise;
8439
+ // Failed native prepares can leave content-addressed bytes behind. Recovery
8440
+ // deliberately preserves them: the reopened lower log is the liveness
8441
+ // authority, while these unreachable bytes are safer than deleting a CID that
8442
+ // may also belong to an acknowledged, restored, or concurrent operation.
6163
8443
  const useNativeBackboneBlocks = this._nativeBackbone && this._logProperties?.replicate === false;
6164
8444
  const nativeBackboneGraph = this._nativeBackbone
6165
8445
  ? useNativeBackboneBlocks
@@ -6202,6 +8482,29 @@ let SharedLog = (() => {
6202
8482
  },
6203
8483
  indexer: logIndex,
6204
8484
  });
8485
+ try {
8486
+ const recovered = await this.recoverNativeStrictDurableTransactionIntent();
8487
+ if (recovered) {
8488
+ await this.reconcileNativeCoordinatesWithLowerCommitMarkers();
8489
+ }
8490
+ }
8491
+ catch (error) {
8492
+ this.poisonNativeStrictDurableTransaction(error);
8493
+ throw error;
8494
+ }
8495
+ // A fresh wrapper alone is not proof of recovery. Clear the cached poison
8496
+ // only after the failed native transaction was compensated (or its pending
8497
+ // native journals were deliberately discarded during close) and the lower log
8498
+ // reopened successfully. Unreferenced content-addressed bytes are preserved;
8499
+ // the reopened lower-log facts, not block presence, determine liveness.
8500
+ if (localBlocks instanceof NativeBackboneWriteThroughBlockStore &&
8501
+ !localBlocks.getNativeDurableCommitFailure() &&
8502
+ (!recoveringNativeDurableFailure ||
8503
+ this._nativeDurableRecoveryReadyForReopen)) {
8504
+ this._nativeDurableCommitFailure = undefined;
8505
+ this._nativeDurableRecoveryReadyForReopen = false;
8506
+ this._nativeDurableRecoveryCids.clear();
8507
+ }
6205
8508
  const resolveHashesForSymbols = (symbols) => {
6206
8509
  const nativeState = this._nativeBackbone ?? this._nativeSharedLogState;
6207
8510
  if (!nativeState) {
@@ -6410,6 +8713,10 @@ let SharedLog = (() => {
6410
8713
  await rangeIterator.close();
6411
8714
  }
6412
8715
  if (this._nativeBackboneCoordinatePersistence) {
8716
+ // A previous explicit drop may have been interrupted after its durable
8717
+ // tombstone was written. Complete that erase before the adapter can expose
8718
+ // any stale coordinate or document state to this backbone.
8719
+ await this._nativeBackboneCoordinatePersistence.resumeDrop?.();
6413
8720
  await this._nativeBackboneCoordinatePersistence.hydrate(backbone);
6414
8721
  this._nativeBackboneCoordinateJournalLastFlushMs = Date.now();
6415
8722
  this.hydrateNativeCoordinateStateFromBackbone(backbone);
@@ -6437,6 +8744,58 @@ let SharedLog = (() => {
6437
8744
  await iterator.close();
6438
8745
  }
6439
8746
  }
8747
+ async reconcileNativeCoordinatesWithLowerCommitMarkers() {
8748
+ if (!this._nativeBackbone) {
8749
+ return;
8750
+ }
8751
+ const hashes = new Set(this._residentEntryCoordinatesByHash?.keys() ?? []);
8752
+ const iterator = this.entryCoordinatesIndex.iterate({});
8753
+ try {
8754
+ for (;;) {
8755
+ const batch = await iterator.next(256);
8756
+ if (batch.length === 0) {
8757
+ break;
8758
+ }
8759
+ for (const result of batch) {
8760
+ hashes.add(result.value.hash);
8761
+ }
8762
+ }
8763
+ }
8764
+ finally {
8765
+ await iterator.close();
8766
+ }
8767
+ if (hashes.size === 0) {
8768
+ return;
8769
+ }
8770
+ const committed = await this.log.entryIndex.hasMany(hashes);
8771
+ const orphaned = [...hashes].filter((hash) => !committed.has(hash));
8772
+ if (orphaned.length === 0) {
8773
+ return;
8774
+ }
8775
+ const coordinateIndex = this.entryCoordinatesIndex;
8776
+ if (coordinateIndex.delIdsNoReturn) {
8777
+ await coordinateIndex.delIdsNoReturn(orphaned);
8778
+ }
8779
+ else if (coordinateIndex.delIds) {
8780
+ await coordinateIndex.delIds(orphaned);
8781
+ }
8782
+ else {
8783
+ await coordinateIndex.del({
8784
+ query: orphaned.length === 1
8785
+ ? { hash: orphaned[0] }
8786
+ : new Or(orphaned.map((hash) => new StringMatch({ key: "hash", value: hash }))),
8787
+ });
8788
+ }
8789
+ for (const hash of orphaned) {
8790
+ this._nativeBackbone.deleteEntryCoordinates(hash);
8791
+ this._nativeSharedLogState?.deleteEntryCoordinates(hash);
8792
+ this._residentEntryCoordinatesByHash?.delete(hash);
8793
+ }
8794
+ const flushed = this.flushNativeBackboneCoordinateJournal();
8795
+ if (isPromiseLike(flushed)) {
8796
+ await flushed;
8797
+ }
8798
+ }
6440
8799
  hydrateNativeCoordinateStateFromBackbone(backbone) {
6441
8800
  const fields = backbone.getEntryCoordinateFields();
6442
8801
  this._nativeSharedLogState?.clearEntryCoordinates();
@@ -6511,7 +8870,10 @@ let SharedLog = (() => {
6511
8870
  }
6512
8871
  async openNativeBackbone(options) {
6513
8872
  this._nativeBackboneCoordinatePersistence = undefined;
8873
+ this._nativeBackboneCoordinatePersistenceStore = undefined;
8874
+ this._nativeBackboneDropStarted = false;
6514
8875
  this._nativeBackboneCoordinateJournalLastFlushMs = 0;
8876
+ this._nativeStrictDurableTransactionJournalState = undefined;
6515
8877
  if (!options) {
6516
8878
  return undefined;
6517
8879
  }
@@ -6548,6 +8910,17 @@ let SharedLog = (() => {
6548
8910
  // peer to re-derive from. Memory-only nodes (no directory) keep the
6549
8911
  // previous in-memory behavior.
6550
8912
  if (options.coordinatePersistence) {
8913
+ if ("store" in options.coordinatePersistence) {
8914
+ this._nativeBackboneCoordinatePersistenceStore =
8915
+ options.coordinatePersistence.store;
8916
+ }
8917
+ else if (options.coordinatePersistence.intentStore) {
8918
+ this._nativeBackboneCoordinatePersistenceStore =
8919
+ options.coordinatePersistence.intentStore;
8920
+ }
8921
+ else if (this.node.directory != null) {
8922
+ throw new Error("Durable nativeBackbone.coordinatePersistence adapters must expose intentStore");
8923
+ }
6551
8924
  this._nativeBackboneCoordinatePersistence =
6552
8925
  createNativeBackboneCoordinatePersistence(options.coordinatePersistence);
6553
8926
  }
@@ -6555,6 +8928,24 @@ let SharedLog = (() => {
6555
8928
  this._nativeBackboneCoordinatePersistence =
6556
8929
  await this.createAutoDerivedCoordinatePersistence(nativeBackboneModule);
6557
8930
  }
8931
+ if (this.node.directory != null &&
8932
+ this._nativeBackboneCoordinatePersistence) {
8933
+ if (this._nativeBackboneCoordinatePersistence.durableBarrier !== true ||
8934
+ typeof this._nativeBackboneCoordinatePersistenceStore
8935
+ ?.durableBarrier !== "function") {
8936
+ throw new Error("Durable nativeBackbone coordinate persistence requires an explicit physical durability barrier");
8937
+ }
8938
+ }
8939
+ if (this._nativeBackboneCoordinatePersistence &&
8940
+ (this._nativeBackboneCoordinatePersistence.compactMaxJournalBytes !=
8941
+ null ||
8942
+ this._nativeBackboneCoordinatePersistence.compactMaxJournalRecords !=
8943
+ null) &&
8944
+ this._nativeBackboneCoordinatePersistence.crashSafeCompaction !== true) {
8945
+ // Durable custom adapters must explicitly advertise an atomic generation
8946
+ // protocol before SharedLog permits automatic WAL compaction.
8947
+ throw new Error("Durable native coordinate persistence compaction thresholds require crashSafeCompaction");
8948
+ }
6558
8949
  return backbone;
6559
8950
  }
6560
8951
  catch (error) {
@@ -6605,6 +8996,7 @@ let SharedLog = (() => {
6605
8996
  directory: ["coordinates", namespace],
6606
8997
  });
6607
8998
  }
8999
+ this._nativeBackboneCoordinatePersistenceStore = store;
6608
9000
  return createNativeBackboneCoordinatePersistence({
6609
9001
  store,
6610
9002
  buffered: true,
@@ -7563,115 +9955,167 @@ let SharedLog = (() => {
7563
9955
  throw error;
7564
9956
  }
7565
9957
  }
7566
- async _close() {
7567
- if (!this._entryCoordinatesIndex && !this._replicationRangeIndex) {
7568
- return;
7569
- }
7570
- if (this._wireSyncSession) {
7571
- this._wireSyncSession.unregisterTopic(this.topic);
7572
- this._wireSyncSession = undefined;
7573
- }
7574
- await this.closeNativeBackboneCoordinatePersistence();
7575
- await this.syncronizer?.close();
7576
- for (const [_key, peerMap] of this.pendingMaturity ?? []) {
7577
- for (const [_key2, info] of peerMap) {
7578
- clearTimeout(info.timeout);
9958
+ async _close(options) {
9959
+ const preserveDropRetryResources = options?.preserveDropRetryResources === true;
9960
+ let firstError;
9961
+ const capture = async (operation) => {
9962
+ try {
9963
+ await operation();
7579
9964
  }
7580
- }
7581
- this.pendingMaturity?.clear();
7582
- this.distributeQueue?.clear();
7583
- this._closeFanoutChannel();
7584
- try {
7585
- this._providerHandle?.close();
7586
- }
7587
- catch {
7588
- // ignore
7589
- }
9965
+ catch (error) {
9966
+ firstError ??= error;
9967
+ }
9968
+ };
9969
+ const captureSync = (operation) => {
9970
+ try {
9971
+ operation();
9972
+ }
9973
+ catch (error) {
9974
+ firstError ??= error;
9975
+ }
9976
+ };
9977
+ captureSync(() => this.cancelCurrentReplicationStateAnnouncementRetry());
9978
+ this.replicationAnnouncementRetryDebounced = undefined;
9979
+ captureSync(() => {
9980
+ if (this._wireSyncSession) {
9981
+ this._wireSyncSession.unregisterTopic(this.topic);
9982
+ this._wireSyncSession = undefined;
9983
+ }
9984
+ });
9985
+ await capture(() => this.closeNativeBackboneCoordinatePersistence());
9986
+ await capture(() => this.syncronizer?.close());
9987
+ captureSync(() => {
9988
+ for (const [_key, peerMap] of this.pendingMaturity ?? []) {
9989
+ for (const [_key2, info] of peerMap)
9990
+ clearTimeout(info.timeout);
9991
+ }
9992
+ this.pendingMaturity?.clear();
9993
+ this.distributeQueue?.clear();
9994
+ });
9995
+ captureSync(() => this._closeFanoutChannel());
9996
+ captureSync(() => this._providerHandle?.close());
7590
9997
  this._providerHandle = undefined;
7591
- this.coordinateToHash?.clear();
7592
- this.recentlyRebalanced?.clear();
7593
- this.uniqueReplicators?.clear();
7594
- this._topicSubscribersCache?.clear();
7595
- this._closeController.abort();
7596
- clearInterval(this.interval);
7597
- this.stopReplicatorLivenessSweep();
7598
- this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn);
7599
- this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn);
7600
- for (const timer of this._repairRetryTimers ?? []) {
7601
- clearTimeout(timer);
7602
- }
7603
- this._repairRetryTimers?.clear();
7604
- this._recentRepairDispatch?.clear();
7605
- this._repairSweepRunning = false;
7606
- this._repairSweepPendingModes?.clear();
7607
- for (const peers of this._repairSweepPendingPeersByMode?.values() ?? []) {
7608
- peers.clear();
7609
- }
7610
- this._repairSweepOptimisticGidPeersPending?.clear();
7611
- this._entryKnownPeers?.clear();
7612
- this._entryKnownPeerObservedAt?.clear();
7613
- this._nativeSharedLogState?.clearEntryKnownPeers();
7614
- this._nativeBackbone?.clearEntryKnownPeers();
7615
- for (const timer of this._joinAuthoritativeRepairTimersByDelay?.values() ??
7616
- []) {
7617
- clearTimeout(timer);
7618
- }
7619
- this._joinAuthoritativeRepairTimersByDelay?.clear();
7620
- this._joinAuthoritativeRepairPeersByDelay?.clear();
7621
- for (const targets of this._repairFrontierByMode?.values() ?? []) {
7622
- targets.clear();
7623
- }
7624
- for (const targets of this._repairFrontierActiveTargetsByMode?.values() ??
7625
- []) {
7626
- targets.clear();
7627
- }
7628
- for (const targets of this._repairFrontierBypassKnownPeersByMode?.values() ??
7629
- []) {
7630
- targets.clear();
7631
- }
7632
- if (this._appendBackfillTimer) {
7633
- clearTimeout(this._appendBackfillTimer);
7634
- this._appendBackfillTimer = undefined;
7635
- }
7636
- this._appendBackfillPendingByTarget?.clear();
7637
- for (const [_k, v] of this._pendingIHave ?? []) {
7638
- v.clear();
7639
- }
7640
- if (this._pendingIHaveExpiryTimer) {
7641
- clearTimeout(this._pendingIHaveExpiryTimer);
7642
- this._pendingIHaveExpiryTimer = undefined;
7643
- this._pendingIHaveExpiryDeadline = Number.POSITIVE_INFINITY;
7644
- }
7645
- this._checkedPrune.close();
7646
- await this.remoteBlocks?.stop?.();
7647
- this._pendingIHave?.clear();
7648
- this.latestReplicationInfoMessage?.clear();
7649
- this._gidPeersHistory?.clear();
7650
- this._peerSyncCapabilities?.clear();
7651
- this._liveRawGossipBatches?.clear();
7652
- this._nativeSharedLogState?.clearGidPeers();
7653
- this._nativeBackbone?.clearGidPeers();
7654
- // Cancel any pending debounced timers so they can't fire after we've torn down
7655
- // indexes/RPC state.
7656
- this.rebalanceParticipationDebounced?.close();
7657
- this.replicationChangeDebounceFn?.close?.();
7658
- this.pruneDebouncedFn?.close?.();
7659
- this.responseToPruneDebouncedFn?.close?.();
9998
+ captureSync(() => {
9999
+ this.coordinateToHash?.clear();
10000
+ this.recentlyRebalanced?.clear();
10001
+ this.uniqueReplicators?.clear();
10002
+ this._topicSubscribersCache?.clear();
10003
+ this._closeController.abort();
10004
+ clearInterval(this.interval);
10005
+ this.stopReplicatorLivenessSweep();
10006
+ });
10007
+ captureSync(() => this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn));
10008
+ captureSync(() => this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn));
10009
+ captureSync(() => {
10010
+ for (const timer of this._repairRetryTimers ?? [])
10011
+ clearTimeout(timer);
10012
+ this._repairRetryTimers?.clear();
10013
+ this._recentRepairDispatch?.clear();
10014
+ this._repairSweepRunning = false;
10015
+ this._repairSweepPendingModes?.clear();
10016
+ for (const peers of this._repairSweepPendingPeersByMode?.values() ?? [])
10017
+ peers.clear();
10018
+ this._repairSweepOptimisticGidPeersPending?.clear();
10019
+ this._entryKnownPeers?.clear();
10020
+ this._entryKnownPeerObservedAt?.clear();
10021
+ this._nativeSharedLogState?.clearEntryKnownPeers();
10022
+ this._nativeBackbone?.clearEntryKnownPeers();
10023
+ for (const timer of this._joinAuthoritativeRepairTimersByDelay?.values() ??
10024
+ [])
10025
+ clearTimeout(timer);
10026
+ this._joinAuthoritativeRepairTimersByDelay?.clear();
10027
+ this._joinAuthoritativeRepairPeersByDelay?.clear();
10028
+ for (const targets of this._repairFrontierByMode?.values() ?? [])
10029
+ targets.clear();
10030
+ for (const targets of this._repairFrontierActiveTargetsByMode?.values() ??
10031
+ [])
10032
+ targets.clear();
10033
+ for (const targets of this._repairFrontierBypassKnownPeersByMode?.values() ??
10034
+ [])
10035
+ targets.clear();
10036
+ if (this._appendBackfillTimer) {
10037
+ clearTimeout(this._appendBackfillTimer);
10038
+ this._appendBackfillTimer = undefined;
10039
+ }
10040
+ this._appendBackfillPendingByTarget?.clear();
10041
+ for (const [_key, value] of this._pendingIHave ?? [])
10042
+ value.clear();
10043
+ if (this._pendingIHaveExpiryTimer) {
10044
+ clearTimeout(this._pendingIHaveExpiryTimer);
10045
+ this._pendingIHaveExpiryTimer = undefined;
10046
+ this._pendingIHaveExpiryDeadline = Number.POSITIVE_INFINITY;
10047
+ }
10048
+ });
10049
+ captureSync(() => this._checkedPrune.close());
10050
+ if (!preserveDropRetryResources) {
10051
+ await capture(() => this.remoteBlocks?.stop?.());
10052
+ }
10053
+ captureSync(() => {
10054
+ this._pendingIHave?.clear();
10055
+ this.latestReplicationInfoMessage?.clear();
10056
+ this._gidPeersHistory?.clear();
10057
+ this._peerSyncCapabilities?.clear();
10058
+ this._liveRawGossipBatches?.clear();
10059
+ this._nativeSharedLogState?.clearGidPeers();
10060
+ this._nativeBackbone?.clearGidPeers();
10061
+ });
10062
+ // Cancel every debounce independently so one faulty close hook cannot keep
10063
+ // the remaining timers or indexes alive.
10064
+ captureSync(() => this.rebalanceParticipationDebounced?.close());
10065
+ captureSync(() => this.replicationChangeDebounceFn?.close?.());
10066
+ captureSync(() => this.pruneDebouncedFn?.close?.());
10067
+ captureSync(() => this.responseToPruneDebouncedFn?.close?.());
7660
10068
  this.pruneDebouncedFn = undefined;
7661
10069
  this.rebalanceParticipationDebounced = undefined;
7662
- await Promise.all([
7663
- this._replicationRangeIndex?.stop?.(),
7664
- this._entryCoordinatesIndex?.stop?.(),
7665
- ]);
7666
- this._replicationRangeIndex = undefined;
7667
- this._entryCoordinatesIndex = undefined;
10070
+ if (!preserveDropRetryResources) {
10071
+ const stopIndex = async (index, forget) => {
10072
+ if (!index) {
10073
+ return;
10074
+ }
10075
+ try {
10076
+ await index.stop?.();
10077
+ forget();
10078
+ }
10079
+ catch (error) {
10080
+ firstError ??= error;
10081
+ }
10082
+ };
10083
+ await stopIndex(this._replicationRangeIndex, () => {
10084
+ this._replicationRangeIndex = undefined;
10085
+ });
10086
+ await stopIndex(this._entryCoordinatesIndex, () => {
10087
+ this._entryCoordinatesIndex = undefined;
10088
+ });
10089
+ }
7668
10090
  this._nativeRangePlanner = undefined;
7669
10091
  this._nativeSharedLogState = undefined;
7670
10092
  this._residentEntryCoordinatesByHash = undefined;
7671
- this.cpuUsage?.stop?.();
7672
- /* this._totalParticipation = 0; */
10093
+ captureSync(() => this.cpuUsage?.stop?.());
10094
+ if (firstError !== undefined) {
10095
+ throw firstError;
10096
+ }
10097
+ }
10098
+ classifyTerminalOwnership(from) {
10099
+ if (this.closed) {
10100
+ return "terminal";
10101
+ }
10102
+ const parentIndex = this.parents?.findIndex((parent) => parent === from) ?? -1;
10103
+ if (from && parentIndex === -1) {
10104
+ throw new TerminalOperationNotStartedError("Could not find from in parents");
10105
+ }
10106
+ return parentIndex !== -1 && (this.parents?.length ?? 0) > 1
10107
+ ? "nonterminal"
10108
+ : "terminal";
7673
10109
  }
7674
10110
  async close(from) {
10111
+ if (this.classifyTerminalOwnership(from) === "nonterminal") {
10112
+ return super.close(from);
10113
+ }
10114
+ // Match Program.end()'s synchronous terminal admission fence before any
10115
+ // SharedLog-specific await or observable teardown can admit a new owner.
10116
+ this.preventParentAttachments();
10117
+ this.ensureNativeDurabilityRuntimeState();
10118
+ this.cancelCurrentReplicationStateAnnouncementRetry();
7675
10119
  // Best-effort: announce that we are going offline before tearing down
7676
10120
  // RPC/subscription state.
7677
10121
  //
@@ -7724,15 +10168,71 @@ let SharedLog = (() => {
7724
10168
  catch {
7725
10169
  // ignore: close should be resilient even if we were never fully started
7726
10170
  }
7727
- const superClosed = await super.close(from);
7728
- if (!superClosed) {
7729
- return superClosed;
10171
+ let firstError;
10172
+ let superClosed = false;
10173
+ try {
10174
+ superClosed = await super.close(from);
10175
+ }
10176
+ catch (error) {
10177
+ if (!this.closed || this.pendingTerminalOperation !== "close") {
10178
+ // Child/base admission failed before Program committed this terminal
10179
+ // transition (including a cleanly closed instance). Lower resources are
10180
+ // still live data or belong to a completed generation, so do not mutate
10181
+ // them while merely propagating the base error.
10182
+ throw error;
10183
+ }
10184
+ firstError = error;
10185
+ }
10186
+ if (!superClosed && firstError === undefined) {
10187
+ return false;
10188
+ }
10189
+ this._nativeStrictDurableTransactionsClosing = true;
10190
+ let strictTransactionsSettled = false;
10191
+ try {
10192
+ await this.settleNativeStrictDurableTransactionsForClose();
10193
+ strictTransactionsSettled = true;
10194
+ }
10195
+ catch (error) {
10196
+ firstError ??= error;
10197
+ }
10198
+ if (!strictTransactionsSettled) {
10199
+ throw firstError;
10200
+ }
10201
+ try {
10202
+ await this.log.close();
10203
+ }
10204
+ catch (error) {
10205
+ firstError ??= error;
10206
+ }
10207
+ try {
10208
+ await this._close();
10209
+ }
10210
+ catch (error) {
10211
+ firstError ??= error;
10212
+ }
10213
+ if (firstError !== undefined) {
10214
+ throw firstError;
7730
10215
  }
7731
- await this._close();
7732
- await this.log.close();
7733
10216
  return true;
7734
10217
  }
7735
10218
  async drop(from) {
10219
+ if (this.classifyTerminalOwnership(from) === "nonterminal") {
10220
+ return super.drop(from);
10221
+ }
10222
+ this.ensureNativeDurabilityRuntimeState();
10223
+ const nativePersistence = this._nativeBackboneCoordinatePersistence;
10224
+ if (nativePersistence &&
10225
+ (typeof nativePersistence.drop !== "function" ||
10226
+ typeof nativePersistence.resumeDrop !== "function" ||
10227
+ nativePersistence.supportsDrop !== true ||
10228
+ nativePersistence.dropIsTerminal !== true)) {
10229
+ // Reject before `super.drop()` can drop child programs or any lower index.
10230
+ throw new TerminalOperationNotStartedError("NativeBackbone coordinate persistence adapters must expose a terminal underlying drop capability and resumeDrop before SharedLog.drop() can erase their namespace");
10231
+ }
10232
+ // Adapter capability validation above is explicitly unstarted. Establish
10233
+ // the terminal fence only after that precondition succeeds.
10234
+ this.preventParentAttachments();
10235
+ this.cancelCurrentReplicationStateAnnouncementRetry();
7736
10236
  // Best-effort: announce that we are going offline before tearing down
7737
10237
  // RPC/subscription state (same reasoning as in `close()`).
7738
10238
  try {
@@ -7769,14 +10269,132 @@ let SharedLog = (() => {
7769
10269
  catch {
7770
10270
  // ignore: drop should be resilient even if we were never fully started
7771
10271
  }
7772
- const superDropped = await super.drop(from);
7773
- if (!superDropped) {
7774
- return superDropped;
10272
+ let firstError;
10273
+ let superDropped = false;
10274
+ try {
10275
+ superDropped = await super.drop(from);
10276
+ }
10277
+ catch (error) {
10278
+ if (!this.closed || this.pendingTerminalOperation !== "drop") {
10279
+ // A fresh drop on a cleanly closed Program is an API rejection, not
10280
+ // permission to erase the already-closed lower log. Likewise, a child
10281
+ // failure before the base drop commits must leave all lower data intact.
10282
+ throw error;
10283
+ }
10284
+ firstError = error;
10285
+ }
10286
+ if (!superDropped && firstError === undefined) {
10287
+ return false;
10288
+ }
10289
+ this._nativeStrictDurableTransactionsClosing = true;
10290
+ const capture = async (operation) => {
10291
+ try {
10292
+ await operation();
10293
+ }
10294
+ catch (error) {
10295
+ firstError ??= error;
10296
+ }
10297
+ };
10298
+ let strictTransactionsSettled = false;
10299
+ try {
10300
+ await this.settleNativeStrictDurableTransactionsForClose();
10301
+ strictTransactionsSettled = true;
10302
+ }
10303
+ catch (error) {
10304
+ firstError ??= error;
10305
+ }
10306
+ if (!strictTransactionsSettled) {
10307
+ throw firstError;
10308
+ }
10309
+ if (nativePersistence) {
10310
+ try {
10311
+ const additionalFiles = this._nativeBackboneCoordinatePersistenceStore
10312
+ ? NATIVE_STRICT_DURABLE_TRANSACTION_INTENT_FILES
10313
+ : [];
10314
+ if (this._nativeBackboneDropStarted) {
10315
+ let resumed;
10316
+ try {
10317
+ resumed = await nativePersistence.resumeDrop();
10318
+ }
10319
+ catch (resumeError) {
10320
+ try {
10321
+ // A corrupt or partial tombstone deliberately restores the
10322
+ // adapter to active so explicit drop can overwrite it. For
10323
+ // transient read/remove failures the adapter stays dropping,
10324
+ // this fallback rejects, and the original recovery error wins.
10325
+ await nativePersistence.drop(additionalFiles);
10326
+ resumed = true;
10327
+ }
10328
+ catch (restartError) {
10329
+ throw new AggregateError([resumeError, restartError], "Failed to resume or restart native backbone drop");
10330
+ }
10331
+ }
10332
+ if (!resumed) {
10333
+ await nativePersistence.drop(additionalFiles);
10334
+ }
10335
+ }
10336
+ else {
10337
+ this._nativeBackboneDropStarted = true;
10338
+ await nativePersistence.drop(additionalFiles);
10339
+ }
10340
+ }
10341
+ catch (error) {
10342
+ firstError ??= error;
10343
+ // Quiesce the failed generation without closing the lower block/index
10344
+ // handles: exact drop retry still owns their destructive cleanup.
10345
+ await capture(() => this.log.close());
10346
+ await capture(() => this._close({ preserveDropRetryResources: true }));
10347
+ throw firstError;
10348
+ }
10349
+ // These in-memory states only stop being recovery-authoritative after all
10350
+ // six persistence files and both alternating intent slots are durably gone.
10351
+ this._nativeStrictDurableTransactionJournalState = undefined;
10352
+ this._nativeStrictDurableDocumentRecoveryDeferred = false;
10353
+ this._nativeStrictDurableTransactionTail = undefined;
10354
+ this._nativeStrictDurableTransactions?.clear();
10355
+ this.clearNativeStrictDurableTransactionFailure();
10356
+ this._nativeDurableCommitFailure = undefined;
10357
+ this._nativeDurableRecoveryReadyForReopen = false;
10358
+ this._nativeDurableRecoveryCids.clear();
10359
+ }
10360
+ let destructiveCleanupFailed = false;
10361
+ const dropIndex = async (index, forget) => {
10362
+ if (!index) {
10363
+ return;
10364
+ }
10365
+ try {
10366
+ await index.drop();
10367
+ forget();
10368
+ }
10369
+ catch (error) {
10370
+ firstError ??= error;
10371
+ destructiveCleanupFailed = true;
10372
+ }
10373
+ };
10374
+ await dropIndex(this._entryCoordinatesIndex, () => {
10375
+ this._entryCoordinatesIndex = undefined;
10376
+ });
10377
+ await dropIndex(this._replicationRangeIndex, () => {
10378
+ this._replicationRangeIndex = undefined;
10379
+ });
10380
+ try {
10381
+ await this.log.drop();
10382
+ }
10383
+ catch (error) {
10384
+ firstError ??= error;
10385
+ destructiveCleanupFailed = true;
10386
+ }
10387
+ if (destructiveCleanupFailed) {
10388
+ // Exact drop retry still owns every failed destructive handle. Quiesce
10389
+ // the rest of the generation without turning an erase failure into a
10390
+ // successful close or forgetting the only object that can retry it.
10391
+ await capture(() => this._close({ preserveDropRetryResources: true }));
10392
+ throw firstError;
10393
+ }
10394
+ await capture(() => this._close());
10395
+ if (firstError !== undefined) {
10396
+ throw firstError;
7775
10397
  }
7776
- await this._entryCoordinatesIndex.drop();
7777
- await this._replicationRangeIndex.drop();
7778
- await this.log.drop();
7779
- await this._close();
7780
10398
  return true;
7781
10399
  }
7782
10400
  async recover() {
@@ -7825,6 +10443,7 @@ let SharedLog = (() => {
7825
10443
  ? msg
7826
10444
  : undefined;
7827
10445
  try {
10446
+ this.throwIfNativeDurableCommitFailed();
7828
10447
  if (!context.from) {
7829
10448
  throw new Error("Missing from in update role message");
7830
10449
  }
@@ -7932,8 +10551,7 @@ let SharedLog = (() => {
7932
10551
  hashes,
7933
10552
  from: rawFrom,
7934
10553
  });
7935
- rawPreparedReceiveSelectionValue =
7936
- await rawPreparedReceiveSelection;
10554
+ rawPreparedReceiveSelectionValue = await rawPreparedReceiveSelection;
7937
10555
  return rawPreparedReceiveSelectionValue;
7938
10556
  };
7939
10557
  // Receive fusion: when this message was resolved from the wire
@@ -7995,8 +10613,7 @@ let SharedLog = (() => {
7995
10613
  if (!prepared) {
7996
10614
  return undefined;
7997
10615
  }
7998
- rawPreparedReceiveSelectionValue =
7999
- prepared.selection;
10616
+ rawPreparedReceiveSelectionValue = prepared.selection;
8000
10617
  rawPreparedReceiveSelection = Promise.resolve(rawPreparedReceiveSelectionValue);
8001
10618
  return { columns: prepared.columns };
8002
10619
  }
@@ -8081,9 +10698,7 @@ let SharedLog = (() => {
8081
10698
  ? msg.preparedHashes
8082
10699
  : heads.map(getExchangeHeadHash);
8083
10700
  const isRepairHint = (msg.reserved[0] & EXCHANGE_HEADS_REPAIR_HINT) !== 0;
8084
- logger.trace(`${this.node.identity.publicKey.hashcode()}: Recieved heads: ${heads.length === 1
8085
- ? headHashes[0]
8086
- : "#" + heads.length}, logId: ${this.log.idString}`);
10701
+ logger.trace(`${this.node.identity.publicKey.hashcode()}: Recieved heads: ${heads.length === 1 ? headHashes[0] : "#" + heads.length}, logId: ${this.log.idString}`);
8087
10702
  if (heads) {
8088
10703
  let filteredHeads;
8089
10704
  let filteredHeadHashes;
@@ -8196,8 +10811,7 @@ let SharedLog = (() => {
8196
10811
  maxReplicas: this.replicas.max?.getValue(this),
8197
10812
  };
8198
10813
  if (!isReplicating &&
8199
- rawPreparedReceiveSelectionValue
8200
- ?.retainedGroupLeaderPlans &&
10814
+ rawPreparedReceiveSelectionValue?.retainedGroupLeaderPlans &&
8201
10815
  rawPreparedReceiveSelectionValue.retainedHashes.length ===
8202
10816
  filteredHeadHashes.length &&
8203
10817
  rawPreparedReceiveSelectionValue.retainedHashes.every((hash, index) => hash === filteredHeadHashes[index])) {
@@ -8217,8 +10831,7 @@ let SharedLog = (() => {
8217
10831
  this._nativeBackbone.planPreparedRawReceiveGroupAssignments?.(filteredHeadHashes, replicaOptions, nativeLeaderOptions, contextFromHash);
8218
10832
  if (nativeRawGroupAssignmentPlans &&
8219
10833
  !nativeRawGroupAssignmentPlans.every((plan) => {
8220
- const keepAsLeader = plan.isLeader ||
8221
- (isRepairHint && plan.fromIsLeader);
10834
+ const keepAsLeader = plan.isLeader || (isRepairHint && plan.fromIsLeader);
8222
10835
  const canKeepWithoutWait = isReplicating
8223
10836
  ? plan.isLeader
8224
10837
  : keepAsLeader;
@@ -8494,8 +11107,7 @@ let SharedLog = (() => {
8494
11107
  }
8495
11108
  let usedNativeReceiveGroupLeaderPlans = false;
8496
11109
  if (!isReplicating) {
8497
- let leaderPlans = usedNativeRawGroupLeaderPlans ||
8498
- usedNativeRawGroupAssignmentPlans
11110
+ let leaderPlans = usedNativeRawGroupLeaderPlans || usedNativeRawGroupAssignmentPlans
8499
11111
  ? receiveGroups.map((group) => group.leaderPlan)
8500
11112
  : usedNativeRawGroups && this._nativeBackbone
8501
11113
  ? await this.planNativeBackboneReceiveGroupLeaders(receiveGroups)
@@ -8673,8 +11285,7 @@ let SharedLog = (() => {
8673
11285
  for (const group of receiveGroups) {
8674
11286
  const fromIsLeader = group.fromIsLeader ?? false;
8675
11287
  const keepAsLeader = group.isLeader === true || (isRepairHint && fromIsLeader);
8676
- if (group.maxReplicasFromNewEntries <
8677
- group.maxReplicasFromHead) {
11288
+ if (group.maxReplicasFromNewEntries < group.maxReplicasFromHead) {
8678
11289
  canUseAllKeptNativeJoinPlan = false;
8679
11290
  break;
8680
11291
  }
@@ -8757,8 +11368,7 @@ let SharedLog = (() => {
8757
11368
  if (fromIsLeader) {
8758
11369
  this.addPeersToGidPeerHistory(group.gid, [contextFromHash]);
8759
11370
  }
8760
- if (group.maxReplicasFromNewEntries <
8761
- group.maxReplicasFromHead) {
11371
+ if (group.maxReplicasFromNewEntries < group.maxReplicasFromHead) {
8762
11372
  (maybeDelete || (maybeDelete = [])).push(group.entries);
8763
11373
  }
8764
11374
  }
@@ -8801,7 +11411,8 @@ let SharedLog = (() => {
8801
11411
  isLeader =
8802
11412
  isLeader ||
8803
11413
  this.node.identity.publicKey.hashcode() === key;
8804
- fromIsLeader = fromIsLeader || contextFromHash === key;
11414
+ fromIsLeader =
11415
+ fromIsLeader || contextFromHash === key;
8805
11416
  },
8806
11417
  });
8807
11418
  }
@@ -8977,10 +11588,7 @@ let SharedLog = (() => {
8977
11588
  this._pendingIHave.size === 0;
8978
11589
  let mergeEntryByHash;
8979
11590
  const materializeMergedEntry = (hash) => {
8980
- mergeEntryByHash ??= new Map(allToMerge.map((entry) => [
8981
- getExchangeHeadHash(entry),
8982
- entry,
8983
- ]));
11591
+ mergeEntryByHash ??= new Map(allToMerge.map((entry) => [getExchangeHeadHash(entry), entry]));
8984
11592
  const entryRef = mergeEntryByHash.get(hash);
8985
11593
  if (!entryRef) {
8986
11594
  throw new Error("Missing merged entry for appended hash");
@@ -9471,18 +12079,20 @@ let SharedLog = (() => {
9471
12079
  }
9472
12080
  }
9473
12081
  else {
9474
- const gid = nativeEntryGid ?? nativeEntry?.gid ?? indexedEntry.value.meta.gid;
12082
+ const gid = nativeEntryGid ??
12083
+ nativeEntry?.gid ??
12084
+ indexedEntry.value.meta.gid;
9475
12085
  const replicaCountByIndex = nativeLeaderHints.replicaCountsByIndex?.[i];
9476
12086
  const replicas = replicaCountByIndex != null && replicaCountByIndex > 0
9477
12087
  ? replicaCountByIndex
9478
- : nativeLeaderHints.replicaCounts.get(hash) ??
12088
+ : (nativeLeaderHints.replicaCounts.get(hash) ??
9479
12089
  decodeReplicas({
9480
12090
  meta: {
9481
12091
  data: nativeEntryData ??
9482
12092
  nativeEntry?.data ??
9483
12093
  indexedEntry.value.meta.data,
9484
12094
  },
9485
- }).getValue(this);
12095
+ }).getValue(this));
9486
12096
  if (!nativeLeaderHints.peerHistoryRemovedFlags?.[i] &&
9487
12097
  !nativeLeaderHints.peerHistoryRemovedHashes.has(hash)) {
9488
12098
  this.removePeerFromGidPeerHistory(from, gid);
@@ -9740,6 +12350,9 @@ let SharedLog = (() => {
9740
12350
  }
9741
12351
  }
9742
12352
  catch (e) {
12353
+ if (e instanceof NativeDurableCommitError) {
12354
+ throw e;
12355
+ }
9743
12356
  if (e instanceof AbortError ||
9744
12357
  e instanceof NotStartedError ||
9745
12358
  e instanceof IndexNotStartedError) {
@@ -9756,20 +12369,28 @@ let SharedLog = (() => {
9756
12369
  logger.error(e);
9757
12370
  }
9758
12371
  finally {
9759
- if (stashBackedRawMessage && stashBackedRawMessage.release()) {
9760
- const syncProfile = this._logProperties?.sync?.profile;
9761
- if (syncProfile) {
9762
- emitSyncProfileEvent(syncProfile, {
9763
- name: "sharedLog.rawReceive.wireStashRelease",
9764
- component: "shared-log",
9765
- entries: stashBackedRawMessage.heads.length,
9766
- messages: 1,
9767
- details: {
9768
- bytesMaterialized: stashBackedRawMessage.bytesMaterializedCount,
9769
- },
9770
- });
12372
+ try {
12373
+ if (stashBackedRawMessage && stashBackedRawMessage.release()) {
12374
+ const syncProfile = this._logProperties?.sync?.profile;
12375
+ if (syncProfile) {
12376
+ emitSyncProfileEvent(syncProfile, {
12377
+ name: "sharedLog.rawReceive.wireStashRelease",
12378
+ component: "shared-log",
12379
+ entries: stashBackedRawMessage.heads.length,
12380
+ messages: 1,
12381
+ details: {
12382
+ bytesMaterialized: stashBackedRawMessage.bytesMaterializedCount,
12383
+ },
12384
+ });
12385
+ }
9771
12386
  }
9772
12387
  }
12388
+ finally {
12389
+ // Every return and every locally swallowed receive error passes this
12390
+ // boundary. Release a native wire stash exactly once first, then surface
12391
+ // any durable mutation poison that arose while handling the message.
12392
+ this.throwIfNativeDurableCommitFailed();
12393
+ }
9773
12394
  }
9774
12395
  }
9775
12396
  async calculateTotalParticipation(options) {
@@ -9880,6 +12501,7 @@ let SharedLog = (() => {
9880
12501
  return set;
9881
12502
  }
9882
12503
  async join(entries, options) {
12504
+ this.throwIfNativeDurableCommitFailed();
9883
12505
  let entriesToReplicate = [];
9884
12506
  const localHashes = options?.replicate && this.log.length > 0
9885
12507
  ? await this.log.entryIndex.hasMany(entries.map((element) => typeof element === "string" ? element : element.hash))
@@ -10007,9 +12629,7 @@ let SharedLog = (() => {
10007
12629
  await persistCoordinate(entry);
10008
12630
  }
10009
12631
  if (messageToSend) {
10010
- await this.rpc.send(messageToSend, {
10011
- priority: CONVERGENCE_MESSAGE_PRIORITY,
10012
- });
12632
+ await this.sendReplicationAnnouncement(messageToSend);
10013
12633
  }
10014
12634
  }
10015
12635
  }
@@ -10431,7 +13051,8 @@ let SharedLog = (() => {
10431
13051
  const skipHashes = this._checkedPrune.pendingDeletes.size > 0
10432
13052
  ? [...this._checkedPrune.pendingDeletes.keys()]
10433
13053
  : [];
10434
- const nativeBackboneHintArrays = this._nativeBackbone;
13054
+ const nativeBackboneHintArrays = this
13055
+ ._nativeBackbone;
10435
13056
  if (skipHashes.length === 0) {
10436
13057
  const allConfirmed = nativeBackboneHintArrays.planRequestPruneAllConfirmed?.(hashes, from, {
10437
13058
  ...this.createNativeLeaderOptions(context),
@@ -10937,29 +13558,46 @@ let SharedLog = (() => {
10937
13558
  return materialized;
10938
13559
  }
10939
13560
  snapshotResidentCoordinateEntries(hashes) {
10940
- if (!this._residentEntryCoordinatesByHash) {
13561
+ const uniqueHashes = new Set([...hashes].filter(Boolean));
13562
+ if (uniqueHashes.size === 0) {
10941
13563
  return undefined;
10942
13564
  }
10943
- const snapshot = new Map();
10944
- for (const hash of new Set([...hashes].filter(Boolean))) {
10945
- const entry = this._residentEntryCoordinatesByHash.get(hash);
13565
+ const entries = new Map();
13566
+ const generations = new Map();
13567
+ const mutationGenerations = (this._nativeCoordinateMutationGenerations ??=
13568
+ new Map());
13569
+ for (const hash of uniqueHashes) {
13570
+ const generation = (mutationGenerations.get(hash) ?? 0) + 1;
13571
+ mutationGenerations.set(hash, generation);
13572
+ generations.set(hash, generation);
13573
+ const entry = this._residentEntryCoordinatesByHash?.get(hash);
10946
13574
  if (entry) {
10947
- snapshot.set(hash, entry);
13575
+ entries.set(hash, entry);
10948
13576
  }
10949
13577
  }
10950
- return snapshot.size === 0 ? undefined : snapshot;
13578
+ return { hashes: uniqueHashes, entries, generations };
10951
13579
  }
10952
- rollbackNativeBackboneCoordinateAppend(appendHash, previousEntries) {
13580
+ rollbackNativeBackboneCoordinateAppend(appendHash, rollback) {
10953
13581
  const backbone = this._nativeBackbone;
10954
13582
  if (!backbone) {
10955
13583
  return;
10956
13584
  }
10957
- backbone.deleteEntryCoordinates(appendHash);
10958
- this._residentEntryCoordinatesByHash?.delete(appendHash);
10959
- if (!previousEntries) {
10960
- return;
10961
- }
10962
- for (const [hash, entry] of previousEntries) {
13585
+ const hashes = rollback?.hashes ?? new Set([appendHash]);
13586
+ const mutationGenerations = (this._nativeCoordinateMutationGenerations ??=
13587
+ new Map());
13588
+ for (const hash of hashes) {
13589
+ const expectedGeneration = rollback?.generations.get(hash);
13590
+ if (expectedGeneration !== undefined &&
13591
+ mutationGenerations.get(hash) !== expectedGeneration) {
13592
+ continue;
13593
+ }
13594
+ backbone.deleteEntryCoordinates(hash);
13595
+ this._nativeSharedLogState?.deleteEntryCoordinates(hash);
13596
+ this._residentEntryCoordinatesByHash?.delete(hash);
13597
+ const entry = rollback?.entries.get(hash);
13598
+ if (!entry) {
13599
+ continue;
13600
+ }
10963
13601
  const fields = isEntryReplicated(entry)
10964
13602
  ? {
10965
13603
  hash: entry.hash,
@@ -10973,9 +13611,41 @@ let SharedLog = (() => {
10973
13611
  ? decodeReplicas(entry).getValue(this)
10974
13612
  : fields.coordinates.length;
10975
13613
  backbone.putEntryCoordinates(fields.hash, fields.gid, fields.coordinates, fields.assignedToRangeBoundary, requestedReplicas, fields.hashNumber);
13614
+ this._nativeSharedLogState?.putEntryCoordinates(fields.hash, fields.gid, fields.coordinates, fields.assignedToRangeBoundary, requestedReplicas, fields.hashNumber);
10976
13615
  this._residentEntryCoordinatesByHash?.set(hash, entry);
10977
13616
  }
10978
13617
  }
13618
+ async rollbackNativeBackboneCoordinateAppendDurably(appendHash, rollback) {
13619
+ this.rollbackNativeBackboneCoordinateAppend(appendHash, rollback);
13620
+ const coordinateIndex = this.entryCoordinatesIndex;
13621
+ const hashes = rollback?.hashes ?? new Set([appendHash]);
13622
+ const mutationGenerations = (this._nativeCoordinateMutationGenerations ??=
13623
+ new Map());
13624
+ for (const hash of hashes) {
13625
+ const expectedGeneration = rollback?.generations.get(hash);
13626
+ if (expectedGeneration !== undefined &&
13627
+ mutationGenerations.get(hash) !== expectedGeneration) {
13628
+ continue;
13629
+ }
13630
+ const previous = rollback?.entries.get(hash);
13631
+ if (previous) {
13632
+ await coordinateIndex.put(this.materializeResidentCoordinateEntry(previous));
13633
+ }
13634
+ else if (coordinateIndex.delIds) {
13635
+ await coordinateIndex.delIds([hash]);
13636
+ }
13637
+ else if (coordinateIndex.delIdsNoReturn) {
13638
+ await coordinateIndex.delIdsNoReturn([hash]);
13639
+ }
13640
+ else {
13641
+ await coordinateIndex.del({ query: { hash } });
13642
+ }
13643
+ }
13644
+ const flushed = this.flushNativeBackboneCoordinateJournal();
13645
+ if (isPromiseLike(flushed)) {
13646
+ await flushed;
13647
+ }
13648
+ }
10979
13649
  persistPreparedCoordinate(properties) {
10980
13650
  const { assignedToRangeBoundary, fields } = properties.prepared;
10981
13651
  const deleteHashes = combineCoordinateDeleteHashes(properties.nextHashes, properties.deleteHashes);
@@ -11112,7 +13782,12 @@ let SharedLog = (() => {
11112
13782
  flushNativeBackboneCoordinateJournal() {
11113
13783
  const backbone = this._nativeBackbone;
11114
13784
  const persistence = this._nativeBackboneCoordinatePersistence;
11115
- if (!backbone || !persistence) {
13785
+ if (!backbone || !persistence || this._nativeBackboneDropStarted) {
13786
+ return undefined;
13787
+ }
13788
+ if (backbone.coordinatePendingJournalLength === 0 &&
13789
+ backbone.documentPendingJournalLength === 0 &&
13790
+ backbone.documentSignerPendingJournalLength === 0) {
11116
13791
  return undefined;
11117
13792
  }
11118
13793
  return mapMaybePromise(persistence.flushJournal(backbone), () => {
@@ -11123,7 +13798,7 @@ let SharedLog = (() => {
11123
13798
  flushNativeBackboneCoordinateJournalOnAppend() {
11124
13799
  const backbone = this._nativeBackbone;
11125
13800
  const persistence = this._nativeBackboneCoordinatePersistence;
11126
- if (!backbone || !persistence) {
13801
+ if (!backbone || !persistence || this._nativeBackboneDropStarted) {
11127
13802
  return undefined;
11128
13803
  }
11129
13804
  if (persistence.flushJournalOnAppend) {
@@ -11163,6 +13838,23 @@ let SharedLog = (() => {
11163
13838
  if (!persistence) {
11164
13839
  return;
11165
13840
  }
13841
+ if (this._nativeBackboneDropStarted) {
13842
+ // `drop()` owns the durable namespace lifecycle. Never flush the live wasm
13843
+ // journals or invoke an ordinary custom close after its tombstone/erase has
13844
+ // started: a close implementation that rewrites cached state could resurrect
13845
+ // files after a successful terminal drop.
13846
+ return;
13847
+ }
13848
+ if (this._nativeDurableCommitFailure &&
13849
+ !this._nativeDurableRecoveryReadyForReopen) {
13850
+ // The failed native transaction was never published by the lower log.
13851
+ // Its coordinate/document/signer records are still only in the wasm
13852
+ // pending journals. Closing without flushing discards that generation;
13853
+ // the next backbone hydrates the last acknowledged checkpoint.
13854
+ await persistence.close?.();
13855
+ this._nativeDurableRecoveryReadyForReopen = true;
13856
+ return;
13857
+ }
11166
13858
  await this.flushNativeBackboneCoordinateJournal();
11167
13859
  await persistence.close?.();
11168
13860
  }
@@ -12012,6 +14704,9 @@ let SharedLog = (() => {
12012
14704
  async findFullReplicaLeaders(replicas, roleAge, peerFilter) {
12013
14705
  const now = Date.now();
12014
14706
  const leaders = new Map();
14707
+ // Strict-only peers are not global fallbacks, but may still own an entry's
14708
+ // coordinates. Remember them so a partial fallback cannot bypass sampling.
14709
+ const excludedStrictPeers = new Set();
12015
14710
  const includeStrict = this._logProperties?.strictFullReplicaFallback !== false;
12016
14711
  const iterator = this.replicationIndex.iterate({}, { shape: { hash: true, timestamp: true, mode: true } });
12017
14712
  try {
@@ -12025,10 +14720,11 @@ let SharedLog = (() => {
12025
14720
  if (peerFilter && !peerFilter.has(range.hash)) {
12026
14721
  continue;
12027
14722
  }
12028
- if (!isMatured(range, now, roleAge)) {
14723
+ if (range.mode === ReplicationIntent.Strict && !includeStrict) {
14724
+ excludedStrictPeers.add(range.hash);
12029
14725
  continue;
12030
14726
  }
12031
- if (range.mode === ReplicationIntent.Strict && !includeStrict) {
14727
+ if (!isMatured(range, now, roleAge)) {
12032
14728
  continue;
12033
14729
  }
12034
14730
  leaders.set(range.hash, { intersecting: true });
@@ -12041,6 +14737,11 @@ let SharedLog = (() => {
12041
14737
  finally {
12042
14738
  await iterator.close();
12043
14739
  }
14740
+ for (const hash of excludedStrictPeers) {
14741
+ if (!leaders.has(hash)) {
14742
+ return undefined;
14743
+ }
14744
+ }
12044
14745
  return leaders.size > 0 ? leaders : undefined;
12045
14746
  }
12046
14747
  async findEntryReplicatedLeaderBatch(entries, options) {
@@ -13074,10 +15775,19 @@ let SharedLog = (() => {
13074
15775
  if (!canReplicate) {
13075
15776
  return false;
13076
15777
  }
13077
- await this.startAnnounceReplicating([dynamicRange], {
13078
- checkDuplicates: false,
13079
- reset: false,
13080
- });
15778
+ try {
15779
+ await this.startAnnounceReplicating([dynamicRange], {
15780
+ checkDuplicates: false,
15781
+ reset: false,
15782
+ });
15783
+ }
15784
+ catch (error) {
15785
+ if (isTransientReplicationAnnouncementError(error) &&
15786
+ this._replicationAnnouncementRetryPending) {
15787
+ return false;
15788
+ }
15789
+ throw error;
15790
+ }
13081
15791
  /* await this._updateRole(newRole, onRoleChange); */
13082
15792
  this.rebalanceParticipationDebounced?.call();
13083
15793
  return true;