@optimystic/db-core 1.19.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/structs.d.ts +41 -8
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/cluster/structs.js +18 -7
- package/dist/src/cluster/structs.js.map +1 -1
- package/dist/src/collection/action.d.ts +29 -0
- package/dist/src/collection/action.d.ts.map +1 -1
- package/dist/src/collection/action.js +6 -0
- package/dist/src/collection/action.js.map +1 -1
- package/dist/src/collection/collection.d.ts +533 -120
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +974 -500
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/head-commit.d.ts +110 -0
- package/dist/src/collection/head-commit.d.ts.map +1 -0
- package/dist/src/collection/head-commit.js +65 -0
- package/dist/src/collection/head-commit.js.map +1 -0
- package/dist/src/collection/index.d.ts +2 -0
- package/dist/src/collection/index.d.ts.map +1 -1
- package/dist/src/collection/index.js +2 -0
- package/dist/src/collection/index.js.map +1 -1
- package/dist/src/collection/segments.d.ts +77 -0
- package/dist/src/collection/segments.d.ts.map +1 -0
- package/dist/src/collection/segments.js +110 -0
- package/dist/src/collection/segments.js.map +1 -0
- package/dist/src/collection/struct.d.ts +156 -1
- package/dist/src/collection/struct.d.ts.map +1 -1
- package/dist/src/collection/struct.js +70 -0
- package/dist/src/collection/struct.js.map +1 -1
- package/dist/src/collection/write-attempts.d.ts +245 -0
- package/dist/src/collection/write-attempts.d.ts.map +1 -0
- package/dist/src/collection/write-attempts.js +439 -0
- package/dist/src/collection/write-attempts.js.map +1 -0
- package/dist/src/collections/tree/struct.d.ts +100 -17
- package/dist/src/collections/tree/struct.d.ts.map +1 -1
- package/dist/src/collections/tree/struct.js +38 -7
- package/dist/src/collections/tree/struct.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts +17 -5
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +112 -26
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/network/i-repo.d.ts +3 -0
- package/dist/src/network/i-repo.d.ts.map +1 -1
- package/dist/src/network/index.d.ts +3 -0
- package/dist/src/network/index.d.ts.map +1 -1
- package/dist/src/network/index.js +3 -0
- package/dist/src/network/index.js.map +1 -1
- package/dist/src/network/promise-shortfall-error.d.ts +40 -0
- package/dist/src/network/promise-shortfall-error.d.ts.map +1 -0
- package/dist/src/network/promise-shortfall-error.js +67 -0
- package/dist/src/network/promise-shortfall-error.js.map +1 -0
- package/dist/src/network/refuse-solo.d.ts +8 -0
- package/dist/src/network/refuse-solo.d.ts.map +1 -0
- package/dist/src/network/refuse-solo.js +8 -0
- package/dist/src/network/refuse-solo.js.map +1 -0
- package/dist/src/network/stale-failure.d.ts +20 -3
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +24 -3
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/network/struct.d.ts +148 -2
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/network/validity-rejected-error.d.ts +28 -0
- package/dist/src/network/validity-rejected-error.d.ts.map +1 -0
- package/dist/src/network/validity-rejected-error.js +51 -0
- package/dist/src/network/validity-rejected-error.js.map +1 -0
- package/dist/src/offline/entry.d.ts +72 -0
- package/dist/src/offline/entry.d.ts.map +1 -0
- package/dist/src/offline/entry.js +73 -0
- package/dist/src/offline/entry.js.map +1 -0
- package/dist/src/offline/index.d.ts +5 -0
- package/dist/src/offline/index.d.ts.map +1 -0
- package/dist/src/offline/index.js +5 -0
- package/dist/src/offline/index.js.map +1 -0
- package/dist/src/offline/journal.d.ts +61 -0
- package/dist/src/offline/journal.d.ts.map +1 -0
- package/dist/src/offline/journal.js +2 -0
- package/dist/src/offline/journal.js.map +1 -0
- package/dist/src/offline/queue.d.ts +484 -0
- package/dist/src/offline/queue.d.ts.map +1 -0
- package/dist/src/offline/queue.js +1236 -0
- package/dist/src/offline/queue.js.map +1 -0
- package/dist/src/offline/reachability.d.ts +25 -0
- package/dist/src/offline/reachability.d.ts.map +1 -0
- package/dist/src/offline/reachability.js +13 -0
- package/dist/src/offline/reachability.js.map +1 -0
- package/dist/src/offline/status.d.ts +148 -0
- package/dist/src/offline/status.d.ts.map +1 -0
- package/dist/src/offline/status.js +63 -0
- package/dist/src/offline/status.js.map +1 -0
- package/dist/src/offline/wake.d.ts +30 -0
- package/dist/src/offline/wake.d.ts.map +1 -0
- package/dist/src/offline/wake.js +68 -0
- package/dist/src/offline/wake.js.map +1 -0
- package/dist/src/testing/test-transactor.d.ts +29 -0
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +50 -0
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/commit-failure-class.d.ts +103 -0
- package/dist/src/transaction/commit-failure-class.d.ts.map +1 -0
- package/dist/src/transaction/commit-failure-class.js +96 -0
- package/dist/src/transaction/commit-failure-class.js.map +1 -0
- package/dist/src/transaction/coordinator.d.ts +165 -41
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +605 -300
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transaction/entry-commit.d.ts +139 -0
- package/dist/src/transaction/entry-commit.d.ts.map +1 -0
- package/dist/src/transaction/entry-commit.js +199 -0
- package/dist/src/transaction/entry-commit.js.map +1 -0
- package/dist/src/transaction/index.d.ts +4 -1
- package/dist/src/transaction/index.d.ts.map +1 -1
- package/dist/src/transaction/index.js +1 -0
- package/dist/src/transaction/index.js.map +1 -1
- package/dist/src/transaction/transaction.d.ts +20 -0
- package/dist/src/transaction/transaction.d.ts.map +1 -1
- package/dist/src/transaction/validator.d.ts +7 -0
- package/dist/src/transaction/validator.d.ts.map +1 -1
- package/dist/src/transaction/validator.js +21 -12
- package/dist/src/transaction/validator.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +1 -0
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +61 -28
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor-source.d.ts +22 -11
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +9 -13
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/dist/src/utility/visit-every.d.ts +14 -0
- package/dist/src/utility/visit-every.d.ts.map +1 -0
- package/dist/src/utility/visit-every.js +27 -0
- package/dist/src/utility/visit-every.js.map +1 -0
- package/package.json +1 -1
- package/src/cluster/structs.ts +42 -7
- package/src/collection/action.ts +34 -0
- package/src/collection/collection.ts +1048 -543
- package/src/collection/head-commit.ts +148 -0
- package/src/collection/index.ts +2 -0
- package/src/collection/segments.ts +149 -0
- package/src/collection/struct.ts +183 -1
- package/src/collection/write-attempts.ts +506 -0
- package/src/collections/tree/struct.ts +97 -17
- package/src/collections/tree/tree.ts +125 -29
- package/src/index.ts +2 -0
- package/src/network/i-repo.ts +3 -0
- package/src/network/index.ts +3 -0
- package/src/network/promise-shortfall-error.ts +69 -0
- package/src/network/refuse-solo.ts +7 -0
- package/src/network/stale-failure.ts +27 -4
- package/src/network/struct.ts +154 -2
- package/src/network/validity-rejected-error.ts +55 -0
- package/src/offline/entry.ts +127 -0
- package/src/offline/index.ts +4 -0
- package/src/offline/journal.ts +60 -0
- package/src/offline/queue.ts +1342 -0
- package/src/offline/reachability.ts +32 -0
- package/src/offline/status.ts +183 -0
- package/src/offline/wake.ts +67 -0
- package/src/testing/test-transactor.ts +53 -0
- package/src/transaction/commit-failure-class.ts +133 -0
- package/src/transaction/coordinator.ts +696 -349
- package/src/transaction/entry-commit.ts +285 -0
- package/src/transaction/index.ts +4 -0
- package/src/transaction/transaction.ts +21 -0
- package/src/transaction/validator.ts +22 -12
- package/src/transactor/network-transactor.ts +62 -28
- package/src/transactor/transactor-source.ts +29 -13
- package/src/utility/visit-every.ts +25 -0
|
@@ -1,15 +1,22 @@
|
|
|
1
1
|
import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, PendResult, CommitRequest, CommitResult, WriteFailure, ActionId, WriteDurability, BlockContentDigests } from "../index.js";
|
|
2
|
+
import type { BlockBaseRevs } from "../network/struct.js";
|
|
2
3
|
import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions, ReadDependency } from "./transaction.js";
|
|
3
4
|
import type { PeerId } from "../network/types.js";
|
|
4
|
-
import { isConflictFailure, isExpiryFailure } from "../network/stale-failure.js";
|
|
5
|
+
import { isConflictFailure, isExpiryFailure, isRejectedFailure, isShortfallFailure } from "../network/stale-failure.js";
|
|
5
6
|
import { withBlockRevs } from "../network/durability.js";
|
|
6
7
|
import { TransactionExpiredError } from "../network/transaction-expired-error.js";
|
|
8
|
+
import { PromiseShortfallError } from "../network/promise-shortfall-error.js";
|
|
9
|
+
import { ValidityRejectedError } from "../network/validity-rejected-error.js";
|
|
10
|
+
import { refuseSoloField } from "../network/refuse-solo.js";
|
|
7
11
|
import { TransactionTooLargeError } from "../network/transaction-size.js";
|
|
8
12
|
import { BlockTooLargeError, oversizedBlock } from "../network/block-size.js";
|
|
9
13
|
import type { Collection, CollectionSnapshot, RefreshReport } from "../collection/collection.js";
|
|
10
|
-
import type { SyncOptions } from "../collection/index.js";
|
|
14
|
+
import type { InFlightCycle, SyncOptions } from "../collection/index.js";
|
|
15
|
+
import type { StagedAttempt } from "../collection/write-attempts.js";
|
|
11
16
|
import { TornActionError } from "../collection/struct.js";
|
|
12
17
|
import { isTransactionExpired, clampPriority } from "./transaction.js";
|
|
18
|
+
import { EntryParticipants, writeContextOf } from "./entry-commit.js";
|
|
19
|
+
import type { EntryCommit, RefreshParticipant, WriteContext } from "./entry-commit.js";
|
|
13
20
|
import { Log } from "../log/log.js";
|
|
14
21
|
import { blockIdsForTransforms, copyTransforms } from "../transform/helpers.js";
|
|
15
22
|
import { describeTouchedBlocks, blockDigestsField, baseRevsField } from "../transform/digest.js";
|
|
@@ -17,6 +24,7 @@ import { collectOperations, hashOperations } from "./operations-hash.js";
|
|
|
17
24
|
import { CoordinatorConcurrentStampError, CoordinatorPartialCommitError, CoordinatorStaleLossError } from "./errors.js";
|
|
18
25
|
import { recordCollectionDurability, transactionDurability, type TransactionDurability } from "./durability.js";
|
|
19
26
|
import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
|
|
27
|
+
import { visitEveryThenThrow } from "../utility/visit-every.js";
|
|
20
28
|
import { createLogger } from "../logger.js";
|
|
21
29
|
|
|
22
30
|
const log = createLogger('trx:coordinator');
|
|
@@ -52,44 +60,80 @@ class PendRejectedError extends Error {
|
|
|
52
60
|
}
|
|
53
61
|
}
|
|
54
62
|
|
|
55
|
-
/** The error a refused pend of `collectionIds` stands for: its
|
|
56
|
-
*
|
|
63
|
+
/** The error a refused pend of `collectionIds` stands for: its validity rejection — never retried, since
|
|
64
|
+
* a re-drive sends the same content to the same checks — its expiry — never retried, since a re-drive's
|
|
65
|
+
* expiration comes from the same clock — or its promise shortfall — never retried, since a re-drive
|
|
66
|
+
* meets the same silent members — or a rejection carrying whether a re-read can clear it.
|
|
57
67
|
* Retryability comes from the response itself: a producer that classified the failure sets `conflict`,
|
|
58
68
|
* and only where none did is it inferred from `missing`/`pending` (`isConflictFailure`). */
|
|
59
|
-
function pendRefusalError(collectionIds: readonly CollectionId[], refusal: WriteFailure): TransactionExpiredError | PendRejectedError {
|
|
60
|
-
return
|
|
61
|
-
|
|
62
|
-
|
|
69
|
+
function pendRefusalError(collectionIds: readonly CollectionId[], refusal: WriteFailure): ValidityRejectedError | TransactionExpiredError | PromiseShortfallError | PendRejectedError {
|
|
70
|
+
if (isRejectedFailure(refusal)) return new ValidityRejectedError(refusal.rejected);
|
|
71
|
+
if (isExpiryFailure(refusal)) return new TransactionExpiredError(refusal.expired);
|
|
72
|
+
if (isShortfallFailure(refusal)) return new PromiseShortfallError(refusal.shortfall);
|
|
73
|
+
return new PendRejectedError(collectionIds, isConflictFailure(refusal), refusal.reason, refusal.staleAt);
|
|
63
74
|
}
|
|
64
75
|
|
|
65
76
|
/** A transaction refused before anything was sent: its pend, or a block it would leave, too large to send. */
|
|
66
77
|
type TooLargeError = TransactionTooLargeError | BlockTooLargeError;
|
|
67
78
|
|
|
68
79
|
/** How the pend phase failed. */
|
|
69
|
-
type PendPhaseFailure = { success: false; error: string; staleLoss?: boolean; expired?: TransactionExpiredError; tooLarge?: TooLargeError };
|
|
80
|
+
type PendPhaseFailure = { success: false; error: string; staleLoss?: boolean; rejected?: ValidityRejectedError; expired?: TransactionExpiredError; shortfall?: PromiseShortfallError; tooLarge?: TooLargeError };
|
|
70
81
|
|
|
71
|
-
/**
|
|
72
|
-
*
|
|
73
|
-
|
|
82
|
+
/**
|
|
83
|
+
* One collection's share of an attempt, as the phases send it. For a transaction's attempt it is
|
|
84
|
+
* described from the live tracker after the log append ({@link TransactionCoordinator.declareTouchedBlocks});
|
|
85
|
+
* for a queued entry's it is read off the head commit's `StagedAttempt` ({@link collectionAttemptOf}),
|
|
86
|
+
* which `stageAttempt` described the same way.
|
|
87
|
+
*/
|
|
88
|
+
type CollectionAttempt = {
|
|
89
|
+
readonly transforms: Transforms;
|
|
90
|
+
/** The log tail block its entry was appended to — the critical block, committed first. */
|
|
91
|
+
readonly tailId: BlockId;
|
|
92
|
+
/** The revision its log entry was stamped with: the ONE number the pend, the commit and the
|
|
93
|
+
* local record must all repeat. Captured at the append, never recomputed after a round trip. */
|
|
94
|
+
readonly rev: number;
|
|
95
|
+
/** The base each of its update-only blocks' operations were computed against
|
|
96
|
+
* (`PendRequest.baseRevs`), from the tracker's pins. */
|
|
97
|
+
readonly baseRevs: BlockBaseRevs;
|
|
98
|
+
/** What its commit declares about its blocks (`CommitRequest.blockDigests`), or `undefined` when
|
|
99
|
+
* nothing could be described. */
|
|
100
|
+
readonly blockDigests: BlockContentDigests | undefined;
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
/** A head commit's staged attempt as the phases send it: its transforms are the attempt tracker's. */
|
|
104
|
+
function collectionAttemptOf(staged: StagedAttempt): CollectionAttempt {
|
|
105
|
+
return {
|
|
106
|
+
transforms: staged.tracker.transforms,
|
|
107
|
+
tailId: staged.tailId,
|
|
108
|
+
rev: staged.rev,
|
|
109
|
+
baseRevs: staged.baseRevs,
|
|
110
|
+
blockDigests: staged.blockDigests,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
74
113
|
|
|
75
114
|
/**
|
|
76
115
|
* The pend phase's failure for `reasons` — what every failed pend threw or stood for, at least one.
|
|
77
116
|
* The error text is the first reason's. The attempt is a retryable clean stale loss only if at least one
|
|
78
117
|
* reason was a conflicting pend ({@link PendRejectedError}`.conflict`) AND none was a hard failure: a
|
|
79
118
|
* single hard failure (storage/policy rejection, or a thrown/unavailable transactor) will not clear on a
|
|
80
|
-
* re-read, so re-driving it would just burn the retry budget.
|
|
81
|
-
* to send, are hard failures carried whole: each is the error
|
|
119
|
+
* re-read, so re-driving it would just burn the retry budget. A validity rejection, an expiry, a promise
|
|
120
|
+
* shortfall, and a pend or a block too large to send, are hard failures carried whole: each is the error
|
|
121
|
+
* the commit surfaces.
|
|
82
122
|
*/
|
|
83
123
|
function pendFailure(reasons: readonly unknown[]): PendPhaseFailure {
|
|
84
124
|
const first = reasons[0];
|
|
85
125
|
const conflicts = reasons.filter(reason => reason instanceof PendRejectedError && reason.conflict);
|
|
126
|
+
const rejected = reasons.find((reason): reason is ValidityRejectedError => reason instanceof ValidityRejectedError);
|
|
86
127
|
const expired = reasons.find((reason): reason is TransactionExpiredError => reason instanceof TransactionExpiredError);
|
|
128
|
+
const shortfall = reasons.find((reason): reason is PromiseShortfallError => reason instanceof PromiseShortfallError);
|
|
87
129
|
const tooLarge = reasons.find((reason): reason is TooLargeError => reason instanceof TransactionTooLargeError || reason instanceof BlockTooLargeError);
|
|
88
130
|
return {
|
|
89
131
|
success: false,
|
|
90
132
|
error: first instanceof Error ? first.message : String(first),
|
|
91
133
|
staleLoss: conflicts.length > 0 && conflicts.length === reasons.length,
|
|
134
|
+
...(rejected === undefined ? {} : { rejected }),
|
|
92
135
|
...(expired === undefined ? {} : { expired }),
|
|
136
|
+
...(shortfall === undefined ? {} : { shortfall }),
|
|
93
137
|
...(tooLarge === undefined ? {} : { tooLarge }),
|
|
94
138
|
};
|
|
95
139
|
}
|
|
@@ -122,8 +166,17 @@ type CommitCycle = {
|
|
|
122
166
|
* carries who holds it, `undefined` where the refresh that saved it could not say (see
|
|
123
167
|
* {@link TransactionDurability}). */
|
|
124
168
|
saved: Map<CollectionId, WriteDurability | undefined>;
|
|
169
|
+
/** What every send of the commit carries ({@link InFlightCycle}) — recorded with each
|
|
170
|
+
* participant's in-flight mark, so a refresh that finishes a torn attempt re-sends it on the same
|
|
171
|
+
* terms, and read into every attempt's {@link WriteContext}. Taken from the commit's options. */
|
|
172
|
+
readonly inFlight: InFlightCycle;
|
|
125
173
|
};
|
|
126
174
|
|
|
175
|
+
/** What a commit's `options` ask of every send of its cycle. */
|
|
176
|
+
function inFlightCycleOf(options: SyncOptions | undefined): InFlightCycle {
|
|
177
|
+
return options?.refuseSoloCommit === true ? { refuseSolo: true } : {};
|
|
178
|
+
}
|
|
179
|
+
|
|
127
180
|
/** How one collection's commit ended ({@link TransactionCoordinator.commitCollection}, or its share of
|
|
128
181
|
* a batched commit's one answer). */
|
|
129
182
|
type CollectionCommitOutcome =
|
|
@@ -179,9 +232,15 @@ type CoordinationResult =
|
|
|
179
232
|
/** True when the failure was a clean optimistic-concurrency conflict (stale loss / pending
|
|
180
233
|
* contention) with nothing durable — i.e. safe to re-drive after a re-read. */
|
|
181
234
|
staleLoss?: boolean;
|
|
235
|
+
/** Present when a pend was refused because the members that sank it judged the transaction
|
|
236
|
+
* invalid: the error the commit surfaces, never a stale loss. */
|
|
237
|
+
rejected?: ValidityRejectedError;
|
|
182
238
|
/** Present when a pend or a commit was refused because the transaction had expired by the
|
|
183
239
|
* cohort's clocks: the error the commit surfaces, never a stale loss. */
|
|
184
240
|
expired?: TransactionExpiredError;
|
|
241
|
+
/** Present when a pend was refused because too few cohort members answered its promise round:
|
|
242
|
+
* the error the commit surfaces, never a stale loss. */
|
|
243
|
+
shortfall?: PromiseShortfallError;
|
|
185
244
|
/** Present when the transaction was refused before anything was sent because its pend, or a block
|
|
186
245
|
* it would leave, was too large to send: the error the commit surfaces, never a stale loss. */
|
|
187
246
|
tooLarge?: TooLargeError;
|
|
@@ -194,6 +253,135 @@ function writtenDurability(written: ReadonlyMap<CollectionId, WriteDurability |
|
|
|
194
253
|
return written.size === 0 ? undefined : transactionDurability(written);
|
|
195
254
|
}
|
|
196
255
|
|
|
256
|
+
/**
|
|
257
|
+
* The error an attempt whose phases failed with NOTHING committed is thrown as. A clean stale loss
|
|
258
|
+
* — every refusal a conflict a re-read can clear — is {@link CoordinatorStaleLossError}, which the
|
|
259
|
+
* retry loop re-drives. The four refusals a re-drive can never clear are thrown by name: a validity
|
|
260
|
+
* rejection (the retry's pend would send the same content to the same checks), an expiry (the
|
|
261
|
+
* retry's expiration comes from the same clock), a promise shortfall (the retry meets the same
|
|
262
|
+
* silent members), and a pend or block too large to send. Anything else is a plain error, the
|
|
263
|
+
* historical fail-fast for a hard failure.
|
|
264
|
+
*/
|
|
265
|
+
function cleanFailureError(result: CoordinationResult & { success: false }, failed: readonly CollectionId[]): Error {
|
|
266
|
+
if (result.staleLoss) {
|
|
267
|
+
return new CoordinatorStaleLossError(failed, result.error);
|
|
268
|
+
}
|
|
269
|
+
return result.rejected ?? result.expired ?? result.shortfall ?? result.tooLarge
|
|
270
|
+
?? new Error(`Transaction commit failed: ${result.error}`);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* What one retry cycle ({@link TransactionCoordinator.commitAttempts}) drives: a transaction's
|
|
275
|
+
* whole-collection commit ({@link TransactionCoordinator.commit}) or a queued entry's slice commit
|
|
276
|
+
* ({@link TransactionCoordinator.commitEntry}). Each attempt and each refresh round is the driver's;
|
|
277
|
+
* the loop owns the budget, the backoff, and the exit where a refresh saved everything.
|
|
278
|
+
*/
|
|
279
|
+
type AttemptDriver = {
|
|
280
|
+
/** Whether the cycle opens with a refresh round before its first attempt. The per-entry commit
|
|
281
|
+
* does, as `Collection.commitHead` does: the queue re-drives an entry under the id of an attempt
|
|
282
|
+
* that may have landed, and the opening refresh is what finds and finishes that attempt's entry
|
|
283
|
+
* instead of committing a second copy. A transaction's commit does not; its id is fresh. */
|
|
284
|
+
readonly opensWithRefresh: boolean;
|
|
285
|
+
/** One attempt under the aged retry `priority`: every collection it committed, with who holds
|
|
286
|
+
* it. Throws {@link CoordinatorStaleLossError} for a clean loss, and for a partial landing retried
|
|
287
|
+
* forward, whose committed participants it has already recorded in `cycle.saved`. */
|
|
288
|
+
attempt(priority: number): Promise<ReadonlyMap<CollectionId, WriteDurability>>;
|
|
289
|
+
/** The refresh round between attempts — see {@link TransactionCoordinator.refreshBetweenAttempts}. */
|
|
290
|
+
refresh(lastChance: boolean): Promise<TornActionError | undefined>;
|
|
291
|
+
/** Whether an attempt starting now would have nothing left to commit, so that once a refresh has
|
|
292
|
+
* saved a participant the write is finished without going round again. */
|
|
293
|
+
nothingLeft(): boolean;
|
|
294
|
+
};
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* The retry budget one {@link TransactionCoordinator.commitAttempts} cycle spends: attempts and
|
|
298
|
+
* refused refresh rounds alike are counted against `maxAttempts`, under the wall-clock `deadlineMs`
|
|
299
|
+
* and the abort `signal`, with the same jittered backoff policy `Collection.sync` uses between
|
|
300
|
+
* rounds. There is no forward-progress notion here (a commit either lands or it does not), so this
|
|
301
|
+
* simply bounds how many times a losing write is re-driven. The single-collection accounting
|
|
302
|
+
* (`AttemptCycle`) also checks for a wedged revision view; this one cannot yet, since the confirmed
|
|
303
|
+
* revision does not reach it (backlog `debt-multi-collection-retry-cannot-see-the-taken-revision`).
|
|
304
|
+
*/
|
|
305
|
+
class RetryBudget {
|
|
306
|
+
private readonly maxAttempts: number;
|
|
307
|
+
private readonly backoff: { baseMs: number; capMs: number };
|
|
308
|
+
private readonly deadlineMs: number | undefined;
|
|
309
|
+
private readonly signal: AbortSignal | undefined;
|
|
310
|
+
private readonly rand: SyncOptions['rand'];
|
|
311
|
+
private readonly startedAt = Date.now();
|
|
312
|
+
/** Consecutive clean stale losses: attempts, and refresh rounds refused for a cause that can clear. */
|
|
313
|
+
private losses = 0;
|
|
314
|
+
private lastLoss: CoordinatorStaleLossError | undefined;
|
|
315
|
+
|
|
316
|
+
constructor(options?: SyncOptions) {
|
|
317
|
+
this.maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
|
|
318
|
+
this.backoff = {
|
|
319
|
+
baseMs: options?.baseBackoffMs ?? DefaultBaseBackoffMs,
|
|
320
|
+
capMs: options?.maxBackoffMs ?? DefaultMaxBackoffMs,
|
|
321
|
+
};
|
|
322
|
+
this.deadlineMs = options?.deadlineMs;
|
|
323
|
+
this.signal = options?.signal;
|
|
324
|
+
this.rand = options?.rand;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** The checks before an attempt: the abort signal, and — a progress-agnostic ceiling, once at
|
|
328
|
+
* least one loss was taken — the wall-clock deadline, reported as that loss. */
|
|
329
|
+
assertOpen(): void {
|
|
330
|
+
if (this.signal?.aborted) {
|
|
331
|
+
throw makeAbortError(this.signal);
|
|
332
|
+
}
|
|
333
|
+
if (this.lastLoss !== undefined && this.pastDeadline()) {
|
|
334
|
+
throw this.lastLoss;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** The aged retry priority for the next attempt, from the losses taken so far, so a
|
|
339
|
+
* repeatedly-losing write out-ranks fresh (priority-0) rivals in the cluster's `resolveRace`.
|
|
340
|
+
* Fairness-only and capped at `MaxPriority`; 0 on the first attempt, so the initial pend
|
|
341
|
+
* serializes exactly as before. */
|
|
342
|
+
priority(): number {
|
|
343
|
+
return clampPriority(this.losses);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/** Count a clean stale loss of an attempt; thrown back when that spends the budget. */
|
|
347
|
+
recordLoss(loss: CoordinatorStaleLossError): void {
|
|
348
|
+
this.lastLoss = loss;
|
|
349
|
+
this.losses++;
|
|
350
|
+
if (this.losses >= this.maxAttempts) {
|
|
351
|
+
throw loss;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/** The backoff before a refresh round: none ahead of a cycle's opening round, there being no
|
|
356
|
+
* race to space out; otherwise jittered and growing with the losses taken, so a herd of writers
|
|
357
|
+
* that lost the same race does not re-collide on the next tick. */
|
|
358
|
+
async pause(): Promise<void> {
|
|
359
|
+
if (this.losses === 0) {
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
await abortableDelay(jitteredBackoffMs(this.losses - 1, this.backoff, this.rand), this.signal);
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** Whether the next refresh round spends the last of the budget, so a participant whose
|
|
366
|
+
* half-landed write is refused again settles it rather than ask for a round it will not get. */
|
|
367
|
+
lastChance(): boolean {
|
|
368
|
+
return this.losses + 1 >= this.maxAttempts;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/** Count a refresh round refused for a cause that can clear (a `completion-refused`
|
|
372
|
+
* {@link TornActionError}); thrown back when that spends the budget or passes the deadline. */
|
|
373
|
+
recordRefusedRound(refused: TornActionError): void {
|
|
374
|
+
this.losses++;
|
|
375
|
+
if (this.losses >= this.maxAttempts || this.pastDeadline()) {
|
|
376
|
+
throw refused;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
private pastDeadline(): boolean {
|
|
381
|
+
return this.deadlineMs !== undefined && Date.now() - this.startedAt >= this.deadlineMs;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
197
385
|
/**
|
|
198
386
|
* What a pend this coordinator sends carries for the receiving cluster members to re-check.
|
|
199
387
|
*
|
|
@@ -426,7 +614,9 @@ export class TransactionCoordinator {
|
|
|
426
614
|
* unavailable transactor, unreachable cluster) also propagates without retry — including a pend
|
|
427
615
|
* or commit the cohort refused because the transaction had expired by its members' clocks, which
|
|
428
616
|
* surfaces as {@link TransactionExpiredError} (inside a partial commit's `reason` once a sibling
|
|
429
|
-
* has landed)
|
|
617
|
+
* has landed), a pend too few of the cohort's members answered, which surfaces as
|
|
618
|
+
* {@link PromiseShortfallError}, and a pend the members that sank it judged invalid, which surfaces
|
|
619
|
+
* as {@link ValidityRejectedError}.
|
|
430
620
|
*
|
|
431
621
|
* A participant can also be SAVED between attempts: an attempt that reported a loss may still
|
|
432
622
|
* have stored that participant's log tail, and the refresh before the next attempt then finishes
|
|
@@ -464,13 +654,44 @@ export class TransactionCoordinator {
|
|
|
464
654
|
inFlightDisposers: [],
|
|
465
655
|
participants: new Set(),
|
|
466
656
|
saved: new Map(),
|
|
657
|
+
inFlight: inFlightCycleOf(options),
|
|
658
|
+
};
|
|
659
|
+
const driver: AttemptDriver = {
|
|
660
|
+
opensWithRefresh: false,
|
|
661
|
+
attempt: priority => {
|
|
662
|
+
// Age the transaction's advisory priority by the losses taken so far (see
|
|
663
|
+
// RetryBudget.priority). Set on the transaction itself because a validated pend carries
|
|
664
|
+
// the transaction, and a member's race resolution reads the priority off it; excluded
|
|
665
|
+
// from the tx id / client signature, so bumping it does not churn identity. Left
|
|
666
|
+
// untouched on the first attempt so the initial pend serializes exactly as before.
|
|
667
|
+
if (priority > 0) {
|
|
668
|
+
transaction.priority = priority;
|
|
669
|
+
}
|
|
670
|
+
return this.commitOnce(transaction, cycle);
|
|
671
|
+
},
|
|
672
|
+
refresh: lastChance => this.refreshBetweenAttempts(this.registeredRefreshers(), cycle, lastChance),
|
|
673
|
+
// Tested on what is still STAGED, not on "every participant saved": a participant can be
|
|
674
|
+
// saved and still hold actions staged after the attempt began, and those the next attempt
|
|
675
|
+
// must commit.
|
|
676
|
+
nothingLeft: () => this.stagedCollections().length === 0,
|
|
467
677
|
};
|
|
468
678
|
try {
|
|
469
|
-
|
|
679
|
+
const written = await this.commitAttempts(driver, cycle, options);
|
|
680
|
+
// Released on every successful exit. An attempt that commits releases it itself; this is
|
|
681
|
+
// for the exit where the refresh between attempts saved what was left, whose consume and
|
|
682
|
+
// replay already gave each saved participant the success path's local treatment.
|
|
683
|
+
this.stampData.delete(transaction.stamp.id);
|
|
684
|
+
return written;
|
|
470
685
|
} catch (err) {
|
|
471
686
|
// The ONE place a failure leaves this commit, so no exit — present or future — can skip
|
|
472
687
|
// reporting a participant that is already saved.
|
|
473
|
-
|
|
688
|
+
const reported = this.reportSaved(transaction.id, cycle, err);
|
|
689
|
+
if (reported instanceof CoordinatorPartialCommitError) {
|
|
690
|
+
// The stamp is released, as on the attempt-level partial path (which has already
|
|
691
|
+
// released it): the transaction can no longer be cleanly rolled back.
|
|
692
|
+
this.stampData.delete(transaction.stamp.id);
|
|
693
|
+
}
|
|
694
|
+
throw reported;
|
|
474
695
|
} finally {
|
|
475
696
|
// The in-flight marks must outlive each individual attempt (see CommitCycle), so they are
|
|
476
697
|
// cleared here, on every exit: return, stale-loss exhaustion, a partial landing, a hard
|
|
@@ -481,45 +702,32 @@ export class TransactionCoordinator {
|
|
|
481
702
|
}
|
|
482
703
|
}
|
|
483
704
|
|
|
484
|
-
/**
|
|
485
|
-
*
|
|
486
|
-
*
|
|
487
|
-
*
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
throw makeAbortError(signal);
|
|
505
|
-
}
|
|
506
|
-
// Progress-agnostic ceiling: once we've taken at least one loss, give up if the
|
|
507
|
-
// wall-clock deadline passed (independent of the attempt cap).
|
|
508
|
-
if (deadlineMs !== undefined && lastLoss && Date.now() - startedAt >= deadlineMs) {
|
|
509
|
-
throw lastLoss;
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
// Age the transaction's advisory priority by the number of losses taken so far, so a
|
|
513
|
-
// repeatedly-losing transaction out-ranks fresh (priority-0) rivals in the cluster's
|
|
514
|
-
// resolveRace. Fairness-only and capped at MaxPriority; excluded from the tx id / client
|
|
515
|
-
// signature, so bumping it here does not churn identity. Left untouched on the first
|
|
516
|
-
// attempt (staleLosses == 0) so the initial pend serializes exactly as before.
|
|
517
|
-
if (staleLosses > 0) {
|
|
518
|
-
transaction.priority = clampPriority(staleLosses);
|
|
705
|
+
/**
|
|
706
|
+
* The retry loop behind {@link commit} and {@link commitEntry}: every attempt, and the refresh
|
|
707
|
+
* rounds between them, over the `driver`'s steps and the budget `options` sets
|
|
708
|
+
* ({@link RetryBudget}). Records what it learns in `cycle`; the caller owns turning a failure into
|
|
709
|
+
* the right report. On success, reports every participant the cycle saved together with what the
|
|
710
|
+
* final attempt committed (see {@link commit}'s return).
|
|
711
|
+
*
|
|
712
|
+
* Only a CLEAN stale loss is retried. A partial landing on a hard failure, an expired transaction,
|
|
713
|
+
* an unavailable transactor, etc. all propagate unchanged. Once a refresh round has saved a
|
|
714
|
+
* participant and the driver has nothing left to commit, the write is finished and the loop
|
|
715
|
+
* returns at once rather than going round again, where an abort, the deadline or an expiry check
|
|
716
|
+
* could still report failure for a write that is entirely saved.
|
|
717
|
+
*/
|
|
718
|
+
private async commitAttempts(driver: AttemptDriver, cycle: CommitCycle, options?: SyncOptions): Promise<TransactionDurability | undefined> {
|
|
719
|
+
const budget = new RetryBudget(options);
|
|
720
|
+
if (driver.opensWithRefresh) {
|
|
721
|
+
budget.assertOpen();
|
|
722
|
+
await this.refreshRounds(driver, budget);
|
|
723
|
+
if (cycle.saved.size > 0 && driver.nothingLeft()) {
|
|
724
|
+
return transactionDurability(cycle.saved);
|
|
519
725
|
}
|
|
520
|
-
|
|
726
|
+
}
|
|
727
|
+
for (;;) {
|
|
728
|
+
budget.assertOpen();
|
|
521
729
|
try {
|
|
522
|
-
const committed = await
|
|
730
|
+
const committed = await driver.attempt(budget.priority());
|
|
523
731
|
// Merged, not overwritten, should one participant be both saved and committed (see the
|
|
524
732
|
// NOTE in reportSaved).
|
|
525
733
|
const written = new Map(cycle.saved);
|
|
@@ -528,58 +736,12 @@ export class TransactionCoordinator {
|
|
|
528
736
|
}
|
|
529
737
|
return writtenDurability(written);
|
|
530
738
|
} catch (err) {
|
|
531
|
-
// Only a CLEAN stale loss is retryable. A partial landing, an expired transaction, an
|
|
532
|
-
// unavailable transactor, etc. all propagate unchanged.
|
|
533
739
|
if (!(err instanceof CoordinatorStaleLossError)) {
|
|
534
740
|
throw err;
|
|
535
741
|
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
if (
|
|
539
|
-
throw err;
|
|
540
|
-
}
|
|
541
|
-
// Re-read fresh state before re-attempting so the next commit pends against current
|
|
542
|
-
// revisions (mirrors how Collection.sync calls updateInternal() before retrying).
|
|
543
|
-
//
|
|
544
|
-
// A participant whose log tail landed despite the reported loss finds its own entry
|
|
545
|
-
// here and FINISHES that action before consuming it (Collection.completeOwnEntry,
|
|
546
|
-
// from the attempt commitOnceLatched retained). When finishing is refused for a cause
|
|
547
|
-
// that can clear, the refresh — not the commit — is what gets retried: the
|
|
548
|
-
// participant's log entry is already durable, and once a refresh adopts a newer
|
|
549
|
-
// revision commitOnce appends a SECOND entry under this transaction's id. (At an
|
|
550
|
-
// unchanged revision the re-append is byte-identical — see
|
|
551
|
-
// Collection.logAppendBlockIds — so what is at stake is the duplicate entry, not two
|
|
552
|
-
// versions of one tail block.)
|
|
553
|
-
// Each such round counts against the same budget as a stale loss; a permanent
|
|
554
|
-
// refusal (a rival holds the revision) escapes as the named TornActionError at once.
|
|
555
|
-
// Re-refreshing collections an earlier round already refreshed is a no-op.
|
|
556
|
-
// The round that would spend the last of the budget is told so, which makes each
|
|
557
|
-
// participant SETTLE a half-landed write rather than ask for another round: the
|
|
558
|
-
// error thrown below then says whether the write can still land (`final`). A
|
|
559
|
-
// deadline cannot be foreseen the same way; a write given up on it stays unsettled.
|
|
560
|
-
for (;;) {
|
|
561
|
-
const delay = jitteredBackoffMs(staleLosses - 1, { baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
|
|
562
|
-
await abortableDelay(delay, signal);
|
|
563
|
-
const refused = await this.refreshBetweenAttempts(cycle, staleLosses + 1 >= maxAttempts);
|
|
564
|
-
if (refused === undefined) {
|
|
565
|
-
break;
|
|
566
|
-
}
|
|
567
|
-
staleLosses++;
|
|
568
|
-
if (staleLosses >= maxAttempts
|
|
569
|
-
|| (deadlineMs !== undefined && Date.now() - startedAt >= deadlineMs)) {
|
|
570
|
-
throw refused;
|
|
571
|
-
}
|
|
572
|
-
}
|
|
573
|
-
if (cycle.saved.size > 0 && this.stagedCollections().length === 0) {
|
|
574
|
-
// The refresh saved what was left: nothing is staged any more, so this transaction
|
|
575
|
-
// is committed. Return now rather than go round again, where an abort, the deadline
|
|
576
|
-
// or an expiry check could still report failure for a transaction that is entirely
|
|
577
|
-
// saved. The refresh's consume and replay already gave each saved participant the
|
|
578
|
-
// success path's local treatment, so only the stamp is left to release.
|
|
579
|
-
// Tested on what is still STAGED, not on "every participant saved": a participant
|
|
580
|
-
// can be saved and still hold actions staged after the attempt began, and those
|
|
581
|
-
// the next attempt must commit.
|
|
582
|
-
this.stampData.delete(transaction.stamp.id);
|
|
742
|
+
budget.recordLoss(err);
|
|
743
|
+
await this.refreshRounds(driver, budget);
|
|
744
|
+
if (cycle.saved.size > 0 && driver.nothingLeft()) {
|
|
583
745
|
return transactionDurability(cycle.saved);
|
|
584
746
|
}
|
|
585
747
|
}
|
|
@@ -587,37 +749,82 @@ export class TransactionCoordinator {
|
|
|
587
749
|
}
|
|
588
750
|
|
|
589
751
|
/**
|
|
590
|
-
*
|
|
591
|
-
*
|
|
592
|
-
*
|
|
752
|
+
* Back off, refresh, and keep refreshing while a round reports that a participant's half-landed
|
|
753
|
+
* write could not be finished YET. Re-reading fresh state before re-attempting is what lets the
|
|
754
|
+
* next attempt pend against current revisions (mirrors how Collection.sync calls updateInternal()
|
|
755
|
+
* before retrying), and a participant whose log tail landed despite the reported loss finds its
|
|
756
|
+
* own entry here and FINISHES that action before consuming it (Collection.completeOwnEntry, from
|
|
757
|
+
* the attempt the previous one retained).
|
|
758
|
+
*
|
|
759
|
+
* When finishing is refused for a cause that can clear, the refresh — not the commit — is what
|
|
760
|
+
* gets retried: the participant's log entry is already durable, and once a refresh adopts a newer
|
|
761
|
+
* revision the next attempt appends a SECOND entry under the write's id. (At an unchanged
|
|
762
|
+
* revision the re-append is byte-identical — see Collection.logAppendBlockIds — so what is at
|
|
763
|
+
* stake is the duplicate entry, not two versions of one tail block.) Each such round counts
|
|
764
|
+
* against the same budget as a stale loss; a permanent refusal (a rival holds the revision)
|
|
765
|
+
* escapes as the named TornActionError at once. Re-refreshing a collection an earlier round
|
|
766
|
+
* already refreshed is a no-op. The round that would spend the last of the budget is told so,
|
|
767
|
+
* which makes each participant SETTLE a half-landed write rather than ask for another round: the
|
|
768
|
+
* error thrown then says whether the write can still land (`final`). A deadline cannot be
|
|
769
|
+
* foreseen the same way; a write given up on it stays unsettled.
|
|
770
|
+
*/
|
|
771
|
+
private async refreshRounds(driver: AttemptDriver, budget: RetryBudget): Promise<void> {
|
|
772
|
+
for (;;) {
|
|
773
|
+
await budget.pause();
|
|
774
|
+
const refused = await driver.refresh(budget.lastChance());
|
|
775
|
+
if (refused === undefined) {
|
|
776
|
+
return;
|
|
777
|
+
}
|
|
778
|
+
budget.recordRefusedRound(refused);
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
/** Every registered collection as a refresh-round participant of a transaction's commit.
|
|
783
|
+
*
|
|
784
|
+
* NOTE: EVERY registered collection, not only the participants of the transaction. Not free: a
|
|
785
|
+
* non-participant's refresh throws CollectionHeaderVanishedError if its header momentarily reads
|
|
786
|
+
* absent while it holds a committed revision, aborting this retry. And because a throw does not
|
|
787
|
+
* stop the round, an unreachable cluster costs one failed read per registered collection rather
|
|
788
|
+
* than one. The registered set is small today; if that (or retry latency) ever bites, narrow this
|
|
789
|
+
* to the transaction's participants. (The per-entry commit already refreshes its participants
|
|
790
|
+
* only, through their handles — see `EntryParticipants.refreshers`.) */
|
|
791
|
+
private registeredRefreshers(): RefreshParticipant[] {
|
|
792
|
+
return [...this.collections].map(([collectionId, collection]) => ({
|
|
793
|
+
collectionId,
|
|
794
|
+
refresh: (report, lastChance) => collection.refreshInFlight(report, lastChance),
|
|
795
|
+
}));
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/**
|
|
799
|
+
* One refresh round between attempts: refresh every participant, recording in `cycle.saved` each
|
|
800
|
+
* one whose refresh finished the write's own log entry (see {@link Collection.refreshInFlight}),
|
|
801
|
+
* with who holds it — `undefined` when the refresh could not say, which still records it as
|
|
802
|
+
* saved — and telling the participant so (`RefreshParticipant.saved`).
|
|
593
803
|
*
|
|
594
|
-
* Visits EVERY
|
|
595
|
-
* participant later in the
|
|
804
|
+
* Visits EVERY participant even after one throws. Stopping at the first throw would leave a
|
|
805
|
+
* participant later in the round unfinished whenever an earlier one is torn — its log tail stored
|
|
596
806
|
* and its other blocks never landed, the silent loss `Collection.completeOwnEntry` exists to
|
|
597
807
|
* prevent — and unreported. Each refresh is independent (its own latch, its own instance state),
|
|
598
808
|
* so one failing says nothing about the next.
|
|
599
809
|
*
|
|
600
|
-
* NOTE:
|
|
601
|
-
*
|
|
602
|
-
*
|
|
603
|
-
* a throw no longer stops the round, an unreachable cluster costs one failed read per registered
|
|
604
|
-
* collection rather than one. The registered set is small today; if that (or retry latency) ever
|
|
605
|
-
* bites, narrow this to the transaction's participants.
|
|
810
|
+
* NOTE: no `tailFloor` is passed to any refresh — the coordinator's loop loses the confirmed
|
|
811
|
+
* revision before the round runs (backlog `debt-multi-collection-retry-cannot-see-the-taken-revision`);
|
|
812
|
+
* the per-entry commit has the same gap, through the same round.
|
|
606
813
|
*
|
|
607
814
|
* @param lastChance - this is the round that spends the last of the retry budget, so a
|
|
608
815
|
* participant whose half-landed write is refused again settles it (`Collection.completeOwnEntry`)
|
|
609
816
|
* rather than asking for a round it will not get; the error returned then carries `final`.
|
|
610
817
|
* @returns the first `completion-refused` {@link TornActionError} when that is the only way any
|
|
611
818
|
* refresh failed — the round is worth retrying; `undefined` when every refresh succeeded.
|
|
612
|
-
* @throws the first other error, once every
|
|
819
|
+
* @throws the first other error, once every participant has been visited.
|
|
613
820
|
*/
|
|
614
|
-
private async refreshBetweenAttempts(cycle: CommitCycle, lastChance: boolean): Promise<TornActionError | undefined> {
|
|
821
|
+
private async refreshBetweenAttempts(participants: readonly RefreshParticipant[], cycle: CommitCycle, lastChance: boolean): Promise<TornActionError | undefined> {
|
|
615
822
|
let terminal: { error: unknown } | undefined;
|
|
616
823
|
let refused: TornActionError | undefined;
|
|
617
|
-
for (const
|
|
824
|
+
for (const participant of participants) {
|
|
618
825
|
const report: RefreshReport = {};
|
|
619
826
|
try {
|
|
620
|
-
await
|
|
827
|
+
await participant.refresh(report, lastChance);
|
|
621
828
|
} catch (err) {
|
|
622
829
|
if (err instanceof TornActionError && err.reason === 'completion-refused') {
|
|
623
830
|
refused ??= err;
|
|
@@ -628,7 +835,8 @@ export class TransactionCoordinator {
|
|
|
628
835
|
// Read after the try, not only on success: a refresh can throw AFTER it finished the
|
|
629
836
|
// entry, and that participant is saved either way.
|
|
630
837
|
if (report.ownEntryFinished !== undefined) {
|
|
631
|
-
recordCollectionDurability(cycle.saved, collectionId, report.ownEntryFinished.durability);
|
|
838
|
+
recordCollectionDurability(cycle.saved, participant.collectionId, report.ownEntryFinished.durability);
|
|
839
|
+
participant.saved?.();
|
|
632
840
|
}
|
|
633
841
|
}
|
|
634
842
|
if (terminal !== undefined) {
|
|
@@ -673,10 +881,11 @@ export class TransactionCoordinator {
|
|
|
673
881
|
*
|
|
674
882
|
* Local state already matches that report: a saved participant was consumed by its refresh, and
|
|
675
883
|
* an unsaved one kept its staged actions (a failed finish throws before touching anything, and
|
|
676
|
-
* commitOnceLatched restores every participant it did not commit
|
|
677
|
-
*
|
|
884
|
+
* commitOnceLatched restores every participant it did not commit; the per-entry commit leaves a
|
|
885
|
+
* refused participant's segment sealed and first). Releasing a transaction's stamp on the partial
|
|
886
|
+
* report is {@link commit}'s, which reads the class of what this returns.
|
|
678
887
|
*/
|
|
679
|
-
private reportSaved(
|
|
888
|
+
private reportSaved(actionId: ActionId, cycle: CommitCycle, err: unknown): unknown {
|
|
680
889
|
if (cycle.saved.size === 0) {
|
|
681
890
|
return err;
|
|
682
891
|
}
|
|
@@ -693,9 +902,8 @@ export class TransactionCoordinator {
|
|
|
693
902
|
// committed. Unreachable while a session stages and commits on one call path; if staging ever
|
|
694
903
|
// races a commit, report such a participant in both lists.
|
|
695
904
|
const failed = [...cycle.participants].filter(id => !committed.has(id));
|
|
696
|
-
log('commit:partial-after-refresh tx=%s committed=%o failed=%o reason=%s',
|
|
905
|
+
log('commit:partial-after-refresh tx=%s committed=%o failed=%o reason=%s', actionId,
|
|
697
906
|
[...committed.keys()], failed, reason instanceof Error ? reason.name : String(reason));
|
|
698
|
-
this.stampData.delete(transaction.stamp.id);
|
|
699
907
|
return new CoordinatorPartialCommitError(committed, failed, reason);
|
|
700
908
|
}
|
|
701
909
|
|
|
@@ -780,7 +988,7 @@ export class TransactionCoordinator {
|
|
|
780
988
|
// (Collection.completeOwnEntry). `transaction.id` is stable across retries, so re-marking on a
|
|
781
989
|
// later attempt re-states the same fact. Only participants are marked; a registered
|
|
782
990
|
// non-participant is left unmarked and its refresh behaves exactly as a reader's.
|
|
783
|
-
cycle.inFlightDisposers.push(collection.beginInFlightAction(transaction.id));
|
|
991
|
+
cycle.inFlightDisposers.push(collection.beginInFlightAction(transaction.id, cycle.inFlight));
|
|
784
992
|
}
|
|
785
993
|
return await this.commitOnceLatched(transaction, collectionData, cycle);
|
|
786
994
|
} finally {
|
|
@@ -850,12 +1058,14 @@ export class TransactionCoordinator {
|
|
|
850
1058
|
preCommitSnapshots.set(collectionId, collection.snapshotPending());
|
|
851
1059
|
}
|
|
852
1060
|
|
|
1061
|
+
// Read per attempt: commitAttempts ages the transaction's priority between attempts.
|
|
1062
|
+
const context = writeContextOf(transaction, cycle.inFlight);
|
|
853
1063
|
let coordResult: CoordinationResult;
|
|
854
1064
|
try {
|
|
855
1065
|
for (const { collectionId, collection } of collectionData) {
|
|
856
1066
|
const applyResult = await this.applyActionsToCollection(
|
|
857
1067
|
{ collectionId, actions: collection.getPendingActions() },
|
|
858
|
-
|
|
1068
|
+
context,
|
|
859
1069
|
allCollectionIds
|
|
860
1070
|
);
|
|
861
1071
|
if (!applyResult.success) {
|
|
@@ -871,7 +1081,7 @@ export class TransactionCoordinator {
|
|
|
871
1081
|
|
|
872
1082
|
// Execute consensus phases (GATHER, PEND, COMMIT)
|
|
873
1083
|
coordResult = await this.coordinateTransaction(
|
|
874
|
-
|
|
1084
|
+
context,
|
|
875
1085
|
validation,
|
|
876
1086
|
collectionTransforms,
|
|
877
1087
|
criticalBlocks,
|
|
@@ -884,7 +1094,7 @@ export class TransactionCoordinator {
|
|
|
884
1094
|
// to its pre-append snapshot — a genuinely clean rollback that leaves each
|
|
885
1095
|
// tracker pristine for retry (see txn-failed-commit-leaves-staged-log-entry).
|
|
886
1096
|
for (const { collectionId, collection } of collectionData) {
|
|
887
|
-
collection.restorePending(preCommitSnapshots.get(collectionId)!);
|
|
1097
|
+
await collection.restorePending(preCommitSnapshots.get(collectionId)!);
|
|
888
1098
|
}
|
|
889
1099
|
throw err;
|
|
890
1100
|
}
|
|
@@ -903,7 +1113,8 @@ export class TransactionCoordinator {
|
|
|
903
1113
|
// so a re-send declares none, and an undeclared block falls back to member-side corroboration.
|
|
904
1114
|
for (const { collectionId, collection } of collectionData) {
|
|
905
1115
|
if (committed.has(collectionId)) continue;
|
|
906
|
-
collection.retainInFlightAttempt(
|
|
1116
|
+
collection.retainInFlightAttempt({
|
|
1117
|
+
actionId: transaction.id,
|
|
907
1118
|
rev: pendedRevs.get(collectionId)!,
|
|
908
1119
|
transforms: copyTransforms(collectionTransforms.get(collectionId)!),
|
|
909
1120
|
tailId: criticalBlocks.get(collectionId)!,
|
|
@@ -914,26 +1125,32 @@ export class TransactionCoordinator {
|
|
|
914
1125
|
// while another failed permanently. A uniform pre-append restore would
|
|
915
1126
|
// corrupt the committed half — re-staging its already-durable actions as
|
|
916
1127
|
// still-pending, so tracker memory would disagree with cluster storage.
|
|
917
|
-
// Split the local handling instead:
|
|
1128
|
+
// Split the local handling instead, in two passes: the committed half is folded
|
|
1129
|
+
// first, in one await-free pass (so a committed read observes every committed
|
|
1130
|
+
// participant pre-fold or post-fold, as the success fold below promises), and the
|
|
1131
|
+
// failed half is restored after it — a restore can replay, so it is awaited, which
|
|
1132
|
+
// admits no interleaved stage because every participant's latch is held across
|
|
1133
|
+
// this span.
|
|
918
1134
|
for (const { collectionId, collection } of collectionData) {
|
|
919
|
-
if (committed.has(collectionId))
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
1135
|
+
if (!committed.has(collectionId)) continue;
|
|
1136
|
+
// Committed → the success-path local treatment (see below): fold the
|
|
1137
|
+
// committed transforms into the read cache BEFORE resetting the tracker,
|
|
1138
|
+
// then drop the now-durable pending actions so a retry cannot re-log them.
|
|
1139
|
+
// NOTE: no-double-apply on retry depends on clearPendingActions() running for
|
|
1140
|
+
// EVERY committed collection here before any re-drive of commit(). If a committed
|
|
1141
|
+
// collection kept its pending queue, a subsequent commit() would re-append and
|
|
1142
|
+
// re-log its already-durable actions — a duplicate log entry on the winner. The
|
|
1143
|
+
// no-double-apply-on-retry test in transaction.spec.ts locks this.
|
|
1144
|
+
const rev = collection.recordCommitted(transaction.id, pendedRevs.get(collectionId)!);
|
|
1145
|
+
collection.applyCommittedToCache(collectionTransforms.get(collectionId)!, rev);
|
|
1146
|
+
collection.tracker.reset();
|
|
1147
|
+
collection.clearPendingActions();
|
|
1148
|
+
}
|
|
1149
|
+
for (const { collectionId, collection } of collectionData) {
|
|
1150
|
+
if (committed.has(collectionId)) continue;
|
|
1151
|
+
// Failed / never-committed → restore the pre-append snapshot so a retry
|
|
1152
|
+
// re-appends cleanly (no duplicate log entry).
|
|
1153
|
+
await collection.restorePending(preCommitSnapshots.get(collectionId)!);
|
|
937
1154
|
}
|
|
938
1155
|
const failed = [...(coordResult.failedCollections ?? new Set<CollectionId>())];
|
|
939
1156
|
if (coordResult.staleLoss) {
|
|
@@ -974,25 +1191,14 @@ export class TransactionCoordinator {
|
|
|
974
1191
|
// EMPTY committed set: PEND failed, or the whole commit failed cleanly with
|
|
975
1192
|
// nothing durable. Restore every tracker so each is pristine for retry.
|
|
976
1193
|
for (const { collectionId, collection } of collectionData) {
|
|
977
|
-
collection.restorePending(preCommitSnapshots.get(collectionId)!);
|
|
1194
|
+
await collection.restorePending(preCommitSnapshots.get(collectionId)!);
|
|
978
1195
|
}
|
|
979
1196
|
// Distinguish a genuine optimistic-concurrency conflict (a stale loss / pending
|
|
980
1197
|
// contention — retryable after a re-read) from a hard failure (unavailable transactor,
|
|
981
1198
|
// storage rejection, expired). Only the former is worth re-driving; the retry wrapper in
|
|
982
1199
|
// commit() catches CoordinatorStaleLossError and re-attempts, while a plain Error escapes
|
|
983
1200
|
// immediately (preserving the historical fail-fast behaviour for hard failures).
|
|
984
|
-
|
|
985
|
-
throw new CoordinatorStaleLossError([...(coordResult.failedCollections ?? new Set(allCollectionIds))], coordResult.error);
|
|
986
|
-
}
|
|
987
|
-
// An expiry is surfaced by name and never re-driven: the retry's pend would carry an
|
|
988
|
-
// expiration from the same clock.
|
|
989
|
-
if (coordResult.expired !== undefined) {
|
|
990
|
-
throw coordResult.expired;
|
|
991
|
-
}
|
|
992
|
-
if (coordResult.tooLarge !== undefined) {
|
|
993
|
-
throw coordResult.tooLarge;
|
|
994
|
-
}
|
|
995
|
-
throw new Error(`Transaction commit failed: ${coordResult.error}`);
|
|
1201
|
+
throw cleanFailureError(coordResult, [...(coordResult.failedCollections ?? new Set(allCollectionIds))]);
|
|
996
1202
|
}
|
|
997
1203
|
|
|
998
1204
|
// Advance actionContext, fold the committed transforms into each
|
|
@@ -1018,6 +1224,156 @@ export class TransactionCoordinator {
|
|
|
1018
1224
|
return coordResult.committedCollections;
|
|
1019
1225
|
}
|
|
1020
1226
|
|
|
1227
|
+
/**
|
|
1228
|
+
* Commit one queued entry's slice of every collection it wrote, as ONE all-or-nothing round
|
|
1229
|
+
* (docs/offline-sync.md §One entry, one network transaction): for each collection in
|
|
1230
|
+
* `entry.collectionIds`, exactly its HEAD sealed segment, which must be `entry.entryId`, as one
|
|
1231
|
+
* log entry under `entry.actionId` and `entry.timestamp`. The offline-sync queue is the caller,
|
|
1232
|
+
* for an entry that wrote more than one collection; an entry that wrote one commits through
|
|
1233
|
+
* `Collection.commitHead`, whose steps this drives in lock-step across the participants
|
|
1234
|
+
* (`HeadCommitHandle` in collection/head-commit.ts): open a handle per participant, in sorted
|
|
1235
|
+
* collection-id order; the opening refresh, which finishes and consumes an entry an earlier
|
|
1236
|
+
* `commitEntry` under this action id left in a participant's log; then per attempt, build every
|
|
1237
|
+
* participant's attempt on its head tracker, record them through `entry.recordAttempt`, release
|
|
1238
|
+
* every latch, run the phases ({@link coordinateAttempts}: pend every participant, then commit),
|
|
1239
|
+
* retake every latch, and apply the outcome — fold then replay on a commit, the recorded refusal
|
|
1240
|
+
* re-checked and the attempt retained on a refusal — with the retry loop of {@link commit}
|
|
1241
|
+
* ({@link commitAttempts}) between attempts.
|
|
1242
|
+
*
|
|
1243
|
+
* What the slice deliberately does NOT do, where {@link commit} does: no pre-commit snapshots or
|
|
1244
|
+
* `restorePending` (an attempt never touches the live tracker; a failed one leaves it as it was
|
|
1245
|
+
* by construction); no `getPendingActions`, `clearPendingActions` or `tracker.reset` (the head
|
|
1246
|
+
* segment is dropped and folded through the handle); no stamp (the one-open-stamp rule is about
|
|
1247
|
+
* staged state, which a slice does not stage — a slice may run while a stamp is open on another
|
|
1248
|
+
* coordinator over the same collections, which is the plugin's open SQL transaction; the
|
|
1249
|
+
* collection latch is what serialises them); no expiry check (an entry has no stamp to expire).
|
|
1250
|
+
* Its refresh rounds refresh the entry's participants only, through their handles, never the
|
|
1251
|
+
* coordinator's whole map.
|
|
1252
|
+
*
|
|
1253
|
+
* Requires a coordinator built with `pendValidation: 'none'`, and throws before touching
|
|
1254
|
+
* anything otherwise: a queued entry has no statements for a member to re-execute, and the pend
|
|
1255
|
+
* carries bare transforms exactly as the plugin's `commitBatchLegacy` sends them. The queue builds
|
|
1256
|
+
* one coordinator per submission over exactly the entry's collections; running `commitEntry`
|
|
1257
|
+
* while a {@link commit} of the same coordinator is open is not a supported combination, and
|
|
1258
|
+
* nothing but the collection latches guards it.
|
|
1259
|
+
*
|
|
1260
|
+
* @returns who holds each collection the entry wrote, across every attempt, as {@link commit}
|
|
1261
|
+
* answers it — a participant the refresh saved included, listed with an `undefined` class (and
|
|
1262
|
+
* `overall` then absent) when the refresh established it was saved under `entry.actionId` without
|
|
1263
|
+
* re-sending (the status-read fallback of `Collection.completeOwnEntry`, where
|
|
1264
|
+
* `Collection.commitHead` answers `undefined`). Never `undefined`: a slice always writes at least
|
|
1265
|
+
* one collection, so the "nothing was written" answer `commit` has is unrepresentable here.
|
|
1266
|
+
* @throws what {@link commit} throws — a {@link CoordinatorStaleLossError} once the retry budget
|
|
1267
|
+
* is spent, a {@link CoordinatorPartialCommitError} for a partial landing (every participant it
|
|
1268
|
+
* names as committed has had its head consumed; every failed one's head is still sealed and
|
|
1269
|
+
* first, its attempt retained), the named refusals, `TornActionError` out of a refresh — plus the
|
|
1270
|
+
* recorded refusal of any participant's head (an `ActionRefusedError`), thrown with every other
|
|
1271
|
+
* participant's segment intact since nothing was sent or the send was refused, and
|
|
1272
|
+
* `SealedSegmentsError` from `Collection.openHeadCommit`'s preconditions, before anything is done.
|
|
1273
|
+
*/
|
|
1274
|
+
async commitEntry(entry: EntryCommit, options?: SyncOptions): Promise<TransactionDurability> {
|
|
1275
|
+
if (this.pendValidation !== 'none') {
|
|
1276
|
+
throw new Error(`commitEntry ${entry.entryId}: a queued entry commits through a coordinator built with `
|
|
1277
|
+
+ `pendValidation: 'none' — it has no statements for a member to re-execute`);
|
|
1278
|
+
}
|
|
1279
|
+
const inFlight = inFlightCycleOf(options);
|
|
1280
|
+
const participants = await EntryParticipants.open(this.collections, entry, inFlight);
|
|
1281
|
+
const cycle: CommitCycle = {
|
|
1282
|
+
// The handles own the in-flight marks (closed with them), so none to dispose here.
|
|
1283
|
+
inFlightDisposers: [],
|
|
1284
|
+
participants: new Set(participants.ids()),
|
|
1285
|
+
saved: new Map(),
|
|
1286
|
+
inFlight,
|
|
1287
|
+
};
|
|
1288
|
+
const driver: AttemptDriver = {
|
|
1289
|
+
opensWithRefresh: true,
|
|
1290
|
+
attempt: priority => this.commitEntryOnce(entry, participants, cycle, priority),
|
|
1291
|
+
refresh: lastChance => this.refreshBetweenAttempts(participants.refreshers(), cycle, lastChance),
|
|
1292
|
+
nothingLeft: () => participants.size === 0,
|
|
1293
|
+
};
|
|
1294
|
+
try {
|
|
1295
|
+
const written = await this.commitAttempts(driver, cycle, options);
|
|
1296
|
+
if (written === undefined) {
|
|
1297
|
+
throw new Error(`commitEntry ${entry.entryId}: the cycle ended with nothing committed and nothing saved`);
|
|
1298
|
+
}
|
|
1299
|
+
return written;
|
|
1300
|
+
} catch (err) {
|
|
1301
|
+
throw this.reportSaved(entry.actionId, cycle, err);
|
|
1302
|
+
} finally {
|
|
1303
|
+
participants.close();
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
/**
|
|
1308
|
+
* One attempt of {@link commitEntry}, under the latches `participants` hold: every remaining
|
|
1309
|
+
* participant's attempt is built (`HeadCommitHandle.stage`, which re-checks the head's recorded
|
|
1310
|
+
* refusal and throws it), recorded through the write-ahead hook — a throw there aborts with
|
|
1311
|
+
* nothing sent and nothing changed — then sent as one round with every latch released, and the
|
|
1312
|
+
* outcome applied under the retaken latches:
|
|
1313
|
+
*
|
|
1314
|
+
* - a commit: fold every participant in one await-free pass, then replay each
|
|
1315
|
+
* (`EntryParticipants.landed`); the attempt's durability per collection is the answer;
|
|
1316
|
+
* - a partial landing: the committed participants folded, replayed and recorded in `cycle.saved`
|
|
1317
|
+
* — before their local treatment, so a throw in it is still reported over the right committed
|
|
1318
|
+
* set — and the rest refused (their recorded refusal re-checked, their attempt retained); then
|
|
1319
|
+
* {@link CoordinatorStaleLossError} when every refusal was a returned one, so the retry loop
|
|
1320
|
+
* finishes the refused participants forward over what is left (as `commitOnceLatched`'s
|
|
1321
|
+
* partial-retry branch does), or {@link CoordinatorPartialCommitError} on a hard failure;
|
|
1322
|
+
* - nothing committed: every participant refused, then the failure thrown by its class
|
|
1323
|
+
* ({@link cleanFailureError}).
|
|
1324
|
+
*
|
|
1325
|
+
* The refusal re-check runs after the committed participants are folded, never before: a
|
|
1326
|
+
* sibling's recorded refusal must not keep a durable commit from being applied locally.
|
|
1327
|
+
*/
|
|
1328
|
+
private async commitEntryOnce(
|
|
1329
|
+
entry: EntryCommit,
|
|
1330
|
+
participants: EntryParticipants,
|
|
1331
|
+
cycle: CommitCycle,
|
|
1332
|
+
priority: number
|
|
1333
|
+
): Promise<ReadonlyMap<CollectionId, WriteDurability>> {
|
|
1334
|
+
const staged = await participants.stage();
|
|
1335
|
+
if (entry.recordAttempt !== undefined) {
|
|
1336
|
+
await entry.recordAttempt(participants.retainedAttempts(staged));
|
|
1337
|
+
}
|
|
1338
|
+
const context: WriteContext = {
|
|
1339
|
+
actionId: entry.actionId,
|
|
1340
|
+
timestamp: entry.timestamp,
|
|
1341
|
+
reads: [],
|
|
1342
|
+
priority,
|
|
1343
|
+
...refuseSoloField(cycle.inFlight.refuseSolo),
|
|
1344
|
+
};
|
|
1345
|
+
const attempts = new Map(Array.from(staged, ([collectionId, attempt]) => [collectionId, collectionAttemptOf(attempt)]));
|
|
1346
|
+
|
|
1347
|
+
participants.release();
|
|
1348
|
+
let coordResult: CoordinationResult;
|
|
1349
|
+
try {
|
|
1350
|
+
coordResult = await this.coordinateAttempts(context, undefined, attempts);
|
|
1351
|
+
} finally {
|
|
1352
|
+
await participants.retake();
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
if (coordResult.success) {
|
|
1356
|
+
await participants.landed(staged, participants.ids());
|
|
1357
|
+
return coordResult.committedCollections;
|
|
1358
|
+
}
|
|
1359
|
+
const committed = coordResult.committedCollections ?? new Map<CollectionId, WriteDurability>();
|
|
1360
|
+
const failed = participants.ids().filter(collectionId => !committed.has(collectionId));
|
|
1361
|
+
if (committed.size === 0) {
|
|
1362
|
+
participants.refused(staged, failed);
|
|
1363
|
+
throw cleanFailureError(coordResult, failed);
|
|
1364
|
+
}
|
|
1365
|
+
for (const [collectionId, durability] of committed) {
|
|
1366
|
+
recordCollectionDurability(cycle.saved, collectionId, durability);
|
|
1367
|
+
}
|
|
1368
|
+
await participants.landed(staged, committed.keys());
|
|
1369
|
+
participants.refused(staged, failed);
|
|
1370
|
+
if (coordResult.staleLoss) {
|
|
1371
|
+
log('commitEntry:partial-retry entry=%s action=%s committed=%o failed=%o reason=%s', entry.entryId, entry.actionId, [...committed.keys()], failed, coordResult.error);
|
|
1372
|
+
throw new CoordinatorStaleLossError(failed, coordResult.error);
|
|
1373
|
+
}
|
|
1374
|
+
throw new CoordinatorPartialCommitError(committed, failed, coordResult.expired ?? coordResult.error);
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1021
1377
|
/**
|
|
1022
1378
|
* Rollback a transaction (undo only the given stampId's applied actions).
|
|
1023
1379
|
*
|
|
@@ -1065,9 +1421,14 @@ export class TransactionCoordinator {
|
|
|
1065
1421
|
// Accepted: the tracker half already discards their transforms, and symmetric is the safer
|
|
1066
1422
|
// state — leaving such an action queued while its transforms are gone is exactly the
|
|
1067
1423
|
// phantom that a conflicting sync's replayActions would resurrect as live data.
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1424
|
+
// Awaited: a restore across a change to the collection's sealed segments replays them
|
|
1425
|
+
// (Collection.restorePending), and the single-call-path assumption above is what keeps a
|
|
1426
|
+
// stage from landing on the collection while that replay runs. Every participant is
|
|
1427
|
+
// restored even after one throws (a refusal of the restored open segment, or a failed
|
|
1428
|
+
// re-read inside the replay): each restore is independent, and stopping at the first throw
|
|
1429
|
+
// would leave the later participants holding the rolled-back stamp's actions, which the
|
|
1430
|
+
// next transaction would then log as its own.
|
|
1431
|
+
await visitEveryThenThrow(data.preSnapshot, ([collection, snapshot]) => collection.restorePending(snapshot));
|
|
1071
1432
|
}
|
|
1072
1433
|
|
|
1073
1434
|
/**
|
|
@@ -1287,10 +1648,11 @@ export class TransactionCoordinator {
|
|
|
1287
1648
|
// Released in the finally: execute has early failure returns.
|
|
1288
1649
|
const latchReleases: (() => void)[] = [];
|
|
1289
1650
|
try {
|
|
1290
|
-
// NOTE: deadlock-freedom against a concurrent commitOnce
|
|
1291
|
-
// the SAME order, not merely each in a sorted one. `CollectionId` is a
|
|
1292
|
-
// default `.sort()
|
|
1293
|
-
//
|
|
1651
|
+
// NOTE: deadlock-freedom against a concurrent commitOnce or commitEntry needs ALL three
|
|
1652
|
+
// paths to acquire in the SAME order, not merely each in a sorted one. `CollectionId` is a
|
|
1653
|
+
// string, so this default `.sort()`, commitOnce's explicit `<`/`>` comparator and
|
|
1654
|
+
// `compareCollectionIds` in entry-commit.ts agree today. If any spelling changes — a
|
|
1655
|
+
// custom comparator, a non-string id — change all three together.
|
|
1294
1656
|
for (const collectionId of [...allCollectionIds].sort()) {
|
|
1295
1657
|
const collection = this.collections.get(collectionId);
|
|
1296
1658
|
if (collection) {
|
|
@@ -1304,10 +1666,11 @@ export class TransactionCoordinator {
|
|
|
1304
1666
|
await this.collections.get(collectionId)?.restageIfBasesMoved();
|
|
1305
1667
|
}
|
|
1306
1668
|
|
|
1669
|
+
const context = writeContextOf(transaction);
|
|
1307
1670
|
for (const [collectionId, actions] of batches) {
|
|
1308
1671
|
const applyResult = await this.applyActionsToCollection(
|
|
1309
1672
|
{ collectionId, actions },
|
|
1310
|
-
|
|
1673
|
+
context,
|
|
1311
1674
|
allCollectionIds
|
|
1312
1675
|
);
|
|
1313
1676
|
|
|
@@ -1329,7 +1692,7 @@ export class TransactionCoordinator {
|
|
|
1329
1692
|
// 4. Coordinate (GATHER if multi-collection)
|
|
1330
1693
|
const tCoord = Date.now();
|
|
1331
1694
|
const coordResult = await this.coordinateTransaction(
|
|
1332
|
-
|
|
1695
|
+
context,
|
|
1333
1696
|
validation,
|
|
1334
1697
|
collectionTransforms,
|
|
1335
1698
|
criticalBlocks,
|
|
@@ -1346,8 +1709,12 @@ export class TransactionCoordinator {
|
|
|
1346
1709
|
// pending queues aren't left mis-tracking already-durable state; the failed subset
|
|
1347
1710
|
// is unwound to its pre-staging snapshot, and the stamp tracking is dropped so no
|
|
1348
1711
|
// unsafe all-or-nothing undo handle survives (rollback() would rewind the winner).
|
|
1349
|
-
//
|
|
1350
|
-
//
|
|
1712
|
+
// Two passes, as in commitOnceLatched: the committed subset's fold is kept
|
|
1713
|
+
// await-free for the same reason the success fold is, and the failed subset's
|
|
1714
|
+
// restore comes after it. That restore is awaited — it can replay the collection's
|
|
1715
|
+
// sealed segments — and the await admits no interleaved stage, because every
|
|
1716
|
+
// participant's latch is held across this span (restorePending stays latch-free by
|
|
1717
|
+
// contract and relies on exactly that).
|
|
1351
1718
|
// NOTE: execute() has no retry loop at all, so a returned refusal after a sibling
|
|
1352
1719
|
// committed is reported here at once, where commit() would re-drive the refused
|
|
1353
1720
|
// collection forward (the partial-retry branch of commitOnceLatched). No production
|
|
@@ -1360,23 +1727,24 @@ export class TransactionCoordinator {
|
|
|
1360
1727
|
if (committed.size > 0) {
|
|
1361
1728
|
for (const collectionId of batches.keys()) {
|
|
1362
1729
|
const collection = this.collections.get(collectionId);
|
|
1363
|
-
if (!collection) continue;
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1730
|
+
if (!collection || !committed.has(collectionId)) continue;
|
|
1731
|
+
const rev = collection.recordCommitted(transaction.id, pendedRevs.get(collectionId)!);
|
|
1732
|
+
collection.applyCommittedToCache(collectionTransforms.get(collectionId)!, rev);
|
|
1733
|
+
collection.tracker.reset();
|
|
1734
|
+
collection.clearPendingActions();
|
|
1735
|
+
}
|
|
1736
|
+
for (const collectionId of batches.keys()) {
|
|
1737
|
+
const collection = this.collections.get(collectionId);
|
|
1738
|
+
if (!collection || committed.has(collectionId)) continue;
|
|
1739
|
+
// NOTE: blind overwrite of this collection's staged state — anything
|
|
1740
|
+
// staged on it between the pre-stage snapshot above and here is discarded.
|
|
1741
|
+
// No sibling stamp can be that writer (at most one stamp is open), so the
|
|
1742
|
+
// only source is the untracked Tree.stage / Collection.act path, and this
|
|
1743
|
+
// is the same single-call-path assumption rollback() documents. It cannot
|
|
1744
|
+
// be softened by re-applying those actions here: collection.act takes the
|
|
1745
|
+
// same non-reentrant instance latch this span already holds.
|
|
1746
|
+
const snapshot = preStageSnapshots.get(collectionId);
|
|
1747
|
+
if (snapshot) await collection.restorePending(snapshot);
|
|
1380
1748
|
}
|
|
1381
1749
|
// The undo handle is now unsafe: rollback() is all-or-nothing and would rewind
|
|
1382
1750
|
// the collections that DID durably commit. Drop it, matching commitOnceLatched.
|
|
@@ -1441,11 +1809,12 @@ export class TransactionCoordinator {
|
|
|
1441
1809
|
/**
|
|
1442
1810
|
* Apply actions to a collection.
|
|
1443
1811
|
*
|
|
1444
|
-
* This runs the action handlers, writes to the log, and collects transforms.
|
|
1812
|
+
* This runs the action handlers, writes to the log, and collects transforms. The log entry is
|
|
1813
|
+
* written under `context`: its action id, its timestamp and its reads.
|
|
1445
1814
|
*/
|
|
1446
1815
|
private async applyActionsToCollection(
|
|
1447
1816
|
collectionActions: CollectionActions,
|
|
1448
|
-
|
|
1817
|
+
context: WriteContext,
|
|
1449
1818
|
allCollectionIds: CollectionId[]
|
|
1450
1819
|
): Promise<{
|
|
1451
1820
|
success: boolean;
|
|
@@ -1472,8 +1841,7 @@ export class TransactionCoordinator {
|
|
|
1472
1841
|
// Get transforms from the collection's tracker
|
|
1473
1842
|
const transforms = collection.tracker.transforms;
|
|
1474
1843
|
|
|
1475
|
-
|
|
1476
|
-
const actionId = transaction.id;
|
|
1844
|
+
const actionId = context.actionId;
|
|
1477
1845
|
const newRev = collection.getNextRev();
|
|
1478
1846
|
|
|
1479
1847
|
// Write actions to the collection's log to get the log tail block ID. The append must be a
|
|
@@ -1523,7 +1891,7 @@ export class TransactionCoordinator {
|
|
|
1523
1891
|
actionId,
|
|
1524
1892
|
newRev,
|
|
1525
1893
|
() => blockIdsForTransforms(transforms),
|
|
1526
|
-
{ collectionIds: allCollectionIds, reads:
|
|
1894
|
+
{ collectionIds: allCollectionIds, reads: [...context.reads], timestamp: context.timestamp }
|
|
1527
1895
|
);
|
|
1528
1896
|
|
|
1529
1897
|
// Return the transforms and log tail block ID
|
|
@@ -1537,9 +1905,11 @@ export class TransactionCoordinator {
|
|
|
1537
1905
|
}
|
|
1538
1906
|
|
|
1539
1907
|
/**
|
|
1540
|
-
* Coordinate a transaction across multiple collections
|
|
1908
|
+
* Coordinate a transaction's attempt across multiple collections: describe what each collection's
|
|
1909
|
+
* staged transforms send ({@link declareTouchedBlocks}), then run the phases
|
|
1910
|
+
* ({@link coordinateAttempts}).
|
|
1541
1911
|
*
|
|
1542
|
-
* @param
|
|
1912
|
+
* @param context - The write being coordinated (its action id and aged priority)
|
|
1543
1913
|
* @param validation - What every pend carries for members to re-check, or undefined for an
|
|
1544
1914
|
* unvalidated pend (see {@link pendValidationFor})
|
|
1545
1915
|
* @param collectionTransforms - Map of collectionId to its transforms
|
|
@@ -1549,38 +1919,45 @@ export class TransactionCoordinator {
|
|
|
1549
1919
|
* and the local record name the same number.
|
|
1550
1920
|
*/
|
|
1551
1921
|
private async coordinateTransaction(
|
|
1552
|
-
|
|
1922
|
+
context: WriteContext,
|
|
1553
1923
|
validation: PendRequest['validation'],
|
|
1554
1924
|
collectionTransforms: Map<CollectionId, Transforms>,
|
|
1555
1925
|
criticalBlocks: Map<CollectionId, BlockId>,
|
|
1556
1926
|
pendedRevs: ReadonlyMap<CollectionId, number>
|
|
1557
1927
|
): Promise<CoordinationResult> {
|
|
1558
|
-
const trxId = transaction.id;
|
|
1559
|
-
const t0 = Date.now();
|
|
1560
|
-
|
|
1561
1928
|
// Before anything is sent: one materialization of every block the transaction touches, measured
|
|
1562
1929
|
// against MAX_BLOCK_BYTES here and declared on the commit below.
|
|
1563
|
-
const described = await this.declareTouchedBlocks(collectionTransforms);
|
|
1930
|
+
const described = await this.declareTouchedBlocks(collectionTransforms, criticalBlocks, pendedRevs);
|
|
1564
1931
|
if (!described.success) {
|
|
1565
|
-
log('trx:phases trxId=%s refused before sending: %s',
|
|
1932
|
+
log('trx:phases trxId=%s refused before sending: %s', context.actionId, described.error);
|
|
1566
1933
|
return described;
|
|
1567
1934
|
}
|
|
1935
|
+
return this.coordinateAttempts(context, validation, described.attempts);
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1938
|
+
/**
|
|
1939
|
+
* The distributed phases of one attempt — GATHER, PEND, COMMIT, and the CANCEL of what a failed
|
|
1940
|
+
* commit left pending — over every collection's share of it, already described
|
|
1941
|
+
* (`CollectionAttempt`). Run by {@link coordinateTransaction} for a transaction's attempt and by
|
|
1942
|
+
* {@link commitEntryOnce} for a queued entry's, whose attempts the head commit described.
|
|
1943
|
+
*/
|
|
1944
|
+
private async coordinateAttempts(
|
|
1945
|
+
context: WriteContext,
|
|
1946
|
+
validation: PendRequest['validation'],
|
|
1947
|
+
attempts: ReadonlyMap<CollectionId, CollectionAttempt>
|
|
1948
|
+
): Promise<CoordinationResult> {
|
|
1949
|
+
const trxId = context.actionId;
|
|
1950
|
+
const t0 = Date.now();
|
|
1568
1951
|
|
|
1569
1952
|
// 1. GATHER phase: collect critical cluster nominees (skip if single collection)
|
|
1570
|
-
const criticalBlockIds = Array.from(
|
|
1953
|
+
const criticalBlockIds = Array.from(attempts.values(), attempt => attempt.tailId);
|
|
1571
1954
|
const tGather = Date.now();
|
|
1572
1955
|
const superclusterNominees = await this.gatherPhase(criticalBlockIds);
|
|
1573
1956
|
const gatherMs = Date.now() - tGather;
|
|
1574
1957
|
|
|
1575
1958
|
// 2. PEND phase: distribute to all block clusters
|
|
1576
1959
|
const tPend = Date.now();
|
|
1577
|
-
const pendResult = await this.pendPhase(
|
|
1578
|
-
transaction,
|
|
1579
|
-
validation,
|
|
1580
|
-
collectionTransforms,
|
|
1581
|
-
pendedRevs,
|
|
1582
|
-
superclusterNominees
|
|
1583
|
-
);
|
|
1960
|
+
const pendResult = await this.pendPhase(context, validation, attempts, superclusterNominees);
|
|
1584
1961
|
const pendMs = Date.now() - tPend;
|
|
1585
1962
|
if (!pendResult.success) {
|
|
1586
1963
|
log('trx:phases trxId=%s gather=%dms pend=%dms (failed) total=%dms', trxId, gatherMs, pendMs, Date.now() - t0);
|
|
@@ -1589,18 +1966,12 @@ export class TransactionCoordinator {
|
|
|
1589
1966
|
|
|
1590
1967
|
// 3. COMMIT phase: commit to all critical blocks (with retry for forward recovery)
|
|
1591
1968
|
const tCommit = Date.now();
|
|
1592
|
-
const commitResult = await this.commitPhase(
|
|
1593
|
-
transaction.id as ActionId,
|
|
1594
|
-
criticalBlockIds,
|
|
1595
|
-
pendResult.pendedBlockIds,
|
|
1596
|
-
pendedRevs,
|
|
1597
|
-
described.declared
|
|
1598
|
-
);
|
|
1969
|
+
const commitResult = await this.commitPhase(context, attempts, pendResult.pendedBlockIds);
|
|
1599
1970
|
const commitMs = Date.now() - tCommit;
|
|
1600
1971
|
if (!commitResult.success) {
|
|
1601
1972
|
// Targeted cancel: only cancel collections that are still pending (not already committed)
|
|
1602
1973
|
await this.cancelPhase(
|
|
1603
|
-
|
|
1974
|
+
context.actionId,
|
|
1604
1975
|
pendResult.pendedBlockIds,
|
|
1605
1976
|
commitResult.committedCollections
|
|
1606
1977
|
);
|
|
@@ -1669,31 +2040,45 @@ export class TransactionCoordinator {
|
|
|
1669
2040
|
}
|
|
1670
2041
|
|
|
1671
2042
|
/**
|
|
1672
|
-
*
|
|
2043
|
+
* What each collection's staged transforms send, described from its live tracker after the log
|
|
2044
|
+
* append: one local materialization of every block the transforms touch
|
|
1673
2045
|
* ({@link describeTouchedBlocks}), taken before anything is sent. A block the write would leave
|
|
1674
2046
|
* over `MAX_BLOCK_BYTES` refuses the transaction here, as a pend-phase failure carrying the
|
|
1675
2047
|
* {@link BlockTooLargeError} — the same shape as a pend too large to send, so `commit` throws it by
|
|
1676
2048
|
* name and `execute` reports it as a failure. Otherwise each collection's digests are kept as its
|
|
1677
|
-
* commit's declaration ({@link commitRequestFor}), so the commit does not materialize again
|
|
2049
|
+
* commit's declaration ({@link commitRequestFor}), so the commit does not materialize again, and
|
|
2050
|
+
* the base each update-only block's operations were computed against is read from the same
|
|
2051
|
+
* tracker's pins for its pend (`PendRequest.baseRevs`; storage keeps it with the pending record so
|
|
2052
|
+
* the record is never applied to a different version of the block).
|
|
1678
2053
|
*
|
|
1679
2054
|
* Purely local: each collection's tracker holds this transaction's staged transforms until the commit
|
|
1680
|
-
* succeeds, under the instance latch the commit span holds
|
|
1681
|
-
*
|
|
1682
|
-
* to report as it always has.
|
|
2055
|
+
* succeeds, under the instance latch the commit span holds. A collection missing from the registry
|
|
2056
|
+
* is a defect here: {@link applyActionsToCollection} has already refused it.
|
|
1683
2057
|
*/
|
|
1684
2058
|
private async declareTouchedBlocks(
|
|
1685
|
-
collectionTransforms: ReadonlyMap<CollectionId, Transforms
|
|
1686
|
-
|
|
1687
|
-
|
|
2059
|
+
collectionTransforms: ReadonlyMap<CollectionId, Transforms>,
|
|
2060
|
+
criticalBlocks: ReadonlyMap<CollectionId, BlockId>,
|
|
2061
|
+
pendedRevs: ReadonlyMap<CollectionId, number>
|
|
2062
|
+
): Promise<{ success: true; attempts: Map<CollectionId, CollectionAttempt> } | PendPhaseFailure> {
|
|
2063
|
+
const attempts = new Map<CollectionId, CollectionAttempt>();
|
|
1688
2064
|
for (const [collectionId, transforms] of collectionTransforms) {
|
|
1689
2065
|
const collection = this.collections.get(collectionId);
|
|
1690
|
-
if (!collection)
|
|
1691
|
-
|
|
2066
|
+
if (!collection) {
|
|
2067
|
+
throw new Error(`Collection not found: ${collectionId}`);
|
|
2068
|
+
}
|
|
2069
|
+
const blockIds = blockIdsForTransforms(transforms);
|
|
2070
|
+
const touched = await describeTouchedBlocks(collection.tracker, blockIds);
|
|
1692
2071
|
const tooLarge = oversizedBlock(touched.bytes);
|
|
1693
2072
|
if (tooLarge) return pendFailure([tooLarge]);
|
|
1694
|
-
|
|
2073
|
+
attempts.set(collectionId, {
|
|
2074
|
+
transforms,
|
|
2075
|
+
tailId: criticalBlocks.get(collectionId)!,
|
|
2076
|
+
rev: pendedRevs.get(collectionId)!,
|
|
2077
|
+
baseRevs: collection.tracker.stagedBaseRevs(blockIds),
|
|
2078
|
+
blockDigests: touched.digests,
|
|
2079
|
+
});
|
|
1695
2080
|
}
|
|
1696
|
-
return { success: true,
|
|
2081
|
+
return { success: true, attempts };
|
|
1697
2082
|
}
|
|
1698
2083
|
|
|
1699
2084
|
/**
|
|
@@ -1702,33 +2087,31 @@ export class TransactionCoordinator {
|
|
|
1702
2087
|
* Two or more collections go as ONE request when the transactor can send them so
|
|
1703
2088
|
* ({@link pendTogether}); otherwise, and for one collection, each collection is pended on its own.
|
|
1704
2089
|
*
|
|
1705
|
-
* @param
|
|
1706
|
-
*
|
|
2090
|
+
* @param context - The write being pended (its action id names the action; its aged priority
|
|
2091
|
+
* rides on an unvalidated pend)
|
|
1707
2092
|
* @param validation - What every pend carries for members to re-check, or undefined for an
|
|
1708
2093
|
* unvalidated pend (see {@link pendValidationFor})
|
|
1709
|
-
* @param
|
|
1710
|
-
*
|
|
1711
|
-
*
|
|
1712
|
-
*
|
|
2094
|
+
* @param attempts - Per collection, its share of the attempt: its transforms, and the revision
|
|
2095
|
+
* its log entry was stamped with — the pend request repeats it verbatim rather than recomputing
|
|
2096
|
+
* from the collection (a recompute after the append could name a different number if the
|
|
2097
|
+
* collection refreshed in between).
|
|
1713
2098
|
* @param superclusterNominees - Nominees for multi-collection consensus (null for single-collection)
|
|
1714
2099
|
*/
|
|
1715
2100
|
private async pendPhase(
|
|
1716
|
-
|
|
2101
|
+
context: WriteContext,
|
|
1717
2102
|
validation: PendRequest['validation'],
|
|
1718
|
-
|
|
1719
|
-
pendedRevs: ReadonlyMap<CollectionId, number>,
|
|
2103
|
+
attempts: ReadonlyMap<CollectionId, CollectionAttempt>,
|
|
1720
2104
|
superclusterNominees: ReadonlySet<PeerId> | null
|
|
1721
2105
|
): Promise<{ success: true; pendedBlockIds: Map<CollectionId, BlockId[]> } | PendPhaseFailure> {
|
|
1722
|
-
if (
|
|
2106
|
+
if (attempts.size === 0) {
|
|
1723
2107
|
return { success: false, error: 'No transforms to pend' };
|
|
1724
2108
|
}
|
|
1725
2109
|
|
|
1726
|
-
const actionId = transaction.id as ActionId;
|
|
1727
2110
|
const nominees = superclusterNominees ? Array.from(superclusterNominees) : undefined;
|
|
1728
2111
|
// Both methods, never the pend alone: the batched pend's mark promises that the commit goes as one
|
|
1729
2112
|
// request too (`PendRequest.soleReservation`), which only `commitMany` can keep.
|
|
1730
|
-
if (
|
|
1731
|
-
const together = await this.pendTogether(
|
|
2113
|
+
if (attempts.size > 1 && this.transactor.pendMany && this.transactor.commitMany) {
|
|
2114
|
+
const together = await this.pendTogether(context, validation, attempts, nominees);
|
|
1732
2115
|
if (together !== undefined) return together;
|
|
1733
2116
|
}
|
|
1734
2117
|
|
|
@@ -1736,7 +2119,7 @@ export class TransactionCoordinator {
|
|
|
1736
2119
|
// `Collection.sync` write does, so its pend may say so (`PendRequest.soleReservation`). Two or
|
|
1737
2120
|
// more pended one by one never may: each collection's commit can land on its own, and a late
|
|
1738
2121
|
// refusal of another collection's pend would surface after it.
|
|
1739
|
-
const sole =
|
|
2122
|
+
const sole = attempts.size === 1;
|
|
1740
2123
|
|
|
1741
2124
|
// Fan out the independent per-collection pends concurrently. Each settles to a
|
|
1742
2125
|
// { collectionId, blockIds } on success, or rejects with the per-collection reason.
|
|
@@ -1744,8 +2127,8 @@ export class TransactionCoordinator {
|
|
|
1744
2127
|
// Transactions touch few collections today; if one ever spans very many, bound this
|
|
1745
2128
|
// with a concurrency limiter so peak in-flight round-trips stays sane. Same for commitPhase.
|
|
1746
2129
|
const outcomes = await Promise.allSettled(
|
|
1747
|
-
Array.from(
|
|
1748
|
-
this.pendCollection(
|
|
2130
|
+
Array.from(attempts.entries()).map(([collectionId, attempt]) =>
|
|
2131
|
+
this.pendCollection(context, validation, collectionId, attempt, nominees, sole)
|
|
1749
2132
|
)
|
|
1750
2133
|
);
|
|
1751
2134
|
|
|
@@ -1766,7 +2149,7 @@ export class TransactionCoordinator {
|
|
|
1766
2149
|
// have pended in parallel, so cancel EVERY successfully-pended collection — not
|
|
1767
2150
|
// only those started before the failure. Cancels are best-effort (cancelPhase
|
|
1768
2151
|
// swallows their errors) so they cannot mask the original pend failure.
|
|
1769
|
-
await this.cancelPhase(actionId, pendedBlockIds);
|
|
2152
|
+
await this.cancelPhase(context.actionId, pendedBlockIds);
|
|
1770
2153
|
return pendFailure(reasons);
|
|
1771
2154
|
}
|
|
1772
2155
|
|
|
@@ -1791,36 +2174,28 @@ export class TransactionCoordinator {
|
|
|
1791
2174
|
* pends each collection on its own with requests rebuilt WITHOUT the mark. The marked requests are
|
|
1792
2175
|
* never re-sent one by one: each carries every block of its collection, so `NetworkTransactor.pend`
|
|
1793
2176
|
* would keep the mark, and one collection's late refusal could then surface after a sibling's commit
|
|
1794
|
-
* landed — the tear the mark exists to prevent.
|
|
1795
|
-
* the registry, so that the per-collection path reports it as it always has.
|
|
2177
|
+
* landed — the tear the mark exists to prevent.
|
|
1796
2178
|
*/
|
|
1797
2179
|
private async pendTogether(
|
|
1798
|
-
|
|
2180
|
+
context: WriteContext,
|
|
1799
2181
|
validation: PendRequest['validation'],
|
|
1800
|
-
|
|
1801
|
-
pendedRevs: ReadonlyMap<CollectionId, number>,
|
|
1802
|
-
actionId: ActionId,
|
|
2182
|
+
attempts: ReadonlyMap<CollectionId, CollectionAttempt>,
|
|
1803
2183
|
nominees: PeerId[] | undefined
|
|
1804
2184
|
): Promise<{ success: true; pendedBlockIds: Map<CollectionId, BlockId[]> } | PendPhaseFailure | undefined> {
|
|
1805
|
-
const requests
|
|
1806
|
-
|
|
1807
|
-
const collection = this.collections.get(collectionId);
|
|
1808
|
-
if (!collection) return undefined;
|
|
1809
|
-
requests.push(this.pendRequestFor(transaction, validation, collection, transforms, pendedRevs.get(collectionId)!, actionId, nominees, true));
|
|
1810
|
-
}
|
|
1811
|
-
const blockIds = new Map(Array.from(collectionTransforms, ([collectionId, transforms]) => [collectionId, blockIdsForTransforms(transforms)]));
|
|
2185
|
+
const requests = Array.from(attempts.values(), attempt => this.pendRequestFor(context, validation, attempt, nominees, true));
|
|
2186
|
+
const blockIds = new Map(Array.from(attempts, ([collectionId, attempt]) => [collectionId, blockIdsForTransforms(attempt.transforms)]));
|
|
1812
2187
|
|
|
1813
2188
|
let result: PendResult | undefined;
|
|
1814
2189
|
try {
|
|
1815
2190
|
result = await this.transactor.pendMany!(requests);
|
|
1816
2191
|
} catch (e) {
|
|
1817
|
-
await this.cancelPhase(actionId, blockIds);
|
|
2192
|
+
await this.cancelPhase(context.actionId, blockIds);
|
|
1818
2193
|
return pendFailure([e]);
|
|
1819
2194
|
}
|
|
1820
2195
|
if (result === undefined) return undefined;
|
|
1821
2196
|
return result.success
|
|
1822
2197
|
? { success: true, pendedBlockIds: blockIds }
|
|
1823
|
-
: pendFailure([pendRefusalError(Array.from(
|
|
2198
|
+
: pendFailure([pendRefusalError(Array.from(attempts.keys()), result)]);
|
|
1824
2199
|
}
|
|
1825
2200
|
|
|
1826
2201
|
/**
|
|
@@ -1829,26 +2204,19 @@ export class TransactionCoordinator {
|
|
|
1829
2204
|
* fan-out in {@link pendPhase} can settle it as a rejection.
|
|
1830
2205
|
*/
|
|
1831
2206
|
private async pendCollection(
|
|
1832
|
-
|
|
2207
|
+
context: WriteContext,
|
|
1833
2208
|
validation: PendRequest['validation'],
|
|
1834
2209
|
collectionId: CollectionId,
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
*
|
|
1838
|
-
|
|
1839
|
-
rev: number,
|
|
1840
|
-
actionId: ActionId,
|
|
2210
|
+
/** Its revision is the one the log entry was stamped with (threaded from the append), NOT
|
|
2211
|
+
* recomputed here: a `getNextRev()` after the append round trips could name a number a
|
|
2212
|
+
* concurrent refresh already moved past. */
|
|
2213
|
+
attempt: CollectionAttempt,
|
|
1841
2214
|
nominees: PeerId[] | undefined,
|
|
1842
2215
|
/** Whether this is the transaction's only collection (see {@link pendPhase}). */
|
|
1843
2216
|
sole: boolean
|
|
1844
2217
|
): Promise<{ collectionId: CollectionId; blockIds: BlockId[] }> {
|
|
1845
|
-
const collection = this.collections.get(collectionId);
|
|
1846
|
-
if (!collection) {
|
|
1847
|
-
throw new Error(`Collection not found: ${collectionId}`);
|
|
1848
|
-
}
|
|
1849
|
-
|
|
1850
2218
|
const pendResult = await this.transactor.pend(
|
|
1851
|
-
this.pendRequestFor(
|
|
2219
|
+
this.pendRequestFor(context, validation, attempt, nominees, sole)
|
|
1852
2220
|
);
|
|
1853
2221
|
if (!pendResult.success) {
|
|
1854
2222
|
throw pendRefusalError([collectionId], pendResult);
|
|
@@ -1858,17 +2226,13 @@ export class TransactionCoordinator {
|
|
|
1858
2226
|
}
|
|
1859
2227
|
|
|
1860
2228
|
/**
|
|
1861
|
-
* The pend request for one collection's
|
|
1862
|
-
* goes on its own ({@link pendCollection}) or with its siblings ({@link pendTogether}).
|
|
2229
|
+
* The pend request for one collection's share of an attempt — the one place a pend request is built,
|
|
2230
|
+
* whether it goes on its own ({@link pendCollection}) or with its siblings ({@link pendTogether}).
|
|
1863
2231
|
*/
|
|
1864
2232
|
private pendRequestFor(
|
|
1865
|
-
|
|
2233
|
+
context: WriteContext,
|
|
1866
2234
|
validation: PendRequest['validation'],
|
|
1867
|
-
|
|
1868
|
-
transforms: Transforms,
|
|
1869
|
-
/** The revision the log entry was stamped with (see {@link pendCollection}). */
|
|
1870
|
-
rev: number,
|
|
1871
|
-
actionId: ActionId,
|
|
2235
|
+
attempt: CollectionAttempt,
|
|
1872
2236
|
nominees: PeerId[] | undefined,
|
|
1873
2237
|
/** Whether this request is the action's whole reservation, alone or with the siblings it is sent
|
|
1874
2238
|
* with (`PendRequest.soleReservation`). */
|
|
@@ -1882,24 +2246,20 @@ export class TransactionCoordinator {
|
|
|
1882
2246
|
// never age. Omitted at 0, exactly as the single-collection path sends it
|
|
1883
2247
|
// (TransactorSource.transact), so the common first-attempt pend is indistinguishable from a
|
|
1884
2248
|
// Collection.sync pend.
|
|
1885
|
-
|
|
1886
|
-
//
|
|
1887
|
-
// tracker the commit's declaration was taken from (declareTouchedBlocks); it still holds this
|
|
1888
|
-
// transaction's staged transforms and pins. Storage keeps it with the pending record so the
|
|
1889
|
-
// record is never applied to a different version of the block (see PendRequest.baseRevs).
|
|
1890
|
-
// Omitted when empty.
|
|
1891
|
-
const baseRevs = collection.tracker.stagedBaseRevs(blockIdsForTransforms(transforms));
|
|
2249
|
+
// The bases (`PendRequest.baseRevs`) were read from the same tracker the commit's declaration was
|
|
2250
|
+
// taken from, when the attempt was described; omitted when empty.
|
|
1892
2251
|
return {
|
|
1893
|
-
actionId,
|
|
1894
|
-
rev,
|
|
1895
|
-
transforms,
|
|
2252
|
+
actionId: context.actionId,
|
|
2253
|
+
rev: attempt.rev,
|
|
2254
|
+
transforms: attempt.transforms,
|
|
1896
2255
|
policy: 'r', // Return policy: fail but return pending actions
|
|
1897
2256
|
...(validation === undefined
|
|
1898
|
-
? (priority > 0 ? { priority } : {})
|
|
2257
|
+
? (context.priority > 0 ? { priority: context.priority } : {})
|
|
1899
2258
|
: { validation }),
|
|
1900
|
-
...baseRevsField(baseRevs),
|
|
2259
|
+
...baseRevsField(attempt.baseRevs),
|
|
1901
2260
|
superclusterNominees: nominees,
|
|
1902
|
-
...(sole ? { soleReservation: true as const } : {})
|
|
2261
|
+
...(sole ? { soleReservation: true as const } : {}),
|
|
2262
|
+
...refuseSoloField(context.refuseSolo)
|
|
1903
2263
|
};
|
|
1904
2264
|
}
|
|
1905
2265
|
|
|
@@ -1913,11 +2273,9 @@ export class TransactionCoordinator {
|
|
|
1913
2273
|
* the caller can do targeted cancel.
|
|
1914
2274
|
*/
|
|
1915
2275
|
private async commitPhase(
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
pendedBlockIds: Map<CollectionId, BlockId[]
|
|
1919
|
-
pendedRevs: ReadonlyMap<CollectionId, number>,
|
|
1920
|
-
declared: DeclaredDigests
|
|
2276
|
+
context: WriteContext,
|
|
2277
|
+
attempts: ReadonlyMap<CollectionId, CollectionAttempt>,
|
|
2278
|
+
pendedBlockIds: Map<CollectionId, BlockId[]>
|
|
1921
2279
|
): Promise<{
|
|
1922
2280
|
success: boolean;
|
|
1923
2281
|
error?: string;
|
|
@@ -1928,7 +2286,7 @@ export class TransactionCoordinator {
|
|
|
1928
2286
|
expired?: TransactionExpiredError;
|
|
1929
2287
|
}> {
|
|
1930
2288
|
const together = pendedBlockIds.size > 1 && this.transactor.commitMany
|
|
1931
|
-
? await this.commitTogether(
|
|
2289
|
+
? await this.commitTogether(context, attempts, pendedBlockIds)
|
|
1932
2290
|
: undefined;
|
|
1933
2291
|
// NOTE: a fallback after a batched pend leaves the multi-collection residual in place: the
|
|
1934
2292
|
// pend's decision may have been held back for a commit that now goes per collection, so a member
|
|
@@ -1940,7 +2298,7 @@ export class TransactionCoordinator {
|
|
|
1940
2298
|
// cache's TTL is at least a minute, its size cap 1000 transactions), so a live lookup can split
|
|
1941
2299
|
// the blocks. If that ever shows up, pin the commit to the coordinator the batched pend answered
|
|
1942
2300
|
// from. See docs/correctness.md §2 Definitions.
|
|
1943
|
-
const outcomes = together ?? await this.commitEach(
|
|
2301
|
+
const outcomes = together ?? await this.commitEach(context, attempts, pendedBlockIds);
|
|
1944
2302
|
|
|
1945
2303
|
const committedCollections = new Map<CollectionId, WriteDurability>();
|
|
1946
2304
|
const failedCollections = new Set<CollectionId>();
|
|
@@ -1987,21 +2345,20 @@ export class TransactionCoordinator {
|
|
|
1987
2345
|
* land (`Collection.completeOwnEntry`) — the path a refused one-collection commit already takes.
|
|
1988
2346
|
*
|
|
1989
2347
|
* `undefined` when they could not go as one, when the request threw, or when a collection's request
|
|
1990
|
-
* could not be built (
|
|
1991
|
-
*
|
|
1992
|
-
*
|
|
1993
|
-
*
|
|
2348
|
+
* could not be built (no log tail among its pended blocks): {@link commitPhase} then commits each
|
|
2349
|
+
* collection on its own, which re-sends everything and reports a request it cannot build as it
|
|
2350
|
+
* always has. The batch is never retried here: a returned refusal cannot be won by the identical
|
|
2351
|
+
* request, and a throw is the per-collection path's to retry.
|
|
1994
2352
|
*/
|
|
1995
2353
|
private async commitTogether(
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
pendedBlockIds: Map<CollectionId, BlockId[]
|
|
1999
|
-
pendedRevs: ReadonlyMap<CollectionId, number>,
|
|
2000
|
-
declared: DeclaredDigests
|
|
2354
|
+
context: WriteContext,
|
|
2355
|
+
attempts: ReadonlyMap<CollectionId, CollectionAttempt>,
|
|
2356
|
+
pendedBlockIds: Map<CollectionId, BlockId[]>
|
|
2001
2357
|
): Promise<CollectionCommitOutcome[] | undefined> {
|
|
2358
|
+
const actionId = context.actionId;
|
|
2002
2359
|
const requests: { collectionId: CollectionId; request: CommitRequest }[] = [];
|
|
2003
2360
|
for (const [collectionId, blockIds] of pendedBlockIds) {
|
|
2004
|
-
const request = this.commitRequestFor(
|
|
2361
|
+
const request = this.commitRequestFor(context, collectionId, attempts.get(collectionId)!, blockIds);
|
|
2005
2362
|
if (typeof request === 'string') return undefined;
|
|
2006
2363
|
requests.push({ collectionId, request });
|
|
2007
2364
|
}
|
|
@@ -2020,15 +2377,13 @@ export class TransactionCoordinator {
|
|
|
2020
2377
|
|
|
2021
2378
|
/** Commits each collection on its own, concurrently ({@link commitCollection}), one outcome per collection. */
|
|
2022
2379
|
private async commitEach(
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
pendedBlockIds: Map<CollectionId, BlockId[]
|
|
2026
|
-
pendedRevs: ReadonlyMap<CollectionId, number>,
|
|
2027
|
-
declared: DeclaredDigests
|
|
2380
|
+
context: WriteContext,
|
|
2381
|
+
attempts: ReadonlyMap<CollectionId, CollectionAttempt>,
|
|
2382
|
+
pendedBlockIds: Map<CollectionId, BlockId[]>
|
|
2028
2383
|
): Promise<CollectionCommitOutcome[]> {
|
|
2029
2384
|
const entries = Array.from(pendedBlockIds.entries());
|
|
2030
2385
|
const settled = await Promise.allSettled(entries.map(([collectionId, blockIds]) =>
|
|
2031
|
-
this.commitCollection(
|
|
2386
|
+
this.commitCollection(context, collectionId, attempts.get(collectionId)!, blockIds)
|
|
2032
2387
|
));
|
|
2033
2388
|
// commitCollection resolves rather than rejects, but treat any unexpected rejection as a (hard)
|
|
2034
2389
|
// failure of its collection so the partitioned sets stay honest.
|
|
@@ -2044,15 +2399,13 @@ export class TransactionCoordinator {
|
|
|
2044
2399
|
* {@link commitEach} can aggregate every result.
|
|
2045
2400
|
*/
|
|
2046
2401
|
private async commitCollection(
|
|
2047
|
-
|
|
2048
|
-
criticalBlockIds: BlockId[],
|
|
2402
|
+
context: WriteContext,
|
|
2049
2403
|
collectionId: CollectionId,
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
declared: DeclaredDigests
|
|
2404
|
+
/** Its revision is the one this collection PENDED at (see {@link commitRequestFor}). */
|
|
2405
|
+
attempt: CollectionAttempt,
|
|
2406
|
+
blockIds: BlockId[]
|
|
2054
2407
|
): Promise<CollectionCommitOutcome> {
|
|
2055
|
-
const commitRequest = this.commitRequestFor(
|
|
2408
|
+
const commitRequest = this.commitRequestFor(context, collectionId, attempt, blockIds);
|
|
2056
2409
|
if (typeof commitRequest === 'string') {
|
|
2057
2410
|
return { collectionId, committed: false, error: commitRequest };
|
|
2058
2411
|
}
|
|
@@ -2083,37 +2436,31 @@ export class TransactionCoordinator {
|
|
|
2083
2436
|
* — or, as a string, why there is none.
|
|
2084
2437
|
*/
|
|
2085
2438
|
private commitRequestFor(
|
|
2086
|
-
|
|
2087
|
-
criticalBlockIds: BlockId[],
|
|
2439
|
+
context: WriteContext,
|
|
2088
2440
|
collectionId: CollectionId,
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
*
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
declared: DeclaredDigests
|
|
2441
|
+
/** Its revision is the one this collection PENDED at, threaded from the log append — the same
|
|
2442
|
+
* bug family as pendCollection's: recomputing `getNextRev()` here, after the pend round trips,
|
|
2443
|
+
* could stamp the CommitRequest with a different number than the pend named. */
|
|
2444
|
+
attempt: CollectionAttempt,
|
|
2445
|
+
blockIds: BlockId[]
|
|
2095
2446
|
): CommitRequest | string {
|
|
2096
|
-
|
|
2097
|
-
if (!
|
|
2098
|
-
return `Collection not found: ${collectionId}`;
|
|
2099
|
-
}
|
|
2100
|
-
|
|
2101
|
-
// Find the critical block (log tail) for this collection
|
|
2102
|
-
const logTailBlockId = criticalBlockIds.find(blockId => blockIds.includes(blockId));
|
|
2103
|
-
if (!logTailBlockId) {
|
|
2447
|
+
// The critical block (log tail) for this collection must be among what the pend reserved.
|
|
2448
|
+
if (!blockIds.includes(attempt.tailId)) {
|
|
2104
2449
|
return `Log tail block not found for collection ${collectionId}`;
|
|
2105
2450
|
}
|
|
2106
2451
|
|
|
2107
2452
|
// Declare what each pended block will contain once committed: the description taken before the
|
|
2108
|
-
// pend (declareTouchedBlocks), over the blocks the pend
|
|
2109
|
-
// is omitted and falls back to corroboration on the
|
|
2110
|
-
// this — CoordinatorRepo.commit forwards without
|
|
2453
|
+
// pend (declareTouchedBlocks, or stageAttempt for a queued entry's), over the blocks the pend
|
|
2454
|
+
// carried. An id whose base was not cached is omitted and falls back to corroboration on the
|
|
2455
|
+
// member side. Only the client can declare this — CoordinatorRepo.commit forwards without
|
|
2456
|
+
// materializing.
|
|
2111
2457
|
return {
|
|
2112
|
-
actionId,
|
|
2458
|
+
actionId: context.actionId,
|
|
2113
2459
|
blockIds,
|
|
2114
|
-
tailId:
|
|
2115
|
-
rev,
|
|
2116
|
-
...blockDigestsField(
|
|
2460
|
+
tailId: attempt.tailId,
|
|
2461
|
+
rev: attempt.rev,
|
|
2462
|
+
...blockDigestsField(attempt.blockDigests),
|
|
2463
|
+
...refuseSoloField(context.refuseSolo)
|
|
2117
2464
|
};
|
|
2118
2465
|
}
|
|
2119
2466
|
|