@optimystic/db-core 1.22.0 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/dist/src/collection/action.d.ts +7 -4
- package/dist/src/collection/action.d.ts.map +1 -1
- package/dist/src/collection/action.js +13 -5
- package/dist/src/collection/action.js.map +1 -1
- package/dist/src/collection/collection.d.ts +134 -46
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +207 -70
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/struct.d.ts +38 -8
- package/dist/src/collection/struct.d.ts.map +1 -1
- package/dist/src/collection/struct.js.map +1 -1
- package/dist/src/collection/write-attempts.d.ts +138 -34
- package/dist/src/collection/write-attempts.d.ts.map +1 -1
- package/dist/src/collection/write-attempts.js +168 -101
- package/dist/src/collection/write-attempts.js.map +1 -1
- package/dist/src/collections/diary/diary.d.ts.map +1 -1
- package/dist/src/collections/diary/diary.js +5 -1
- package/dist/src/collections/diary/diary.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts +15 -9
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +6 -4
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/hlc/clock-store.d.ts +63 -0
- package/dist/src/hlc/clock-store.d.ts.map +1 -0
- package/dist/src/hlc/clock-store.js +49 -0
- package/dist/src/hlc/clock-store.js.map +1 -0
- package/dist/src/hlc/clock.d.ts +64 -0
- package/dist/src/hlc/clock.d.ts.map +1 -0
- package/dist/src/hlc/clock.js +171 -0
- package/dist/src/hlc/clock.js.map +1 -0
- package/dist/src/hlc/hlc.d.ts +38 -0
- package/dist/src/hlc/hlc.d.ts.map +1 -0
- package/dist/src/hlc/hlc.js +41 -0
- package/dist/src/hlc/hlc.js.map +1 -0
- package/dist/src/hlc/index.d.ts +5 -0
- package/dist/src/hlc/index.d.ts.map +1 -0
- package/dist/src/hlc/index.js +5 -0
- package/dist/src/hlc/index.js.map +1 -0
- package/dist/src/hlc/skew.d.ts +35 -0
- package/dist/src/hlc/skew.d.ts.map +1 -0
- package/dist/src/hlc/skew.js +35 -0
- package/dist/src/hlc/skew.js.map +1 -0
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/lane/index.d.ts +2 -0
- package/dist/src/lane/index.d.ts.map +1 -0
- package/dist/src/lane/index.js +2 -0
- package/dist/src/lane/index.js.map +1 -0
- package/dist/src/lane/lane.d.ts +46 -0
- package/dist/src/lane/lane.d.ts.map +1 -0
- package/dist/src/lane/lane.js +26 -0
- package/dist/src/lane/lane.js.map +1 -0
- package/dist/src/log/log.d.ts +50 -22
- package/dist/src/log/log.d.ts.map +1 -1
- package/dist/src/log/log.js +21 -13
- package/dist/src/log/log.js.map +1 -1
- package/dist/src/log/struct.d.ts +61 -1
- package/dist/src/log/struct.d.ts.map +1 -1
- package/dist/src/log/struct.js +24 -0
- package/dist/src/log/struct.js.map +1 -1
- package/dist/src/network/i-repo.d.ts +10 -0
- package/dist/src/network/i-repo.d.ts.map +1 -1
- package/dist/src/network/stale-failure.d.ts +20 -7
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +29 -7
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/network/struct.d.ts +35 -0
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/offline/entry.d.ts +14 -0
- package/dist/src/offline/entry.d.ts.map +1 -1
- package/dist/src/offline/entry.js +2 -0
- package/dist/src/offline/entry.js.map +1 -1
- package/dist/src/offline/journal.d.ts +10 -0
- package/dist/src/offline/journal.d.ts.map +1 -1
- package/dist/src/offline/queue.d.ts +10 -0
- package/dist/src/offline/queue.d.ts.map +1 -1
- package/dist/src/offline/queue.js +29 -3
- package/dist/src/offline/queue.js.map +1 -1
- package/dist/src/offline/status.d.ts +7 -0
- package/dist/src/offline/status.d.ts.map +1 -1
- package/dist/src/offline/status.js.map +1 -1
- package/dist/src/testing/test-transactor.d.ts +9 -1
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +6 -1
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/actions-engine.d.ts +3 -0
- package/dist/src/transaction/actions-engine.d.ts.map +1 -1
- package/dist/src/transaction/actions-engine.js +3 -0
- package/dist/src/transaction/actions-engine.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +10 -5
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +197 -48
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transaction/entry-commit.d.ts +45 -10
- package/dist/src/transaction/entry-commit.d.ts.map +1 -1
- package/dist/src/transaction/entry-commit.js +20 -7
- package/dist/src/transaction/entry-commit.js.map +1 -1
- package/dist/src/transaction/errors.d.ts +69 -1
- package/dist/src/transaction/errors.d.ts.map +1 -1
- package/dist/src/transaction/errors.js +70 -1
- package/dist/src/transaction/errors.js.map +1 -1
- package/dist/src/transaction/index.d.ts +1 -1
- package/dist/src/transaction/index.d.ts.map +1 -1
- package/dist/src/transaction/index.js +1 -1
- package/dist/src/transaction/index.js.map +1 -1
- package/dist/src/transaction/session.d.ts.map +1 -1
- package/dist/src/transaction/session.js +18 -5
- package/dist/src/transaction/session.js.map +1 -1
- package/dist/src/transaction/transaction.d.ts +74 -15
- package/dist/src/transaction/transaction.d.ts.map +1 -1
- package/dist/src/transaction/transaction.js +35 -15
- package/dist/src/transaction/transaction.js.map +1 -1
- package/dist/src/transaction/validator.d.ts +25 -4
- package/dist/src/transaction/validator.d.ts.map +1 -1
- package/dist/src/transaction/validator.js +45 -5
- package/dist/src/transaction/validator.js.map +1 -1
- package/dist/src/transactor/block-floors.d.ts +12 -1
- package/dist/src/transactor/block-floors.d.ts.map +1 -1
- package/dist/src/transactor/block-floors.js +14 -1
- package/dist/src/transactor/block-floors.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +8 -0
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +8 -4
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor-source.d.ts +11 -0
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +5 -1
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/dist/src/transactor/transactor.d.ts +8 -0
- package/dist/src/transactor/transactor.d.ts.map +1 -1
- package/dist/src/transform/cache-source.d.ts +7 -0
- package/dist/src/transform/cache-source.d.ts.map +1 -1
- package/dist/src/transform/cache-source.js +14 -18
- package/dist/src/transform/cache-source.js.map +1 -1
- package/package.json +1 -1
- package/src/collection/action.ts +13 -5
- package/src/collection/collection.ts +250 -76
- package/src/collection/struct.ts +38 -8
- package/src/collection/write-attempts.ts +213 -109
- package/src/collections/diary/diary.ts +5 -1
- package/src/collections/tree/tree.ts +18 -11
- package/src/hlc/clock-store.ts +68 -0
- package/src/hlc/clock.ts +196 -0
- package/src/hlc/hlc.ts +56 -0
- package/src/hlc/index.ts +4 -0
- package/src/hlc/skew.ts +55 -0
- package/src/index.ts +2 -0
- package/src/lane/index.ts +1 -0
- package/src/lane/lane.ts +54 -0
- package/src/log/log.ts +341 -310
- package/src/log/struct.ts +62 -1
- package/src/network/i-repo.ts +11 -1
- package/src/network/stale-failure.ts +31 -7
- package/src/network/struct.ts +805 -770
- package/src/offline/entry.ts +16 -0
- package/src/offline/journal.ts +10 -0
- package/src/offline/queue.ts +38 -4
- package/src/offline/status.ts +7 -0
- package/src/testing/test-transactor.ts +1065 -1053
- package/src/transaction/actions-engine.ts +3 -0
- package/src/transaction/coordinator.ts +229 -54
- package/src/transaction/entry-commit.ts +56 -14
- package/src/transaction/errors.ts +60 -0
- package/src/transaction/index.ts +1 -1
- package/src/transaction/session.ts +18 -5
- package/src/transaction/transaction.ts +90 -17
- package/src/transaction/validator.ts +46 -5
- package/src/transactor/block-floors.ts +15 -1
- package/src/transactor/network-transactor.ts +15 -4
- package/src/transactor/transactor-source.ts +15 -1
- package/src/transactor/transactor.ts +9 -0
- package/src/transform/cache-source.ts +15 -17
|
@@ -1,27 +1,30 @@
|
|
|
1
1
|
import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, PendResult, CommitRequest, CommitResult, WriteFailure, ActionId, WriteDurability, BlockContentDigests } from "../index.js";
|
|
2
2
|
import type { BlockBaseRevs } from "../network/struct.js";
|
|
3
|
+
import { clockOf, type HlcClock } from "../hlc/clock.js";
|
|
3
4
|
import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions, ReadDependency } from "./transaction.js";
|
|
4
5
|
import type { PeerId } from "../network/types.js";
|
|
5
|
-
import { isConflictFailure, isExpiryFailure, isRejectedFailure, isShortfallFailure } from "../network/stale-failure.js";
|
|
6
|
+
import { isConflictFailure, isExpiryFailure, isRejectedFailure, isShortfallFailure, staleAtOwner } from "../network/stale-failure.js";
|
|
6
7
|
import { withBlockRevs } from "../network/durability.js";
|
|
7
8
|
import { TransactionExpiredError } from "../network/transaction-expired-error.js";
|
|
8
9
|
import { PromiseShortfallError } from "../network/promise-shortfall-error.js";
|
|
9
10
|
import { ValidityRejectedError } from "../network/validity-rejected-error.js";
|
|
10
11
|
import { refuseSoloField } from "../network/refuse-solo.js";
|
|
12
|
+
import type { Lane } from "../lane/lane.js";
|
|
11
13
|
import { TransactionTooLargeError } from "../network/transaction-size.js";
|
|
12
14
|
import { BlockTooLargeError, oversizedBlock } from "../network/block-size.js";
|
|
13
15
|
import type { Collection, CollectionSnapshot, RefreshReport } from "../collection/collection.js";
|
|
14
16
|
import type { InFlightCycle, SyncOptions } from "../collection/index.js";
|
|
15
|
-
import
|
|
17
|
+
import { DefaultMaxStalledAttempts, RevisionStallWatch } from "../collection/write-attempts.js";
|
|
18
|
+
import type { ConfirmedRevision, StagedAttempt } from "../collection/write-attempts.js";
|
|
16
19
|
import { TornActionError } from "../collection/struct.js";
|
|
17
20
|
import { isTransactionExpired, clampPriority } from "./transaction.js";
|
|
18
|
-
import { EntryParticipants, writeContextOf } from "./entry-commit.js";
|
|
19
|
-
import type { EntryCommit, RefreshParticipant, WriteContext } from "./entry-commit.js";
|
|
21
|
+
import { EntryParticipants, compareCollectionIds, writeContextOf } from "./entry-commit.js";
|
|
22
|
+
import type { EntryCommit, NextRevision, RefreshParticipant, WriteContext } from "./entry-commit.js";
|
|
20
23
|
import { Log } from "../log/log.js";
|
|
21
24
|
import { blockIdsForTransforms, copyTransforms } from "../transform/helpers.js";
|
|
22
25
|
import { describeTouchedBlocks, blockDigestsField, baseRevsField } from "../transform/digest.js";
|
|
23
26
|
import { collectOperations, hashOperations } from "./operations-hash.js";
|
|
24
|
-
import { CoordinatorConcurrentStampError, CoordinatorPartialCommitError, CoordinatorStaleLossError } from "./errors.js";
|
|
27
|
+
import { CoordinatorConcurrentStampError, CoordinatorPartialCommitError, CoordinatorRevisionStalledError, CoordinatorStaleLossError } from "./errors.js";
|
|
25
28
|
import { recordCollectionDurability, transactionDurability, type TransactionDurability } from "./durability.js";
|
|
26
29
|
import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
|
|
27
30
|
import { visitEveryThenThrow } from "../utility/visit-every.js";
|
|
@@ -53,6 +56,10 @@ class PendRejectedError extends Error {
|
|
|
53
56
|
* message because pendPhase collapses this error to its `.message` string, which is the only
|
|
54
57
|
* form that reaches an embedder through the transaction result's `error` field. */
|
|
55
58
|
staleAt?: { blockId: BlockId; rev: number },
|
|
59
|
+
/** The same number attributed to the collection whose counter it belongs to (see
|
|
60
|
+
* {@link ownStaleAt} and {@link batchStaleAt}), for the retry loop's per-participant stall
|
|
61
|
+
* check; empty when the refusal carried none or named a block no collection of it sent. */
|
|
62
|
+
readonly staleAtByCollection: ReadonlyMap<CollectionId, ConfirmedRevision> = NoStaleAt,
|
|
56
63
|
) {
|
|
57
64
|
super(`Pend failed for ${collectionIds.length === 1 ? 'collection' : 'collections'} ${collectionIds.join(', ')}: ${reason ?? (conflict ? 'stale conflict' : 'rejected')}`
|
|
58
65
|
+ (staleAt ? ` (block ${staleAt.blockId} at rev ${staleAt.rev})` : ''));
|
|
@@ -60,24 +67,51 @@ class PendRejectedError extends Error {
|
|
|
60
67
|
}
|
|
61
68
|
}
|
|
62
69
|
|
|
70
|
+
/** The attribution of a refusal that carried no confirmed number, or one no collection owns. */
|
|
71
|
+
const NoStaleAt: ReadonlyMap<CollectionId, ConfirmedRevision> = new Map();
|
|
72
|
+
|
|
73
|
+
/** A refusal's confirmed revision as the ONE collection's own: a request for one collection
|
|
74
|
+
* (`pendCollection`, `commitCollection`) gets a number under that collection's counter whatever block
|
|
75
|
+
* it names, so it is attributed without looking at the block id — as the single-collection loop does. */
|
|
76
|
+
function ownStaleAt(collectionId: CollectionId, staleAt: ConfirmedRevision | undefined): ReadonlyMap<CollectionId, ConfirmedRevision> {
|
|
77
|
+
return staleAt === undefined ? NoStaleAt : new Map([[collectionId, staleAt]]);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** A batched answer's confirmed revision (`pendTogether`, `commitTogether`), attributed to the one
|
|
81
|
+
* collection whose blocks include the block it names (`staleAtOwner`): the batch carries ONE number,
|
|
82
|
+
* the highest across its collections' counters, and every other collection of the batch was refused
|
|
83
|
+
* with no confirmed number of its own. Nothing is attributed when no collection of the batch owns the
|
|
84
|
+
* block. */
|
|
85
|
+
function batchStaleAt(staleAt: ConfirmedRevision | undefined, blocksByCollection: ReadonlyMap<CollectionId, readonly BlockId[]>): ReadonlyMap<CollectionId, ConfirmedRevision> {
|
|
86
|
+
const owner = staleAtOwner(staleAt, blocksByCollection);
|
|
87
|
+
return owner === undefined || staleAt === undefined ? NoStaleAt : new Map([[owner, staleAt]]);
|
|
88
|
+
}
|
|
89
|
+
|
|
63
90
|
/** The error a refused pend of `collectionIds` stands for: its validity rejection — never retried, since
|
|
64
91
|
* a re-drive sends the same content to the same checks — its expiry — never retried, since a re-drive's
|
|
65
92
|
* expiration comes from the same clock — or its promise shortfall — never retried, since a re-drive
|
|
66
|
-
* meets the same silent members — or a rejection carrying whether a re-read can clear it
|
|
93
|
+
* meets the same silent members — or a rejection carrying whether a re-read can clear it, with its
|
|
94
|
+
* confirmed revision attributed per collection (`staleAtByCollection`, which the caller works out:
|
|
95
|
+
* {@link ownStaleAt} for one collection's pend, {@link batchStaleAt} for a batched one).
|
|
67
96
|
* Retryability comes from the response itself: a producer that classified the failure sets `conflict`,
|
|
68
97
|
* and only where none did is it inferred from `missing`/`pending` (`isConflictFailure`). */
|
|
69
|
-
function pendRefusalError(
|
|
98
|
+
function pendRefusalError(
|
|
99
|
+
collectionIds: readonly CollectionId[],
|
|
100
|
+
refusal: WriteFailure,
|
|
101
|
+
staleAtByCollection: ReadonlyMap<CollectionId, ConfirmedRevision>,
|
|
102
|
+
): ValidityRejectedError | TransactionExpiredError | PromiseShortfallError | PendRejectedError {
|
|
70
103
|
if (isRejectedFailure(refusal)) return new ValidityRejectedError(refusal.rejected);
|
|
71
104
|
if (isExpiryFailure(refusal)) return new TransactionExpiredError(refusal.expired);
|
|
72
105
|
if (isShortfallFailure(refusal)) return new PromiseShortfallError(refusal.shortfall);
|
|
73
|
-
return new PendRejectedError(collectionIds, isConflictFailure(refusal), refusal.reason, refusal.staleAt);
|
|
106
|
+
return new PendRejectedError(collectionIds, isConflictFailure(refusal), refusal.reason, refusal.staleAt, staleAtByCollection);
|
|
74
107
|
}
|
|
75
108
|
|
|
76
109
|
/** A transaction refused before anything was sent: its pend, or a block it would leave, too large to send. */
|
|
77
110
|
type TooLargeError = TransactionTooLargeError | BlockTooLargeError;
|
|
78
111
|
|
|
79
|
-
/** How the pend phase failed.
|
|
80
|
-
|
|
112
|
+
/** How the pend phase failed. `staleAt` is every refused collection's confirmed revision, where its
|
|
113
|
+
* refusal carried one (see `PendRejectedError.staleAtByCollection`). */
|
|
114
|
+
type PendPhaseFailure = { success: false; error: string; staleLoss?: boolean; staleAt?: ReadonlyMap<CollectionId, ConfirmedRevision>; rejected?: ValidityRejectedError; expired?: TransactionExpiredError; shortfall?: PromiseShortfallError; tooLarge?: TooLargeError };
|
|
81
115
|
|
|
82
116
|
/**
|
|
83
117
|
* One collection's share of an attempt, as the phases send it. For a transaction's attempt it is
|
|
@@ -122,15 +156,19 @@ function collectionAttemptOf(staged: StagedAttempt): CollectionAttempt {
|
|
|
122
156
|
*/
|
|
123
157
|
function pendFailure(reasons: readonly unknown[]): PendPhaseFailure {
|
|
124
158
|
const first = reasons[0];
|
|
125
|
-
const conflicts = reasons.filter(reason => reason instanceof PendRejectedError && reason.conflict);
|
|
159
|
+
const conflicts = reasons.filter((reason): reason is PendRejectedError => reason instanceof PendRejectedError && reason.conflict);
|
|
126
160
|
const rejected = reasons.find((reason): reason is ValidityRejectedError => reason instanceof ValidityRejectedError);
|
|
127
161
|
const expired = reasons.find((reason): reason is TransactionExpiredError => reason instanceof TransactionExpiredError);
|
|
128
162
|
const shortfall = reasons.find((reason): reason is PromiseShortfallError => reason instanceof PromiseShortfallError);
|
|
129
163
|
const tooLarge = reasons.find((reason): reason is TooLargeError => reason instanceof TransactionTooLargeError || reason instanceof BlockTooLargeError);
|
|
164
|
+
// Each refusal attributed its number to its own collection(s), and no collection is refused twice
|
|
165
|
+
// in one phase, so the maps are disjoint and merge without a rule.
|
|
166
|
+
const staleAt = new Map(conflicts.flatMap(conflict => [...conflict.staleAtByCollection]));
|
|
130
167
|
return {
|
|
131
168
|
success: false,
|
|
132
169
|
error: first instanceof Error ? first.message : String(first),
|
|
133
170
|
staleLoss: conflicts.length > 0 && conflicts.length === reasons.length,
|
|
171
|
+
staleAt,
|
|
134
172
|
...(rejected === undefined ? {} : { rejected }),
|
|
135
173
|
...(expired === undefined ? {} : { expired }),
|
|
136
174
|
...(shortfall === undefined ? {} : { shortfall }),
|
|
@@ -172,23 +210,27 @@ type CommitCycle = {
|
|
|
172
210
|
readonly inFlight: InFlightCycle;
|
|
173
211
|
};
|
|
174
212
|
|
|
175
|
-
/** What a commit's
|
|
176
|
-
|
|
177
|
-
|
|
213
|
+
/** What every send of a commit's cycle carries: the write's lane — the transaction's or the queued
|
|
214
|
+
* entry's, never `options.lane`, which the transaction and the entry already folded in — and what the
|
|
215
|
+
* commit's `options` ask of every send besides. */
|
|
216
|
+
function inFlightCycleOf(options: SyncOptions | undefined, lane: Lane): InFlightCycle {
|
|
217
|
+
return { lane, ...refuseSoloField(options?.refuseSoloCommit) };
|
|
178
218
|
}
|
|
179
219
|
|
|
180
220
|
/** How one collection's commit ended ({@link TransactionCoordinator.commitCollection}, or its share of
|
|
181
221
|
* a batched commit's one answer). */
|
|
182
222
|
type CollectionCommitOutcome =
|
|
183
223
|
| { collectionId: CollectionId; committed: true; durability: WriteDurability }
|
|
184
|
-
| { collectionId: CollectionId; committed: false; error: string; stale?: boolean; expired?: TransactionExpiredError };
|
|
224
|
+
| { collectionId: CollectionId; committed: false; error: string; stale?: boolean; staleAt?: ConfirmedRevision; expired?: TransactionExpiredError };
|
|
185
225
|
|
|
186
226
|
/**
|
|
187
227
|
* How `collectionId`'s commit ended when `result` answered its `request` — the answer to its own
|
|
188
228
|
* commit, or, in a batched commit, the batch's one answer for every collection of it. A success is
|
|
189
|
-
* stamped with the collection's own blocks at its own revision (`withBlockRevs`).
|
|
229
|
+
* stamped with the collection's own blocks at its own revision (`withBlockRevs`). `staleAt` is the
|
|
230
|
+
* answer's confirmed revision as far as it is THIS collection's: the answer's own for a commit of one
|
|
231
|
+
* collection, the owner's share of a batched answer (`batchStaleAt`), and `undefined` otherwise.
|
|
190
232
|
*/
|
|
191
|
-
function collectionCommitOutcome(collectionId: CollectionId, request: CommitRequest, result: CommitResult): CollectionCommitOutcome {
|
|
233
|
+
function collectionCommitOutcome(collectionId: CollectionId, request: CommitRequest, result: CommitResult, staleAt: ConfirmedRevision | undefined): CollectionCommitOutcome {
|
|
192
234
|
if (result.success) {
|
|
193
235
|
return { collectionId, committed: true, durability: withBlockRevs(result.durability, request.blockIds, request.rev) };
|
|
194
236
|
}
|
|
@@ -215,7 +257,8 @@ function collectionCommitOutcome(collectionId: CollectionId, request: CommitRequ
|
|
|
215
257
|
collectionId,
|
|
216
258
|
committed: false,
|
|
217
259
|
stale: true,
|
|
218
|
-
error: result.reason ?? `Stale commit for collection ${collectionId}
|
|
260
|
+
error: result.reason ?? `Stale commit for collection ${collectionId}`,
|
|
261
|
+
...(staleAt === undefined ? {} : { staleAt }),
|
|
219
262
|
};
|
|
220
263
|
}
|
|
221
264
|
|
|
@@ -232,6 +275,9 @@ type CoordinationResult =
|
|
|
232
275
|
/** True when the failure was a clean optimistic-concurrency conflict (stale loss / pending
|
|
233
276
|
* contention) with nothing durable — i.e. safe to re-drive after a re-read. */
|
|
234
277
|
staleLoss?: boolean;
|
|
278
|
+
/** Per failed collection, the revision its refusal confirmed is taken, from whichever phase
|
|
279
|
+
* failed (`PendPhaseFailure.staleAt`, or the commit phase's). */
|
|
280
|
+
staleAt?: ReadonlyMap<CollectionId, ConfirmedRevision>;
|
|
235
281
|
/** Present when a pend was refused because the members that sank it judged the transaction
|
|
236
282
|
* invalid: the error the commit surfaces, never a stale loss. */
|
|
237
283
|
rejected?: ValidityRejectedError;
|
|
@@ -264,7 +310,7 @@ function writtenDurability(written: ReadonlyMap<CollectionId, WriteDurability |
|
|
|
264
310
|
*/
|
|
265
311
|
function cleanFailureError(result: CoordinationResult & { success: false }, failed: readonly CollectionId[]): Error {
|
|
266
312
|
if (result.staleLoss) {
|
|
267
|
-
return new CoordinatorStaleLossError(failed, result.error);
|
|
313
|
+
return new CoordinatorStaleLossError(failed, result.error, result.staleAt);
|
|
268
314
|
}
|
|
269
315
|
return result.rejected ?? result.expired ?? result.shortfall ?? result.tooLarge
|
|
270
316
|
?? new Error(`Transaction commit failed: ${result.error}`);
|
|
@@ -277,6 +323,8 @@ function cleanFailureError(result: CoordinationResult & { success: false }, fail
|
|
|
277
323
|
* the loop owns the budget, the backoff, and the exit where a refresh saved everything.
|
|
278
324
|
*/
|
|
279
325
|
type AttemptDriver = {
|
|
326
|
+
/** The write's action id, for the loop's log lines. */
|
|
327
|
+
readonly actionId: ActionId;
|
|
280
328
|
/** Whether the cycle opens with a refresh round before its first attempt. The per-entry commit
|
|
281
329
|
* does, as `Collection.commitHead` does: the queue re-drives an entry under the id of an attempt
|
|
282
330
|
* that may have landed, and the opening refresh is what finds and finishes that attempt's entry
|
|
@@ -286,11 +334,15 @@ type AttemptDriver = {
|
|
|
286
334
|
* it. Throws {@link CoordinatorStaleLossError} for a clean loss, and for a partial landing retried
|
|
287
335
|
* forward, whose committed participants it has already recorded in `cycle.saved`. */
|
|
288
336
|
attempt(priority: number): Promise<ReadonlyMap<CollectionId, WriteDurability>>;
|
|
289
|
-
/** The refresh round between attempts — see {@link TransactionCoordinator.refreshBetweenAttempts}.
|
|
290
|
-
|
|
337
|
+
/** The refresh round between attempts — see {@link TransactionCoordinator.refreshBetweenAttempts}.
|
|
338
|
+
* `tailFloor` answers, per collection, the floor its refresh's log-tail read carries. */
|
|
339
|
+
refresh(lastChance: boolean, tailFloor: (collectionId: CollectionId) => number | undefined): Promise<TornActionError | undefined>;
|
|
291
340
|
/** Whether an attempt starting now would have nothing left to commit, so that once a refresh has
|
|
292
341
|
* saved a participant the write is finished without going round again. */
|
|
293
342
|
nothingLeft(): boolean;
|
|
343
|
+
/** The participants an attempt starting now would commit, with the revision each would request
|
|
344
|
+
* and the one it holds, for the stall check ({@link RetryBudget.assertNotStalled}). */
|
|
345
|
+
nextRevisions(): NextRevision[];
|
|
294
346
|
};
|
|
295
347
|
|
|
296
348
|
/**
|
|
@@ -298,12 +350,17 @@ type AttemptDriver = {
|
|
|
298
350
|
* refused refresh rounds alike are counted against `maxAttempts`, under the wall-clock `deadlineMs`
|
|
299
351
|
* and the abort `signal`, with the same jittered backoff policy `Collection.sync` uses between
|
|
300
352
|
* rounds. There is no forward-progress notion here (a commit either lands or it does not), so this
|
|
301
|
-
* simply bounds how many times a losing write is re-driven
|
|
302
|
-
*
|
|
303
|
-
*
|
|
353
|
+
* simply bounds how many times a losing write is re-driven — except for a participant whose view
|
|
354
|
+
* is WEDGED: the confirmed revision each refusal carried is kept per participant
|
|
355
|
+
* (`RevisionStallWatch`, the single-collection accounting's own rule), and a participant whose
|
|
356
|
+
* next attempt is provably identical to the one that just failed ends the commit after
|
|
357
|
+
* `maxStalledAttempts` such strikes ({@link assertNotStalled}) rather than spending the rest of the
|
|
358
|
+
* budget on a decided failure. The same evidence puts a floor on each participant's log-tail read
|
|
359
|
+
* ({@link tailFloor}) and skips the backoff while a member holds the next slot ({@link pause}).
|
|
304
360
|
*/
|
|
305
361
|
class RetryBudget {
|
|
306
362
|
private readonly maxAttempts: number;
|
|
363
|
+
private readonly maxStalledAttempts: number;
|
|
307
364
|
private readonly backoff: { baseMs: number; capMs: number };
|
|
308
365
|
private readonly deadlineMs: number | undefined;
|
|
309
366
|
private readonly signal: AbortSignal | undefined;
|
|
@@ -312,9 +369,14 @@ class RetryBudget {
|
|
|
312
369
|
/** Consecutive clean stale losses: attempts, and refresh rounds refused for a cause that can clear. */
|
|
313
370
|
private losses = 0;
|
|
314
371
|
private lastLoss: CoordinatorStaleLossError | undefined;
|
|
372
|
+
/** One watch per participant of the attempt last checked, created by {@link assertNotStalled}
|
|
373
|
+
* and dropped by {@link recordLoss} for a participant the loss did not name (it committed, or a
|
|
374
|
+
* refresh saved it), so the set is always the participants still to commit. */
|
|
375
|
+
private readonly watches = new Map<CollectionId, RevisionStallWatch>();
|
|
315
376
|
|
|
316
377
|
constructor(options?: SyncOptions) {
|
|
317
378
|
this.maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
|
|
379
|
+
this.maxStalledAttempts = options?.maxStalledAttempts ?? DefaultMaxStalledAttempts;
|
|
318
380
|
this.backoff = {
|
|
319
381
|
baseMs: options?.baseBackoffMs ?? DefaultBaseBackoffMs,
|
|
320
382
|
capMs: options?.maxBackoffMs ?? DefaultMaxBackoffMs,
|
|
@@ -325,7 +387,9 @@ class RetryBudget {
|
|
|
325
387
|
}
|
|
326
388
|
|
|
327
389
|
/** The checks before an attempt: the abort signal, and — a progress-agnostic ceiling, once at
|
|
328
|
-
* least one loss was taken — the wall-clock deadline, reported as that loss.
|
|
390
|
+
* least one loss was taken — the wall-clock deadline, reported as that loss. Runs ahead of
|
|
391
|
+
* {@link assertNotStalled} deliberately: the deadline is the documented outer bound, so a commit
|
|
392
|
+
* that is both past it and stalled reports the deadline, as `SyncOptions.maxStalledAttempts` says. */
|
|
329
393
|
assertOpen(): void {
|
|
330
394
|
if (this.signal?.aborted) {
|
|
331
395
|
throw makeAbortError(this.signal);
|
|
@@ -335,6 +399,34 @@ class RetryBudget {
|
|
|
335
399
|
}
|
|
336
400
|
}
|
|
337
401
|
|
|
402
|
+
/**
|
|
403
|
+
* The stall check before an attempt, per participant in sorted collection-id order
|
|
404
|
+
* (`RevisionStallWatch.check`): a participant whose refresh moved it nowhere while a responder
|
|
405
|
+
* confirmed a revision at or above the one it would request takes a strike, logged as
|
|
406
|
+
* `commit:revision-stalled`; the first to reach `maxStalledAttempts` ends the commit with
|
|
407
|
+
* {@link CoordinatorRevisionStalledError}. Called before EVERY attempt, the first included, so
|
|
408
|
+
* each watch knows the revision the attempt it is judging requested. A participant that joined in
|
|
409
|
+
* a later attempt has no evidence yet and gets no strike.
|
|
410
|
+
*/
|
|
411
|
+
assertNotStalled(actionId: ActionId, participants: readonly NextRevision[]): void {
|
|
412
|
+
const sorted = [...participants].sort((a, b) => compareCollectionIds(a.collectionId, b.collectionId));
|
|
413
|
+
for (const { collectionId, requestedRev, heldRev } of sorted) {
|
|
414
|
+
const verdict = this.watchOf(collectionId).check(requestedRev);
|
|
415
|
+
if (verdict.kind === 'clear') {
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
if (log.enabled) {
|
|
419
|
+
log('commit:revision-stalled tx=%s collection=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d',
|
|
420
|
+
actionId, collectionId, heldRev ?? 'none', requestedRev,
|
|
421
|
+
verdict.staleAt.blockId, verdict.staleAt.rev, verdict.strike, verdict.of);
|
|
422
|
+
}
|
|
423
|
+
if (verdict.kind === 'stalled') {
|
|
424
|
+
throw new CoordinatorRevisionStalledError(collectionId, verdict.staleAt, requestedRev, heldRev,
|
|
425
|
+
this.losses, sorted.map(participant => participant.collectionId), this.lastLoss?.reason);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
338
430
|
/** The aged retry priority for the next attempt, from the losses taken so far, so a
|
|
339
431
|
* repeatedly-losing write out-ranks fresh (priority-0) rivals in the cluster's `resolveRace`.
|
|
340
432
|
* Fairness-only and capped at `MaxPriority`; 0 on the first attempt, so the initial pend
|
|
@@ -343,25 +435,52 @@ class RetryBudget {
|
|
|
343
435
|
return clampPriority(this.losses);
|
|
344
436
|
}
|
|
345
437
|
|
|
346
|
-
/** Count a clean stale loss of an attempt; thrown back when that spends the budget.
|
|
438
|
+
/** Count a clean stale loss of an attempt; thrown back when that spends the budget. Each failed
|
|
439
|
+
* collection's watch notes the refusal with its own confirmed revision — `undefined` for one
|
|
440
|
+
* refused without a confirmed number, or cancelled because a sibling was refused, so it takes no
|
|
441
|
+
* strike that round. A participant the loss did not name committed (the partial-retry branches)
|
|
442
|
+
* or was saved by a refresh, and is done: its watch is dropped. */
|
|
347
443
|
recordLoss(loss: CoordinatorStaleLossError): void {
|
|
348
444
|
this.lastLoss = loss;
|
|
349
445
|
this.losses++;
|
|
446
|
+
const failed = new Set(loss.failedCollections);
|
|
447
|
+
for (const collectionId of [...this.watches.keys()]) {
|
|
448
|
+
if (!failed.has(collectionId)) this.watches.delete(collectionId);
|
|
449
|
+
}
|
|
450
|
+
for (const collectionId of failed) {
|
|
451
|
+
this.watchOf(collectionId).noteRefusal(loss.staleAtByCollection.get(collectionId));
|
|
452
|
+
}
|
|
350
453
|
if (this.losses >= this.maxAttempts) {
|
|
351
454
|
throw loss;
|
|
352
455
|
}
|
|
353
456
|
}
|
|
354
457
|
|
|
355
458
|
/** The backoff before a refresh round: none ahead of a cycle's opening round, there being no
|
|
356
|
-
* race to space out
|
|
357
|
-
*
|
|
459
|
+
* race to space out, and none while a member holds the next slot for a participant still to
|
|
460
|
+
* commit (`RevisionStallWatch.holdsSlot`); otherwise jittered and growing with the losses taken,
|
|
461
|
+
* so a herd of writers that lost the same race does not re-collide on the next tick.
|
|
462
|
+
*
|
|
463
|
+
* NOTE: ANY participant's hold skips the sleep, not every participant's. A member's hold lasts
|
|
464
|
+
* `slotHoldWindowMs` (3 s by default) against a backoff that grows to 5 s, so sleeping would
|
|
465
|
+
* waste a hold that is real now; the herd race another participant may still be in is already
|
|
466
|
+
* handled by the aged priority its pend carries. */
|
|
358
467
|
async pause(): Promise<void> {
|
|
359
468
|
if (this.losses === 0) {
|
|
360
469
|
return;
|
|
361
470
|
}
|
|
471
|
+
for (const watch of this.watches.values()) {
|
|
472
|
+
if (watch.holdsSlot()) return;
|
|
473
|
+
}
|
|
362
474
|
await abortableDelay(jitteredBackoffMs(this.losses - 1, this.backoff, this.rand), this.signal);
|
|
363
475
|
}
|
|
364
476
|
|
|
477
|
+
/** The floor a participant's refresh puts on its log-tail read: the highest revision a refusal of
|
|
478
|
+
* THAT collection confirmed is taken (`RevisionStallWatch.tailFloor`); `undefined` for a
|
|
479
|
+
* non-participant and for a participant none of whose refusals carried one. */
|
|
480
|
+
tailFloor(collectionId: CollectionId): number | undefined {
|
|
481
|
+
return this.watches.get(collectionId)?.tailFloor();
|
|
482
|
+
}
|
|
483
|
+
|
|
365
484
|
/** Whether the next refresh round spends the last of the budget, so a participant whose
|
|
366
485
|
* half-landed write is refused again settles it rather than ask for a round it will not get. */
|
|
367
486
|
lastChance(): boolean {
|
|
@@ -369,14 +488,27 @@ class RetryBudget {
|
|
|
369
488
|
}
|
|
370
489
|
|
|
371
490
|
/** Count a refresh round refused for a cause that can clear (a `completion-refused`
|
|
372
|
-
* {@link TornActionError}); thrown back when that spends the budget or passes the deadline.
|
|
491
|
+
* {@link TornActionError}); thrown back when that spends the budget or passes the deadline. The
|
|
492
|
+
* latest loss then confirms no committed rival for any participant, so the next round sleeps. */
|
|
373
493
|
recordRefusedRound(refused: TornActionError): void {
|
|
374
494
|
this.losses++;
|
|
495
|
+
for (const watch of this.watches.values()) {
|
|
496
|
+
watch.noteRefusedRound();
|
|
497
|
+
}
|
|
375
498
|
if (this.losses >= this.maxAttempts || this.pastDeadline()) {
|
|
376
499
|
throw refused;
|
|
377
500
|
}
|
|
378
501
|
}
|
|
379
502
|
|
|
503
|
+
private watchOf(collectionId: CollectionId): RevisionStallWatch {
|
|
504
|
+
let watch = this.watches.get(collectionId);
|
|
505
|
+
if (watch === undefined) {
|
|
506
|
+
watch = new RevisionStallWatch(this.maxStalledAttempts);
|
|
507
|
+
this.watches.set(collectionId, watch);
|
|
508
|
+
}
|
|
509
|
+
return watch;
|
|
510
|
+
}
|
|
511
|
+
|
|
380
512
|
private pastDeadline(): boolean {
|
|
381
513
|
return this.deadlineMs !== undefined && Date.now() - this.startedAt >= this.deadlineMs;
|
|
382
514
|
}
|
|
@@ -464,6 +596,13 @@ export class TransactionCoordinator {
|
|
|
464
596
|
this.pendValidation = options.pendValidation ?? 'transaction';
|
|
465
597
|
}
|
|
466
598
|
|
|
599
|
+
/** The clock a transaction committed through this coordinator mints its HLC from
|
|
600
|
+
* (`Transaction.hlc`): the transactor's own, or the process-wide fallback (`clockOf`). Exposed so
|
|
601
|
+
* `TransactionSession.commit` mints after its read set is final without holding the transactor. */
|
|
602
|
+
get clock(): HlcClock {
|
|
603
|
+
return clockOf(this.transactor);
|
|
604
|
+
}
|
|
605
|
+
|
|
467
606
|
/**
|
|
468
607
|
* The validation pair every pend of this attempt carries, or `undefined` when this coordinator
|
|
469
608
|
* sends unvalidated pends (see {@link PendValidationMode}). Built once per attempt, AFTER the
|
|
@@ -654,9 +793,10 @@ export class TransactionCoordinator {
|
|
|
654
793
|
inFlightDisposers: [],
|
|
655
794
|
participants: new Set(),
|
|
656
795
|
saved: new Map(),
|
|
657
|
-
inFlight: inFlightCycleOf(options),
|
|
796
|
+
inFlight: inFlightCycleOf(options, transaction.lane),
|
|
658
797
|
};
|
|
659
798
|
const driver: AttemptDriver = {
|
|
799
|
+
actionId: transaction.id as ActionId,
|
|
660
800
|
opensWithRefresh: false,
|
|
661
801
|
attempt: priority => {
|
|
662
802
|
// Age the transaction's advisory priority by the losses taken so far (see
|
|
@@ -669,11 +809,19 @@ export class TransactionCoordinator {
|
|
|
669
809
|
}
|
|
670
810
|
return this.commitOnce(transaction, cycle);
|
|
671
811
|
},
|
|
672
|
-
refresh: lastChance => this.refreshBetweenAttempts(this.registeredRefreshers(), cycle, lastChance),
|
|
812
|
+
refresh: (lastChance, tailFloor) => this.refreshBetweenAttempts(this.registeredRefreshers(), cycle, lastChance, tailFloor),
|
|
673
813
|
// Tested on what is still STAGED, not on "every participant saved": a participant can be
|
|
674
814
|
// saved and still hold actions staged after the attempt began, and those the next attempt
|
|
675
815
|
// must commit.
|
|
676
816
|
nothingLeft: () => this.stagedCollections().length === 0,
|
|
817
|
+
// Read between attempts with no latch held. A reader's concurrent refresh can only RAISE
|
|
818
|
+
// `getNextRev()` (`advanceContext` is monotonic), which the stall check reads as progress,
|
|
819
|
+
// so it can never manufacture a stall; the attempt that follows re-reads under the latch.
|
|
820
|
+
nextRevisions: () => this.stagedCollections().map(({ collectionId, collection }) => ({
|
|
821
|
+
collectionId,
|
|
822
|
+
requestedRev: collection.getNextRev(),
|
|
823
|
+
heldRev: collection.committedRevision(),
|
|
824
|
+
})),
|
|
677
825
|
};
|
|
678
826
|
try {
|
|
679
827
|
const written = await this.commitAttempts(driver, cycle, options);
|
|
@@ -726,6 +874,8 @@ export class TransactionCoordinator {
|
|
|
726
874
|
}
|
|
727
875
|
for (;;) {
|
|
728
876
|
budget.assertOpen();
|
|
877
|
+
// Outside the try below: a stall is the end of the cycle, not a loss to retry.
|
|
878
|
+
budget.assertNotStalled(driver.actionId, driver.nextRevisions());
|
|
729
879
|
try {
|
|
730
880
|
const committed = await driver.attempt(budget.priority());
|
|
731
881
|
// Merged, not overwritten, should one participant be both saved and committed (see the
|
|
@@ -771,7 +921,7 @@ export class TransactionCoordinator {
|
|
|
771
921
|
private async refreshRounds(driver: AttemptDriver, budget: RetryBudget): Promise<void> {
|
|
772
922
|
for (;;) {
|
|
773
923
|
await budget.pause();
|
|
774
|
-
const refused = await driver.refresh(budget.lastChance());
|
|
924
|
+
const refused = await driver.refresh(budget.lastChance(), collectionId => budget.tailFloor(collectionId));
|
|
775
925
|
if (refused === undefined) {
|
|
776
926
|
return;
|
|
777
927
|
}
|
|
@@ -791,7 +941,7 @@ export class TransactionCoordinator {
|
|
|
791
941
|
private registeredRefreshers(): RefreshParticipant[] {
|
|
792
942
|
return [...this.collections].map(([collectionId, collection]) => ({
|
|
793
943
|
collectionId,
|
|
794
|
-
refresh: (report, lastChance) => collection.refreshInFlight(report, lastChance),
|
|
944
|
+
refresh: (report, lastChance, tailFloor) => collection.refreshInFlight(report, lastChance, tailFloor),
|
|
795
945
|
}));
|
|
796
946
|
}
|
|
797
947
|
|
|
@@ -807,24 +957,29 @@ export class TransactionCoordinator {
|
|
|
807
957
|
* prevent — and unreported. Each refresh is independent (its own latch, its own instance state),
|
|
808
958
|
* so one failing says nothing about the next.
|
|
809
959
|
*
|
|
810
|
-
* NOTE: no `tailFloor` is passed to any refresh — the coordinator's loop loses the confirmed
|
|
811
|
-
* revision before the round runs (backlog `debt-multi-collection-retry-cannot-see-the-taken-revision`);
|
|
812
|
-
* the per-entry commit has the same gap, through the same round.
|
|
813
|
-
*
|
|
814
960
|
* @param lastChance - this is the round that spends the last of the retry budget, so a
|
|
815
961
|
* participant whose half-landed write is refused again settles it (`Collection.completeOwnEntry`)
|
|
816
962
|
* rather than asking for a round it will not get; the error returned then carries `final`.
|
|
963
|
+
* @param tailFloor - per collection, the highest revision a refusal of it confirmed is taken
|
|
964
|
+
* (`RetryBudget.tailFloor`), as the floor of its refresh's log-tail read, so a coordinator that has
|
|
965
|
+
* not caught up to the rival is not the only machine asked; `undefined` for a non-participant and
|
|
966
|
+
* for a participant with no confirmed number.
|
|
817
967
|
* @returns the first `completion-refused` {@link TornActionError} when that is the only way any
|
|
818
968
|
* refresh failed — the round is worth retrying; `undefined` when every refresh succeeded.
|
|
819
969
|
* @throws the first other error, once every participant has been visited.
|
|
820
970
|
*/
|
|
821
|
-
private async refreshBetweenAttempts(
|
|
971
|
+
private async refreshBetweenAttempts(
|
|
972
|
+
participants: readonly RefreshParticipant[],
|
|
973
|
+
cycle: CommitCycle,
|
|
974
|
+
lastChance: boolean,
|
|
975
|
+
tailFloor: (collectionId: CollectionId) => number | undefined,
|
|
976
|
+
): Promise<TornActionError | undefined> {
|
|
822
977
|
let terminal: { error: unknown } | undefined;
|
|
823
978
|
let refused: TornActionError | undefined;
|
|
824
979
|
for (const participant of participants) {
|
|
825
980
|
const report: RefreshReport = {};
|
|
826
981
|
try {
|
|
827
|
-
await participant.refresh(report, lastChance);
|
|
982
|
+
await participant.refresh(report, lastChance, tailFloor(participant.collectionId));
|
|
828
983
|
} catch (err) {
|
|
829
984
|
if (err instanceof TornActionError && err.reason === 'completion-refused') {
|
|
830
985
|
refused ??= err;
|
|
@@ -1178,7 +1333,7 @@ export class TransactionCoordinator {
|
|
|
1178
1333
|
recordCollectionDurability(cycle.saved, collectionId, durability);
|
|
1179
1334
|
}
|
|
1180
1335
|
log('commit:partial-retry tx=%s committed=%o failed=%o reason=%s', transaction.id, [...committed.keys()], failed, coordResult.error);
|
|
1181
|
-
throw new CoordinatorStaleLossError(failed, coordResult.error);
|
|
1336
|
+
throw new CoordinatorStaleLossError(failed, coordResult.error, coordResult.staleAt);
|
|
1182
1337
|
}
|
|
1183
1338
|
// Half-landed on a HARD failure (the transport budget spent on a thrown fault, a
|
|
1184
1339
|
// structural rejection): neither cleanly retryable nor cleanly abortable, so drop the
|
|
@@ -1276,7 +1431,7 @@ export class TransactionCoordinator {
|
|
|
1276
1431
|
throw new Error(`commitEntry ${entry.entryId}: a queued entry commits through a coordinator built with `
|
|
1277
1432
|
+ `pendValidation: 'none' — it has no statements for a member to re-execute`);
|
|
1278
1433
|
}
|
|
1279
|
-
const inFlight = inFlightCycleOf(options);
|
|
1434
|
+
const inFlight = inFlightCycleOf(options, entry.lane);
|
|
1280
1435
|
const participants = await EntryParticipants.open(this.collections, entry, inFlight);
|
|
1281
1436
|
const cycle: CommitCycle = {
|
|
1282
1437
|
// The handles own the in-flight marks (closed with them), so none to dispose here.
|
|
@@ -1286,10 +1441,12 @@ export class TransactionCoordinator {
|
|
|
1286
1441
|
inFlight,
|
|
1287
1442
|
};
|
|
1288
1443
|
const driver: AttemptDriver = {
|
|
1444
|
+
actionId: entry.actionId,
|
|
1289
1445
|
opensWithRefresh: true,
|
|
1290
1446
|
attempt: priority => this.commitEntryOnce(entry, participants, cycle, priority),
|
|
1291
|
-
refresh: lastChance => this.refreshBetweenAttempts(participants.refreshers(), cycle, lastChance),
|
|
1447
|
+
refresh: (lastChance, tailFloor) => this.refreshBetweenAttempts(participants.refreshers(), cycle, lastChance, tailFloor),
|
|
1292
1448
|
nothingLeft: () => participants.size === 0,
|
|
1449
|
+
nextRevisions: () => participants.nextRevisions(),
|
|
1293
1450
|
};
|
|
1294
1451
|
try {
|
|
1295
1452
|
const written = await this.commitAttempts(driver, cycle, options);
|
|
@@ -1338,6 +1495,8 @@ export class TransactionCoordinator {
|
|
|
1338
1495
|
const context: WriteContext = {
|
|
1339
1496
|
actionId: entry.actionId,
|
|
1340
1497
|
timestamp: entry.timestamp,
|
|
1498
|
+
hlc: entry.hlc,
|
|
1499
|
+
lane: entry.lane,
|
|
1341
1500
|
reads: [],
|
|
1342
1501
|
priority,
|
|
1343
1502
|
...refuseSoloField(cycle.inFlight.refuseSolo),
|
|
@@ -1369,7 +1528,7 @@ export class TransactionCoordinator {
|
|
|
1369
1528
|
participants.refused(staged, failed);
|
|
1370
1529
|
if (coordResult.staleLoss) {
|
|
1371
1530
|
log('commitEntry:partial-retry entry=%s action=%s committed=%o failed=%o reason=%s', entry.entryId, entry.actionId, [...committed.keys()], failed, coordResult.error);
|
|
1372
|
-
throw new CoordinatorStaleLossError(failed, coordResult.error);
|
|
1531
|
+
throw new CoordinatorStaleLossError(failed, coordResult.error, coordResult.staleAt);
|
|
1373
1532
|
}
|
|
1374
1533
|
throw new CoordinatorPartialCommitError(committed, failed, coordResult.expired ?? coordResult.error);
|
|
1375
1534
|
}
|
|
@@ -1816,7 +1975,7 @@ export class TransactionCoordinator {
|
|
|
1816
1975
|
* Apply actions to a collection.
|
|
1817
1976
|
*
|
|
1818
1977
|
* This runs the action handlers, writes to the log, and collects transforms. The log entry is
|
|
1819
|
-
* written under `context`: its action id, its timestamp and its reads.
|
|
1978
|
+
* written under `context`: its action id, its timestamp, its HLC, its lane and its reads.
|
|
1820
1979
|
*/
|
|
1821
1980
|
private async applyActionsToCollection(
|
|
1822
1981
|
collectionActions: CollectionActions,
|
|
@@ -1856,8 +2015,8 @@ export class TransactionCoordinator {
|
|
|
1856
2015
|
// same revision, and storage keys a saved block revision by `(action id, revision)`. So any
|
|
1857
2016
|
// data block the append mints takes the id an earlier attempt at this revision minted
|
|
1858
2017
|
// (Collection.logAppendBlockIds), and the entry is stamped with the transaction's own
|
|
1859
|
-
// timestamp
|
|
1860
|
-
// the time of the attempt.
|
|
2018
|
+
// timestamp and HLC — the timestamp fixed at BEGIN, the HLC minted once at commit, both
|
|
2019
|
+
// stable across attempts — rather than with the time of the attempt.
|
|
1861
2020
|
//
|
|
1862
2021
|
// NOTE: execute() reaches this method without marking its transaction in flight, so it mints
|
|
1863
2022
|
// afresh every time — correct today, because it re-runs the engine and re-stages, so a
|
|
@@ -1897,7 +2056,7 @@ export class TransactionCoordinator {
|
|
|
1897
2056
|
actionId,
|
|
1898
2057
|
newRev,
|
|
1899
2058
|
() => blockIdsForTransforms(transforms),
|
|
1900
|
-
{ collectionIds: allCollectionIds, reads: [...context.reads], timestamp: context.timestamp }
|
|
2059
|
+
{ collectionIds: allCollectionIds, reads: [...context.reads], timestamp: context.timestamp, hlc: context.hlc, lane: context.lane }
|
|
1901
2060
|
);
|
|
1902
2061
|
|
|
1903
2062
|
// Return the transforms and log tail block ID
|
|
@@ -1991,6 +2150,7 @@ export class TransactionCoordinator {
|
|
|
1991
2150
|
committedCollections: commitResult.committedCollections,
|
|
1992
2151
|
failedCollections: commitResult.failedCollections,
|
|
1993
2152
|
staleLoss: commitResult.staleLoss,
|
|
2153
|
+
staleAt: commitResult.staleAt,
|
|
1994
2154
|
...(commitResult.expired === undefined ? {} : { expired: commitResult.expired }),
|
|
1995
2155
|
};
|
|
1996
2156
|
}
|
|
@@ -2201,7 +2361,7 @@ export class TransactionCoordinator {
|
|
|
2201
2361
|
if (result === undefined) return undefined;
|
|
2202
2362
|
return result.success
|
|
2203
2363
|
? { success: true, pendedBlockIds: blockIds }
|
|
2204
|
-
: pendFailure([pendRefusalError(Array.from(attempts.keys()), result)]);
|
|
2364
|
+
: pendFailure([pendRefusalError(Array.from(attempts.keys()), result, batchStaleAt(result.staleAt, blockIds))]);
|
|
2205
2365
|
}
|
|
2206
2366
|
|
|
2207
2367
|
/**
|
|
@@ -2225,7 +2385,7 @@ export class TransactionCoordinator {
|
|
|
2225
2385
|
this.pendRequestFor(context, validation, attempt, nominees, sole)
|
|
2226
2386
|
);
|
|
2227
2387
|
if (!pendResult.success) {
|
|
2228
|
-
throw pendRefusalError([collectionId], pendResult);
|
|
2388
|
+
throw pendRefusalError([collectionId], pendResult, ownStaleAt(collectionId, pendResult.staleAt));
|
|
2229
2389
|
}
|
|
2230
2390
|
|
|
2231
2391
|
return { collectionId, blockIds: pendResult.blockIds };
|
|
@@ -2254,10 +2414,15 @@ export class TransactionCoordinator {
|
|
|
2254
2414
|
// Collection.sync pend.
|
|
2255
2415
|
// The bases (`PendRequest.baseRevs`) were read from the same tracker the commit's declaration was
|
|
2256
2416
|
// taken from, when the attempt was described; omitted when empty.
|
|
2417
|
+
// The HLC and the lane are the write's (`WriteContext.hlc`, `WriteContext.lane`), the same
|
|
2418
|
+
// values the log entry inside `transforms` carries, so the pend and its validation payload
|
|
2419
|
+
// cannot disagree on them.
|
|
2257
2420
|
return {
|
|
2258
2421
|
actionId: context.actionId,
|
|
2259
2422
|
rev: attempt.rev,
|
|
2260
2423
|
transforms: attempt.transforms,
|
|
2424
|
+
hlc: context.hlc,
|
|
2425
|
+
lane: context.lane,
|
|
2261
2426
|
policy: 'r', // Return policy: fail but return pending actions
|
|
2262
2427
|
...(validation === undefined
|
|
2263
2428
|
? (context.priority > 0 ? { priority: context.priority } : {})
|
|
@@ -2289,6 +2454,8 @@ export class TransactionCoordinator {
|
|
|
2289
2454
|
committedCollections: Map<CollectionId, WriteDurability>;
|
|
2290
2455
|
failedCollections: Set<CollectionId>;
|
|
2291
2456
|
staleLoss?: boolean;
|
|
2457
|
+
/** Per refused collection, the revision its refusal confirmed is taken, where it carried one. */
|
|
2458
|
+
staleAt: ReadonlyMap<CollectionId, ConfirmedRevision>;
|
|
2292
2459
|
expired?: TransactionExpiredError;
|
|
2293
2460
|
}> {
|
|
2294
2461
|
const together = pendedBlockIds.size > 1 && this.transactor.commitMany
|
|
@@ -2308,6 +2475,7 @@ export class TransactionCoordinator {
|
|
|
2308
2475
|
|
|
2309
2476
|
const committedCollections = new Map<CollectionId, WriteDurability>();
|
|
2310
2477
|
const failedCollections = new Set<CollectionId>();
|
|
2478
|
+
const staleAt = new Map<CollectionId, ConfirmedRevision>();
|
|
2311
2479
|
const errors = new Set<string>();
|
|
2312
2480
|
// Classify failures: a returned stale loss (someone committed a newer rev) is retryable after
|
|
2313
2481
|
// a re-read; a thrown/transient-exhausted or structural failure is not. staleLoss holds only
|
|
@@ -2324,6 +2492,7 @@ export class TransactionCoordinator {
|
|
|
2324
2492
|
// A set: a batched refusal answers every collection with the one reason.
|
|
2325
2493
|
errors.add(result.error);
|
|
2326
2494
|
if (result.stale) anyStale = true; else anyHard = true;
|
|
2495
|
+
if (result.staleAt !== undefined) staleAt.set(result.collectionId, result.staleAt);
|
|
2327
2496
|
expired ??= result.expired;
|
|
2328
2497
|
}
|
|
2329
2498
|
}
|
|
@@ -2335,11 +2504,12 @@ export class TransactionCoordinator {
|
|
|
2335
2504
|
committedCollections,
|
|
2336
2505
|
failedCollections,
|
|
2337
2506
|
staleLoss: anyStale && !anyHard,
|
|
2507
|
+
staleAt,
|
|
2338
2508
|
...(expired === undefined ? {} : { expired }),
|
|
2339
2509
|
};
|
|
2340
2510
|
}
|
|
2341
2511
|
|
|
2342
|
-
return { success: true, committedCollections, failedCollections };
|
|
2512
|
+
return { success: true, committedCollections, failedCollections, staleAt };
|
|
2343
2513
|
}
|
|
2344
2514
|
|
|
2345
2515
|
/**
|
|
@@ -2376,9 +2546,11 @@ export class TransactionCoordinator {
|
|
|
2376
2546
|
log('commitTogether: batched commit threw; committing each collection on its own actionId=%s: %o', actionId, e);
|
|
2377
2547
|
return undefined;
|
|
2378
2548
|
}
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2549
|
+
if (result === undefined) return undefined;
|
|
2550
|
+
// The batch's one confirmed revision belongs to the collection owning the block it names, and
|
|
2551
|
+
// to no other collection of the batch (see `batchStaleAt`).
|
|
2552
|
+
const attributed = batchStaleAt(result.success ? undefined : result.staleAt, pendedBlockIds);
|
|
2553
|
+
return requests.map(({ collectionId, request }) => collectionCommitOutcome(collectionId, request, result, attributed.get(collectionId)));
|
|
2382
2554
|
}
|
|
2383
2555
|
|
|
2384
2556
|
/** Commits each collection on its own, concurrently ({@link commitCollection}), one outcome per collection. */
|
|
@@ -2428,7 +2600,9 @@ export class TransactionCoordinator {
|
|
|
2428
2600
|
let lastTransientError: string | undefined;
|
|
2429
2601
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
2430
2602
|
try {
|
|
2431
|
-
|
|
2603
|
+
const result = await this.transactor.commit(commitRequest);
|
|
2604
|
+
// A commit of one collection: its confirmed revision is this collection's own.
|
|
2605
|
+
return collectionCommitOutcome(collectionId, commitRequest, result, result.success ? undefined : result.staleAt);
|
|
2432
2606
|
} catch (e) {
|
|
2433
2607
|
lastTransientError = e instanceof Error ? e.message : String(e);
|
|
2434
2608
|
}
|
|
@@ -2466,6 +2640,7 @@ export class TransactionCoordinator {
|
|
|
2466
2640
|
tailId: attempt.tailId,
|
|
2467
2641
|
rev: attempt.rev,
|
|
2468
2642
|
...blockDigestsField(attempt.blockDigests),
|
|
2643
|
+
lane: context.lane,
|
|
2469
2644
|
...refuseSoloField(context.refuseSolo)
|
|
2470
2645
|
};
|
|
2471
2646
|
}
|