@optimystic/db-core 1.22.0 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/dist/src/collection/action.d.ts +7 -4
- package/dist/src/collection/action.d.ts.map +1 -1
- package/dist/src/collection/action.js +13 -5
- package/dist/src/collection/action.js.map +1 -1
- package/dist/src/collection/collection.d.ts +134 -46
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +207 -70
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/struct.d.ts +38 -8
- package/dist/src/collection/struct.d.ts.map +1 -1
- package/dist/src/collection/struct.js.map +1 -1
- package/dist/src/collection/write-attempts.d.ts +138 -34
- package/dist/src/collection/write-attempts.d.ts.map +1 -1
- package/dist/src/collection/write-attempts.js +168 -101
- package/dist/src/collection/write-attempts.js.map +1 -1
- package/dist/src/collections/diary/diary.d.ts.map +1 -1
- package/dist/src/collections/diary/diary.js +5 -1
- package/dist/src/collections/diary/diary.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts +15 -9
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +6 -4
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/hlc/clock-store.d.ts +63 -0
- package/dist/src/hlc/clock-store.d.ts.map +1 -0
- package/dist/src/hlc/clock-store.js +49 -0
- package/dist/src/hlc/clock-store.js.map +1 -0
- package/dist/src/hlc/clock.d.ts +64 -0
- package/dist/src/hlc/clock.d.ts.map +1 -0
- package/dist/src/hlc/clock.js +171 -0
- package/dist/src/hlc/clock.js.map +1 -0
- package/dist/src/hlc/hlc.d.ts +38 -0
- package/dist/src/hlc/hlc.d.ts.map +1 -0
- package/dist/src/hlc/hlc.js +41 -0
- package/dist/src/hlc/hlc.js.map +1 -0
- package/dist/src/hlc/index.d.ts +5 -0
- package/dist/src/hlc/index.d.ts.map +1 -0
- package/dist/src/hlc/index.js +5 -0
- package/dist/src/hlc/index.js.map +1 -0
- package/dist/src/hlc/skew.d.ts +35 -0
- package/dist/src/hlc/skew.d.ts.map +1 -0
- package/dist/src/hlc/skew.js +35 -0
- package/dist/src/hlc/skew.js.map +1 -0
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/lane/index.d.ts +2 -0
- package/dist/src/lane/index.d.ts.map +1 -0
- package/dist/src/lane/index.js +2 -0
- package/dist/src/lane/index.js.map +1 -0
- package/dist/src/lane/lane.d.ts +46 -0
- package/dist/src/lane/lane.d.ts.map +1 -0
- package/dist/src/lane/lane.js +26 -0
- package/dist/src/lane/lane.js.map +1 -0
- package/dist/src/log/log.d.ts +50 -22
- package/dist/src/log/log.d.ts.map +1 -1
- package/dist/src/log/log.js +21 -13
- package/dist/src/log/log.js.map +1 -1
- package/dist/src/log/struct.d.ts +61 -1
- package/dist/src/log/struct.d.ts.map +1 -1
- package/dist/src/log/struct.js +24 -0
- package/dist/src/log/struct.js.map +1 -1
- package/dist/src/network/i-repo.d.ts +10 -0
- package/dist/src/network/i-repo.d.ts.map +1 -1
- package/dist/src/network/stale-failure.d.ts +20 -7
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +29 -7
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/network/struct.d.ts +35 -0
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/offline/entry.d.ts +14 -0
- package/dist/src/offline/entry.d.ts.map +1 -1
- package/dist/src/offline/entry.js +2 -0
- package/dist/src/offline/entry.js.map +1 -1
- package/dist/src/offline/journal.d.ts +10 -0
- package/dist/src/offline/journal.d.ts.map +1 -1
- package/dist/src/offline/queue.d.ts +10 -0
- package/dist/src/offline/queue.d.ts.map +1 -1
- package/dist/src/offline/queue.js +29 -3
- package/dist/src/offline/queue.js.map +1 -1
- package/dist/src/offline/status.d.ts +7 -0
- package/dist/src/offline/status.d.ts.map +1 -1
- package/dist/src/offline/status.js.map +1 -1
- package/dist/src/testing/test-transactor.d.ts +9 -1
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +6 -1
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/actions-engine.d.ts +3 -0
- package/dist/src/transaction/actions-engine.d.ts.map +1 -1
- package/dist/src/transaction/actions-engine.js +3 -0
- package/dist/src/transaction/actions-engine.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +10 -5
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +197 -48
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transaction/entry-commit.d.ts +45 -10
- package/dist/src/transaction/entry-commit.d.ts.map +1 -1
- package/dist/src/transaction/entry-commit.js +20 -7
- package/dist/src/transaction/entry-commit.js.map +1 -1
- package/dist/src/transaction/errors.d.ts +69 -1
- package/dist/src/transaction/errors.d.ts.map +1 -1
- package/dist/src/transaction/errors.js +70 -1
- package/dist/src/transaction/errors.js.map +1 -1
- package/dist/src/transaction/index.d.ts +1 -1
- package/dist/src/transaction/index.d.ts.map +1 -1
- package/dist/src/transaction/index.js +1 -1
- package/dist/src/transaction/index.js.map +1 -1
- package/dist/src/transaction/session.d.ts.map +1 -1
- package/dist/src/transaction/session.js +18 -5
- package/dist/src/transaction/session.js.map +1 -1
- package/dist/src/transaction/transaction.d.ts +74 -15
- package/dist/src/transaction/transaction.d.ts.map +1 -1
- package/dist/src/transaction/transaction.js +35 -15
- package/dist/src/transaction/transaction.js.map +1 -1
- package/dist/src/transaction/validator.d.ts +25 -4
- package/dist/src/transaction/validator.d.ts.map +1 -1
- package/dist/src/transaction/validator.js +45 -5
- package/dist/src/transaction/validator.js.map +1 -1
- package/dist/src/transactor/block-floors.d.ts +12 -1
- package/dist/src/transactor/block-floors.d.ts.map +1 -1
- package/dist/src/transactor/block-floors.js +14 -1
- package/dist/src/transactor/block-floors.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +8 -0
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +8 -4
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor-source.d.ts +11 -0
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +5 -1
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/dist/src/transactor/transactor.d.ts +8 -0
- package/dist/src/transactor/transactor.d.ts.map +1 -1
- package/dist/src/transform/cache-source.d.ts +7 -0
- package/dist/src/transform/cache-source.d.ts.map +1 -1
- package/dist/src/transform/cache-source.js +14 -18
- package/dist/src/transform/cache-source.js.map +1 -1
- package/package.json +1 -1
- package/src/collection/action.ts +13 -5
- package/src/collection/collection.ts +250 -76
- package/src/collection/struct.ts +38 -8
- package/src/collection/write-attempts.ts +213 -109
- package/src/collections/diary/diary.ts +5 -1
- package/src/collections/tree/tree.ts +18 -11
- package/src/hlc/clock-store.ts +68 -0
- package/src/hlc/clock.ts +196 -0
- package/src/hlc/hlc.ts +56 -0
- package/src/hlc/index.ts +4 -0
- package/src/hlc/skew.ts +55 -0
- package/src/index.ts +2 -0
- package/src/lane/index.ts +1 -0
- package/src/lane/lane.ts +54 -0
- package/src/log/log.ts +341 -310
- package/src/log/struct.ts +62 -1
- package/src/network/i-repo.ts +11 -1
- package/src/network/stale-failure.ts +31 -7
- package/src/network/struct.ts +805 -770
- package/src/offline/entry.ts +16 -0
- package/src/offline/journal.ts +10 -0
- package/src/offline/queue.ts +38 -4
- package/src/offline/status.ts +7 -0
- package/src/testing/test-transactor.ts +1065 -1053
- package/src/transaction/actions-engine.ts +3 -0
- package/src/transaction/coordinator.ts +229 -54
- package/src/transaction/entry-commit.ts +56 -14
- package/src/transaction/errors.ts +60 -0
- package/src/transaction/index.ts +1 -1
- package/src/transaction/session.ts +18 -5
- package/src/transaction/transaction.ts +90 -17
- package/src/transaction/validator.ts +46 -5
- package/src/transactor/block-floors.ts +15 -1
- package/src/transactor/network-transactor.ts +15 -4
- package/src/transactor/transactor-source.ts +15 -1
- package/src/transactor/transactor.ts +9 -0
- package/src/transform/cache-source.ts +15 -17
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"coordinator.d.ts","sourceRoot":"","sources":["../../../src/transaction/coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAW,YAAY,EAAE,UAAU,EAAsH,MAAM,aAAa,CAAC;AAEtM,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"coordinator.d.ts","sourceRoot":"","sources":["../../../src/transaction/coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAW,YAAY,EAAE,UAAU,EAAsH,MAAM,aAAa,CAAC;AAEtM,OAAO,EAAW,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAW5H,OAAO,KAAK,EAAE,UAAU,EAAqC,MAAM,6BAA6B,CAAC;AACjG,OAAO,KAAK,EAAiB,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAMzE,OAAO,KAAK,EAAE,WAAW,EAAkD,MAAM,mBAAmB,CAAC;AAMrG,OAAO,EAAqD,KAAK,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAyehH;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG,MAAM,CAAC;AAExD,MAAM,WAAW,kBAAkB;IAClC,+DAA+D;IAC/D,cAAc,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAED;;;;;;;;;GASG;AACH,qBAAa,sBAAsB;IA4CjC,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW;IA5C7B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2EAiCuE;IACvE,OAAO,CAAC,SAAS,CAEZ;IAEL;8BAC0B;IAC1B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;gBAGlC,UAAU,EAAE,WAAW,EACvB,WAAW,EAAE,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,EAChE,OAAO,GAAE,kBAAuB;IAKjC;;uGAEmG;IACnG,IAAI,KAAK,IAAI,QAAQ,CAEpB;IAED;;;;;;;OAOG;YACW,iBAAiB;IAU/B;;;;;;;;;OASG;IACG,YAAY,CACjB,OAAO,EAAE,iBAAiB,EAAE,EAC5B,OAAO,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;IAwBhB;;wCAEoC;IACpC,OAAO,CAAC,kBAAkB;IAO1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,OAAO,CAAC,iBAAiB;IAOzB;;;;;;;;;OASG;YACW,eAAe;IAiB7B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8CG;IACG,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC;IAwEzG;;;;;;;;;;;;OAYG;YACW,cAAc;IAmC5B;;;;;;;;;;;;;;;;;;;OAmBG;YACW,aAAa;IAW3B;;;;;;;;4EAQwE;IACxE,OAAO,CAAC,oBAAoB;IAO5B;;;;;;;;;;;;;;;;;;;;;;OAsBG;YACW,sBAAsB;IAgCpC;;;;;;;;;6CASyC;IACzC,OAAO,CAAC,iBAAiB;IAUzB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,WAAW;IAsBnB;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;YACW,UAAU;IAgExB;;;;;;;;;;;;OAYG;YACW,iBAAiB;IAqN/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8CG;IACG,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAmC5F;;;;;;;;;;;;;;;;;;;;OAoBG;YACW,eAAe;IAmD7B;;;;;;;;;;;;;;;;;;;OAmBG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAqC9C;;;;;;OAMG;IACH,aAAa,IAAI,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC;IAe9C;;;;;;;;;;;OAWG;IACH,eAAe,IAAI,IAAI;IAMvB;;OAEG;IACH,mBAAmB,IAAI,cAAc,EAAE;IAQvC;;OAEG;IACH,qBAAqB,IAAI,IAAI;IAM7B;;;;;;;;OAQG;IACG,OAAO,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,CAAC;IAwT7F;;;;;OAKG;YACW,wBAAwB;IA4FtC;;;;;;;;;;;;;OAaG;YACW,qBAAqB;IAiBnC;;;;;OAKG;YACW,kBAAkB;IAwDhC;;;;;;;OAOG;YACW,WAAW;IAmCzB;;;;;;;;;;;;;;;OAeG;YACW,oBAAoB;IA0BlC;;;;;;;;;;;;;;;OAeG;YACW,SAAS;IA2DvB;;;;;;;;;;;;;;;;;;;OAmBG;YACW,YAAY;IAsB1B;;;;OAIG;YACW,cAAc;IAsB5B;;;OAGG;IACH,OAAO,CAAC,cAAc;IAuCtB;;;;;;;;OAQG;YACW,WAAW;IAqEzB;;;;;;;;;;;;;OAaG;YACW,cAAc;IA2B5B,+GAA+G;YACjG,UAAU;IAgBxB;;;;;OAKG;YACW,gBAAgB;IAkC9B;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IA8BxB;;;;;OAKG;YACW,WAAW;CAsBzB"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { clockOf } from "../hlc/clock.js";
|
|
2
|
+
import { isConflictFailure, isExpiryFailure, isRejectedFailure, isShortfallFailure, staleAtOwner } from "../network/stale-failure.js";
|
|
2
3
|
import { withBlockRevs } from "../network/durability.js";
|
|
3
4
|
import { TransactionExpiredError } from "../network/transaction-expired-error.js";
|
|
4
5
|
import { PromiseShortfallError } from "../network/promise-shortfall-error.js";
|
|
@@ -6,14 +7,15 @@ import { ValidityRejectedError } from "../network/validity-rejected-error.js";
|
|
|
6
7
|
import { refuseSoloField } from "../network/refuse-solo.js";
|
|
7
8
|
import { TransactionTooLargeError } from "../network/transaction-size.js";
|
|
8
9
|
import { BlockTooLargeError, oversizedBlock } from "../network/block-size.js";
|
|
10
|
+
import { DefaultMaxStalledAttempts, RevisionStallWatch } from "../collection/write-attempts.js";
|
|
9
11
|
import { TornActionError } from "../collection/struct.js";
|
|
10
12
|
import { isTransactionExpired, clampPriority } from "./transaction.js";
|
|
11
|
-
import { EntryParticipants, writeContextOf } from "./entry-commit.js";
|
|
13
|
+
import { EntryParticipants, compareCollectionIds, writeContextOf } from "./entry-commit.js";
|
|
12
14
|
import { Log } from "../log/log.js";
|
|
13
15
|
import { blockIdsForTransforms, copyTransforms } from "../transform/helpers.js";
|
|
14
16
|
import { describeTouchedBlocks, blockDigestsField, baseRevsField } from "../transform/digest.js";
|
|
15
17
|
import { collectOperations, hashOperations } from "./operations-hash.js";
|
|
16
|
-
import { CoordinatorConcurrentStampError, CoordinatorPartialCommitError, CoordinatorStaleLossError } from "./errors.js";
|
|
18
|
+
import { CoordinatorConcurrentStampError, CoordinatorPartialCommitError, CoordinatorRevisionStalledError, CoordinatorStaleLossError } from "./errors.js";
|
|
17
19
|
import { recordCollectionDurability, transactionDurability } from "./durability.js";
|
|
18
20
|
import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
|
|
19
21
|
import { visitEveryThenThrow } from "../utility/visit-every.js";
|
|
@@ -35,31 +37,56 @@ const DefaultMaxBackoffMs = 5000;
|
|
|
35
37
|
*/
|
|
36
38
|
class PendRejectedError extends Error {
|
|
37
39
|
conflict;
|
|
40
|
+
staleAtByCollection;
|
|
38
41
|
constructor(collectionIds, conflict, reason,
|
|
39
42
|
/** Confirmed revision the responder holds, from `StaleFailure.staleAt`. Folded into the
|
|
40
43
|
* message because pendPhase collapses this error to its `.message` string, which is the only
|
|
41
44
|
* form that reaches an embedder through the transaction result's `error` field. */
|
|
42
|
-
staleAt
|
|
45
|
+
staleAt,
|
|
46
|
+
/** The same number attributed to the collection whose counter it belongs to (see
|
|
47
|
+
* {@link ownStaleAt} and {@link batchStaleAt}), for the retry loop's per-participant stall
|
|
48
|
+
* check; empty when the refusal carried none or named a block no collection of it sent. */
|
|
49
|
+
staleAtByCollection = NoStaleAt) {
|
|
43
50
|
super(`Pend failed for ${collectionIds.length === 1 ? 'collection' : 'collections'} ${collectionIds.join(', ')}: ${reason ?? (conflict ? 'stale conflict' : 'rejected')}`
|
|
44
51
|
+ (staleAt ? ` (block ${staleAt.blockId} at rev ${staleAt.rev})` : ''));
|
|
45
52
|
this.conflict = conflict;
|
|
53
|
+
this.staleAtByCollection = staleAtByCollection;
|
|
46
54
|
this.name = 'PendRejectedError';
|
|
47
55
|
}
|
|
48
56
|
}
|
|
57
|
+
/** The attribution of a refusal that carried no confirmed number, or one no collection owns. */
|
|
58
|
+
const NoStaleAt = new Map();
|
|
59
|
+
/** A refusal's confirmed revision as the ONE collection's own: a request for one collection
|
|
60
|
+
* (`pendCollection`, `commitCollection`) gets a number under that collection's counter whatever block
|
|
61
|
+
* it names, so it is attributed without looking at the block id — as the single-collection loop does. */
|
|
62
|
+
function ownStaleAt(collectionId, staleAt) {
|
|
63
|
+
return staleAt === undefined ? NoStaleAt : new Map([[collectionId, staleAt]]);
|
|
64
|
+
}
|
|
65
|
+
/** A batched answer's confirmed revision (`pendTogether`, `commitTogether`), attributed to the one
|
|
66
|
+
* collection whose blocks include the block it names (`staleAtOwner`): the batch carries ONE number,
|
|
67
|
+
* the highest across its collections' counters, and every other collection of the batch was refused
|
|
68
|
+
* with no confirmed number of its own. Nothing is attributed when no collection of the batch owns the
|
|
69
|
+
* block. */
|
|
70
|
+
function batchStaleAt(staleAt, blocksByCollection) {
|
|
71
|
+
const owner = staleAtOwner(staleAt, blocksByCollection);
|
|
72
|
+
return owner === undefined || staleAt === undefined ? NoStaleAt : new Map([[owner, staleAt]]);
|
|
73
|
+
}
|
|
49
74
|
/** The error a refused pend of `collectionIds` stands for: its validity rejection — never retried, since
|
|
50
75
|
* a re-drive sends the same content to the same checks — its expiry — never retried, since a re-drive's
|
|
51
76
|
* expiration comes from the same clock — or its promise shortfall — never retried, since a re-drive
|
|
52
|
-
* meets the same silent members — or a rejection carrying whether a re-read can clear it
|
|
77
|
+
* meets the same silent members — or a rejection carrying whether a re-read can clear it, with its
|
|
78
|
+
* confirmed revision attributed per collection (`staleAtByCollection`, which the caller works out:
|
|
79
|
+
* {@link ownStaleAt} for one collection's pend, {@link batchStaleAt} for a batched one).
|
|
53
80
|
* Retryability comes from the response itself: a producer that classified the failure sets `conflict`,
|
|
54
81
|
* and only where none did is it inferred from `missing`/`pending` (`isConflictFailure`). */
|
|
55
|
-
function pendRefusalError(collectionIds, refusal) {
|
|
82
|
+
function pendRefusalError(collectionIds, refusal, staleAtByCollection) {
|
|
56
83
|
if (isRejectedFailure(refusal))
|
|
57
84
|
return new ValidityRejectedError(refusal.rejected);
|
|
58
85
|
if (isExpiryFailure(refusal))
|
|
59
86
|
return new TransactionExpiredError(refusal.expired);
|
|
60
87
|
if (isShortfallFailure(refusal))
|
|
61
88
|
return new PromiseShortfallError(refusal.shortfall);
|
|
62
|
-
return new PendRejectedError(collectionIds, isConflictFailure(refusal), refusal.reason, refusal.staleAt);
|
|
89
|
+
return new PendRejectedError(collectionIds, isConflictFailure(refusal), refusal.reason, refusal.staleAt, staleAtByCollection);
|
|
63
90
|
}
|
|
64
91
|
/** A head commit's staged attempt as the phases send it: its transforms are the attempt tracker's. */
|
|
65
92
|
function collectionAttemptOf(staged) {
|
|
@@ -82,31 +109,39 @@ function collectionAttemptOf(staged) {
|
|
|
82
109
|
*/
|
|
83
110
|
function pendFailure(reasons) {
|
|
84
111
|
const first = reasons[0];
|
|
85
|
-
const conflicts = reasons.filter(reason => reason instanceof PendRejectedError && reason.conflict);
|
|
112
|
+
const conflicts = reasons.filter((reason) => reason instanceof PendRejectedError && reason.conflict);
|
|
86
113
|
const rejected = reasons.find((reason) => reason instanceof ValidityRejectedError);
|
|
87
114
|
const expired = reasons.find((reason) => reason instanceof TransactionExpiredError);
|
|
88
115
|
const shortfall = reasons.find((reason) => reason instanceof PromiseShortfallError);
|
|
89
116
|
const tooLarge = reasons.find((reason) => reason instanceof TransactionTooLargeError || reason instanceof BlockTooLargeError);
|
|
117
|
+
// Each refusal attributed its number to its own collection(s), and no collection is refused twice
|
|
118
|
+
// in one phase, so the maps are disjoint and merge without a rule.
|
|
119
|
+
const staleAt = new Map(conflicts.flatMap(conflict => [...conflict.staleAtByCollection]));
|
|
90
120
|
return {
|
|
91
121
|
success: false,
|
|
92
122
|
error: first instanceof Error ? first.message : String(first),
|
|
93
123
|
staleLoss: conflicts.length > 0 && conflicts.length === reasons.length,
|
|
124
|
+
staleAt,
|
|
94
125
|
...(rejected === undefined ? {} : { rejected }),
|
|
95
126
|
...(expired === undefined ? {} : { expired }),
|
|
96
127
|
...(shortfall === undefined ? {} : { shortfall }),
|
|
97
128
|
...(tooLarge === undefined ? {} : { tooLarge }),
|
|
98
129
|
};
|
|
99
130
|
}
|
|
100
|
-
/** What a commit's
|
|
101
|
-
|
|
102
|
-
|
|
131
|
+
/** What every send of a commit's cycle carries: the write's lane — the transaction's or the queued
|
|
132
|
+
* entry's, never `options.lane`, which the transaction and the entry already folded in — and what the
|
|
133
|
+
* commit's `options` ask of every send besides. */
|
|
134
|
+
function inFlightCycleOf(options, lane) {
|
|
135
|
+
return { lane, ...refuseSoloField(options?.refuseSoloCommit) };
|
|
103
136
|
}
|
|
104
137
|
/**
|
|
105
138
|
* How `collectionId`'s commit ended when `result` answered its `request` — the answer to its own
|
|
106
139
|
* commit, or, in a batched commit, the batch's one answer for every collection of it. A success is
|
|
107
|
-
* stamped with the collection's own blocks at its own revision (`withBlockRevs`).
|
|
140
|
+
* stamped with the collection's own blocks at its own revision (`withBlockRevs`). `staleAt` is the
|
|
141
|
+
* answer's confirmed revision as far as it is THIS collection's: the answer's own for a commit of one
|
|
142
|
+
* collection, the owner's share of a batched answer (`batchStaleAt`), and `undefined` otherwise.
|
|
108
143
|
*/
|
|
109
|
-
function collectionCommitOutcome(collectionId, request, result) {
|
|
144
|
+
function collectionCommitOutcome(collectionId, request, result, staleAt) {
|
|
110
145
|
if (result.success) {
|
|
111
146
|
return { collectionId, committed: true, durability: withBlockRevs(result.durability, request.blockIds, request.rev) };
|
|
112
147
|
}
|
|
@@ -133,7 +168,8 @@ function collectionCommitOutcome(collectionId, request, result) {
|
|
|
133
168
|
collectionId,
|
|
134
169
|
committed: false,
|
|
135
170
|
stale: true,
|
|
136
|
-
error: result.reason ?? `Stale commit for collection ${collectionId}
|
|
171
|
+
error: result.reason ?? `Stale commit for collection ${collectionId}`,
|
|
172
|
+
...(staleAt === undefined ? {} : { staleAt }),
|
|
137
173
|
};
|
|
138
174
|
}
|
|
139
175
|
/** What a successful commit wrote: `written` folded into one report, or `undefined` when it holds
|
|
@@ -153,7 +189,7 @@ function writtenDurability(written) {
|
|
|
153
189
|
*/
|
|
154
190
|
function cleanFailureError(result, failed) {
|
|
155
191
|
if (result.staleLoss) {
|
|
156
|
-
return new CoordinatorStaleLossError(failed, result.error);
|
|
192
|
+
return new CoordinatorStaleLossError(failed, result.error, result.staleAt);
|
|
157
193
|
}
|
|
158
194
|
return result.rejected ?? result.expired ?? result.shortfall ?? result.tooLarge
|
|
159
195
|
?? new Error(`Transaction commit failed: ${result.error}`);
|
|
@@ -163,12 +199,17 @@ function cleanFailureError(result, failed) {
|
|
|
163
199
|
* refused refresh rounds alike are counted against `maxAttempts`, under the wall-clock `deadlineMs`
|
|
164
200
|
* and the abort `signal`, with the same jittered backoff policy `Collection.sync` uses between
|
|
165
201
|
* rounds. There is no forward-progress notion here (a commit either lands or it does not), so this
|
|
166
|
-
* simply bounds how many times a losing write is re-driven
|
|
167
|
-
*
|
|
168
|
-
*
|
|
202
|
+
* simply bounds how many times a losing write is re-driven — except for a participant whose view
|
|
203
|
+
* is WEDGED: the confirmed revision each refusal carried is kept per participant
|
|
204
|
+
* (`RevisionStallWatch`, the single-collection accounting's own rule), and a participant whose
|
|
205
|
+
* next attempt is provably identical to the one that just failed ends the commit after
|
|
206
|
+
* `maxStalledAttempts` such strikes ({@link assertNotStalled}) rather than spending the rest of the
|
|
207
|
+
* budget on a decided failure. The same evidence puts a floor on each participant's log-tail read
|
|
208
|
+
* ({@link tailFloor}) and skips the backoff while a member holds the next slot ({@link pause}).
|
|
169
209
|
*/
|
|
170
210
|
class RetryBudget {
|
|
171
211
|
maxAttempts;
|
|
212
|
+
maxStalledAttempts;
|
|
172
213
|
backoff;
|
|
173
214
|
deadlineMs;
|
|
174
215
|
signal;
|
|
@@ -177,8 +218,13 @@ class RetryBudget {
|
|
|
177
218
|
/** Consecutive clean stale losses: attempts, and refresh rounds refused for a cause that can clear. */
|
|
178
219
|
losses = 0;
|
|
179
220
|
lastLoss;
|
|
221
|
+
/** One watch per participant of the attempt last checked, created by {@link assertNotStalled}
|
|
222
|
+
* and dropped by {@link recordLoss} for a participant the loss did not name (it committed, or a
|
|
223
|
+
* refresh saved it), so the set is always the participants still to commit. */
|
|
224
|
+
watches = new Map();
|
|
180
225
|
constructor(options) {
|
|
181
226
|
this.maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
|
|
227
|
+
this.maxStalledAttempts = options?.maxStalledAttempts ?? DefaultMaxStalledAttempts;
|
|
182
228
|
this.backoff = {
|
|
183
229
|
baseMs: options?.baseBackoffMs ?? DefaultBaseBackoffMs,
|
|
184
230
|
capMs: options?.maxBackoffMs ?? DefaultMaxBackoffMs,
|
|
@@ -188,7 +234,9 @@ class RetryBudget {
|
|
|
188
234
|
this.rand = options?.rand;
|
|
189
235
|
}
|
|
190
236
|
/** The checks before an attempt: the abort signal, and — a progress-agnostic ceiling, once at
|
|
191
|
-
* least one loss was taken — the wall-clock deadline, reported as that loss.
|
|
237
|
+
* least one loss was taken — the wall-clock deadline, reported as that loss. Runs ahead of
|
|
238
|
+
* {@link assertNotStalled} deliberately: the deadline is the documented outer bound, so a commit
|
|
239
|
+
* that is both past it and stalled reports the deadline, as `SyncOptions.maxStalledAttempts` says. */
|
|
192
240
|
assertOpen() {
|
|
193
241
|
if (this.signal?.aborted) {
|
|
194
242
|
throw makeAbortError(this.signal);
|
|
@@ -197,6 +245,30 @@ class RetryBudget {
|
|
|
197
245
|
throw this.lastLoss;
|
|
198
246
|
}
|
|
199
247
|
}
|
|
248
|
+
/**
|
|
249
|
+
* The stall check before an attempt, per participant in sorted collection-id order
|
|
250
|
+
* (`RevisionStallWatch.check`): a participant whose refresh moved it nowhere while a responder
|
|
251
|
+
* confirmed a revision at or above the one it would request takes a strike, logged as
|
|
252
|
+
* `commit:revision-stalled`; the first to reach `maxStalledAttempts` ends the commit with
|
|
253
|
+
* {@link CoordinatorRevisionStalledError}. Called before EVERY attempt, the first included, so
|
|
254
|
+
* each watch knows the revision the attempt it is judging requested. A participant that joined in
|
|
255
|
+
* a later attempt has no evidence yet and gets no strike.
|
|
256
|
+
*/
|
|
257
|
+
assertNotStalled(actionId, participants) {
|
|
258
|
+
const sorted = [...participants].sort((a, b) => compareCollectionIds(a.collectionId, b.collectionId));
|
|
259
|
+
for (const { collectionId, requestedRev, heldRev } of sorted) {
|
|
260
|
+
const verdict = this.watchOf(collectionId).check(requestedRev);
|
|
261
|
+
if (verdict.kind === 'clear') {
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
if (log.enabled) {
|
|
265
|
+
log('commit:revision-stalled tx=%s collection=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d', actionId, collectionId, heldRev ?? 'none', requestedRev, verdict.staleAt.blockId, verdict.staleAt.rev, verdict.strike, verdict.of);
|
|
266
|
+
}
|
|
267
|
+
if (verdict.kind === 'stalled') {
|
|
268
|
+
throw new CoordinatorRevisionStalledError(collectionId, verdict.staleAt, requestedRev, heldRev, this.losses, sorted.map(participant => participant.collectionId), this.lastLoss?.reason);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
200
272
|
/** The aged retry priority for the next attempt, from the losses taken so far, so a
|
|
201
273
|
* repeatedly-losing write out-ranks fresh (priority-0) rivals in the cluster's `resolveRace`.
|
|
202
274
|
* Fairness-only and capped at `MaxPriority`; 0 on the first attempt, so the initial pend
|
|
@@ -204,36 +276,76 @@ class RetryBudget {
|
|
|
204
276
|
priority() {
|
|
205
277
|
return clampPriority(this.losses);
|
|
206
278
|
}
|
|
207
|
-
/** Count a clean stale loss of an attempt; thrown back when that spends the budget.
|
|
279
|
+
/** Count a clean stale loss of an attempt; thrown back when that spends the budget. Each failed
|
|
280
|
+
* collection's watch notes the refusal with its own confirmed revision — `undefined` for one
|
|
281
|
+
* refused without a confirmed number, or cancelled because a sibling was refused, so it takes no
|
|
282
|
+
* strike that round. A participant the loss did not name committed (the partial-retry branches)
|
|
283
|
+
* or was saved by a refresh, and is done: its watch is dropped. */
|
|
208
284
|
recordLoss(loss) {
|
|
209
285
|
this.lastLoss = loss;
|
|
210
286
|
this.losses++;
|
|
287
|
+
const failed = new Set(loss.failedCollections);
|
|
288
|
+
for (const collectionId of [...this.watches.keys()]) {
|
|
289
|
+
if (!failed.has(collectionId))
|
|
290
|
+
this.watches.delete(collectionId);
|
|
291
|
+
}
|
|
292
|
+
for (const collectionId of failed) {
|
|
293
|
+
this.watchOf(collectionId).noteRefusal(loss.staleAtByCollection.get(collectionId));
|
|
294
|
+
}
|
|
211
295
|
if (this.losses >= this.maxAttempts) {
|
|
212
296
|
throw loss;
|
|
213
297
|
}
|
|
214
298
|
}
|
|
215
299
|
/** The backoff before a refresh round: none ahead of a cycle's opening round, there being no
|
|
216
|
-
* race to space out
|
|
217
|
-
*
|
|
300
|
+
* race to space out, and none while a member holds the next slot for a participant still to
|
|
301
|
+
* commit (`RevisionStallWatch.holdsSlot`); otherwise jittered and growing with the losses taken,
|
|
302
|
+
* so a herd of writers that lost the same race does not re-collide on the next tick.
|
|
303
|
+
*
|
|
304
|
+
* NOTE: ANY participant's hold skips the sleep, not every participant's. A member's hold lasts
|
|
305
|
+
* `slotHoldWindowMs` (3 s by default) against a backoff that grows to 5 s, so sleeping would
|
|
306
|
+
* waste a hold that is real now; the herd race another participant may still be in is already
|
|
307
|
+
* handled by the aged priority its pend carries. */
|
|
218
308
|
async pause() {
|
|
219
309
|
if (this.losses === 0) {
|
|
220
310
|
return;
|
|
221
311
|
}
|
|
312
|
+
for (const watch of this.watches.values()) {
|
|
313
|
+
if (watch.holdsSlot())
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
222
316
|
await abortableDelay(jitteredBackoffMs(this.losses - 1, this.backoff, this.rand), this.signal);
|
|
223
317
|
}
|
|
318
|
+
/** The floor a participant's refresh puts on its log-tail read: the highest revision a refusal of
|
|
319
|
+
* THAT collection confirmed is taken (`RevisionStallWatch.tailFloor`); `undefined` for a
|
|
320
|
+
* non-participant and for a participant none of whose refusals carried one. */
|
|
321
|
+
tailFloor(collectionId) {
|
|
322
|
+
return this.watches.get(collectionId)?.tailFloor();
|
|
323
|
+
}
|
|
224
324
|
/** Whether the next refresh round spends the last of the budget, so a participant whose
|
|
225
325
|
* half-landed write is refused again settles it rather than ask for a round it will not get. */
|
|
226
326
|
lastChance() {
|
|
227
327
|
return this.losses + 1 >= this.maxAttempts;
|
|
228
328
|
}
|
|
229
329
|
/** Count a refresh round refused for a cause that can clear (a `completion-refused`
|
|
230
|
-
* {@link TornActionError}); thrown back when that spends the budget or passes the deadline.
|
|
330
|
+
* {@link TornActionError}); thrown back when that spends the budget or passes the deadline. The
|
|
331
|
+
* latest loss then confirms no committed rival for any participant, so the next round sleeps. */
|
|
231
332
|
recordRefusedRound(refused) {
|
|
232
333
|
this.losses++;
|
|
334
|
+
for (const watch of this.watches.values()) {
|
|
335
|
+
watch.noteRefusedRound();
|
|
336
|
+
}
|
|
233
337
|
if (this.losses >= this.maxAttempts || this.pastDeadline()) {
|
|
234
338
|
throw refused;
|
|
235
339
|
}
|
|
236
340
|
}
|
|
341
|
+
watchOf(collectionId) {
|
|
342
|
+
let watch = this.watches.get(collectionId);
|
|
343
|
+
if (watch === undefined) {
|
|
344
|
+
watch = new RevisionStallWatch(this.maxStalledAttempts);
|
|
345
|
+
this.watches.set(collectionId, watch);
|
|
346
|
+
}
|
|
347
|
+
return watch;
|
|
348
|
+
}
|
|
237
349
|
pastDeadline() {
|
|
238
350
|
return this.deadlineMs !== undefined && Date.now() - this.startedAt >= this.deadlineMs;
|
|
239
351
|
}
|
|
@@ -294,6 +406,12 @@ export class TransactionCoordinator {
|
|
|
294
406
|
this.collections = collections;
|
|
295
407
|
this.pendValidation = options.pendValidation ?? 'transaction';
|
|
296
408
|
}
|
|
409
|
+
/** The clock a transaction committed through this coordinator mints its HLC from
|
|
410
|
+
* (`Transaction.hlc`): the transactor's own, or the process-wide fallback (`clockOf`). Exposed so
|
|
411
|
+
* `TransactionSession.commit` mints after its read set is final without holding the transactor. */
|
|
412
|
+
get clock() {
|
|
413
|
+
return clockOf(this.transactor);
|
|
414
|
+
}
|
|
297
415
|
/**
|
|
298
416
|
* The validation pair every pend of this attempt carries, or `undefined` when this coordinator
|
|
299
417
|
* sends unvalidated pends (see {@link PendValidationMode}). Built once per attempt, AFTER the
|
|
@@ -468,9 +586,10 @@ export class TransactionCoordinator {
|
|
|
468
586
|
inFlightDisposers: [],
|
|
469
587
|
participants: new Set(),
|
|
470
588
|
saved: new Map(),
|
|
471
|
-
inFlight: inFlightCycleOf(options),
|
|
589
|
+
inFlight: inFlightCycleOf(options, transaction.lane),
|
|
472
590
|
};
|
|
473
591
|
const driver = {
|
|
592
|
+
actionId: transaction.id,
|
|
474
593
|
opensWithRefresh: false,
|
|
475
594
|
attempt: priority => {
|
|
476
595
|
// Age the transaction's advisory priority by the losses taken so far (see
|
|
@@ -483,11 +602,19 @@ export class TransactionCoordinator {
|
|
|
483
602
|
}
|
|
484
603
|
return this.commitOnce(transaction, cycle);
|
|
485
604
|
},
|
|
486
|
-
refresh: lastChance => this.refreshBetweenAttempts(this.registeredRefreshers(), cycle, lastChance),
|
|
605
|
+
refresh: (lastChance, tailFloor) => this.refreshBetweenAttempts(this.registeredRefreshers(), cycle, lastChance, tailFloor),
|
|
487
606
|
// Tested on what is still STAGED, not on "every participant saved": a participant can be
|
|
488
607
|
// saved and still hold actions staged after the attempt began, and those the next attempt
|
|
489
608
|
// must commit.
|
|
490
609
|
nothingLeft: () => this.stagedCollections().length === 0,
|
|
610
|
+
// Read between attempts with no latch held. A reader's concurrent refresh can only RAISE
|
|
611
|
+
// `getNextRev()` (`advanceContext` is monotonic), which the stall check reads as progress,
|
|
612
|
+
// so it can never manufacture a stall; the attempt that follows re-reads under the latch.
|
|
613
|
+
nextRevisions: () => this.stagedCollections().map(({ collectionId, collection }) => ({
|
|
614
|
+
collectionId,
|
|
615
|
+
requestedRev: collection.getNextRev(),
|
|
616
|
+
heldRev: collection.committedRevision(),
|
|
617
|
+
})),
|
|
491
618
|
};
|
|
492
619
|
try {
|
|
493
620
|
const written = await this.commitAttempts(driver, cycle, options);
|
|
@@ -541,6 +668,8 @@ export class TransactionCoordinator {
|
|
|
541
668
|
}
|
|
542
669
|
for (;;) {
|
|
543
670
|
budget.assertOpen();
|
|
671
|
+
// Outside the try below: a stall is the end of the cycle, not a loss to retry.
|
|
672
|
+
budget.assertNotStalled(driver.actionId, driver.nextRevisions());
|
|
544
673
|
try {
|
|
545
674
|
const committed = await driver.attempt(budget.priority());
|
|
546
675
|
// Merged, not overwritten, should one participant be both saved and committed (see the
|
|
@@ -586,7 +715,7 @@ export class TransactionCoordinator {
|
|
|
586
715
|
async refreshRounds(driver, budget) {
|
|
587
716
|
for (;;) {
|
|
588
717
|
await budget.pause();
|
|
589
|
-
const refused = await driver.refresh(budget.lastChance());
|
|
718
|
+
const refused = await driver.refresh(budget.lastChance(), collectionId => budget.tailFloor(collectionId));
|
|
590
719
|
if (refused === undefined) {
|
|
591
720
|
return;
|
|
592
721
|
}
|
|
@@ -605,7 +734,7 @@ export class TransactionCoordinator {
|
|
|
605
734
|
registeredRefreshers() {
|
|
606
735
|
return [...this.collections].map(([collectionId, collection]) => ({
|
|
607
736
|
collectionId,
|
|
608
|
-
refresh: (report, lastChance) => collection.refreshInFlight(report, lastChance),
|
|
737
|
+
refresh: (report, lastChance, tailFloor) => collection.refreshInFlight(report, lastChance, tailFloor),
|
|
609
738
|
}));
|
|
610
739
|
}
|
|
611
740
|
/**
|
|
@@ -620,24 +749,24 @@ export class TransactionCoordinator {
|
|
|
620
749
|
* prevent — and unreported. Each refresh is independent (its own latch, its own instance state),
|
|
621
750
|
* so one failing says nothing about the next.
|
|
622
751
|
*
|
|
623
|
-
* NOTE: no `tailFloor` is passed to any refresh — the coordinator's loop loses the confirmed
|
|
624
|
-
* revision before the round runs (backlog `debt-multi-collection-retry-cannot-see-the-taken-revision`);
|
|
625
|
-
* the per-entry commit has the same gap, through the same round.
|
|
626
|
-
*
|
|
627
752
|
* @param lastChance - this is the round that spends the last of the retry budget, so a
|
|
628
753
|
* participant whose half-landed write is refused again settles it (`Collection.completeOwnEntry`)
|
|
629
754
|
* rather than asking for a round it will not get; the error returned then carries `final`.
|
|
755
|
+
* @param tailFloor - per collection, the highest revision a refusal of it confirmed is taken
|
|
756
|
+
* (`RetryBudget.tailFloor`), as the floor of its refresh's log-tail read, so a coordinator that has
|
|
757
|
+
* not caught up to the rival is not the only machine asked; `undefined` for a non-participant and
|
|
758
|
+
* for a participant with no confirmed number.
|
|
630
759
|
* @returns the first `completion-refused` {@link TornActionError} when that is the only way any
|
|
631
760
|
* refresh failed — the round is worth retrying; `undefined` when every refresh succeeded.
|
|
632
761
|
* @throws the first other error, once every participant has been visited.
|
|
633
762
|
*/
|
|
634
|
-
async refreshBetweenAttempts(participants, cycle, lastChance) {
|
|
763
|
+
async refreshBetweenAttempts(participants, cycle, lastChance, tailFloor) {
|
|
635
764
|
let terminal;
|
|
636
765
|
let refused;
|
|
637
766
|
for (const participant of participants) {
|
|
638
767
|
const report = {};
|
|
639
768
|
try {
|
|
640
|
-
await participant.refresh(report, lastChance);
|
|
769
|
+
await participant.refresh(report, lastChance, tailFloor(participant.collectionId));
|
|
641
770
|
}
|
|
642
771
|
catch (err) {
|
|
643
772
|
if (err instanceof TornActionError && err.reason === 'completion-refused') {
|
|
@@ -969,7 +1098,7 @@ export class TransactionCoordinator {
|
|
|
969
1098
|
recordCollectionDurability(cycle.saved, collectionId, durability);
|
|
970
1099
|
}
|
|
971
1100
|
log('commit:partial-retry tx=%s committed=%o failed=%o reason=%s', transaction.id, [...committed.keys()], failed, coordResult.error);
|
|
972
|
-
throw new CoordinatorStaleLossError(failed, coordResult.error);
|
|
1101
|
+
throw new CoordinatorStaleLossError(failed, coordResult.error, coordResult.staleAt);
|
|
973
1102
|
}
|
|
974
1103
|
// Half-landed on a HARD failure (the transport budget spent on a thrown fault, a
|
|
975
1104
|
// structural rejection): neither cleanly retryable nor cleanly abortable, so drop the
|
|
@@ -1063,7 +1192,7 @@ export class TransactionCoordinator {
|
|
|
1063
1192
|
throw new Error(`commitEntry ${entry.entryId}: a queued entry commits through a coordinator built with `
|
|
1064
1193
|
+ `pendValidation: 'none' — it has no statements for a member to re-execute`);
|
|
1065
1194
|
}
|
|
1066
|
-
const inFlight = inFlightCycleOf(options);
|
|
1195
|
+
const inFlight = inFlightCycleOf(options, entry.lane);
|
|
1067
1196
|
const participants = await EntryParticipants.open(this.collections, entry, inFlight);
|
|
1068
1197
|
const cycle = {
|
|
1069
1198
|
// The handles own the in-flight marks (closed with them), so none to dispose here.
|
|
@@ -1073,10 +1202,12 @@ export class TransactionCoordinator {
|
|
|
1073
1202
|
inFlight,
|
|
1074
1203
|
};
|
|
1075
1204
|
const driver = {
|
|
1205
|
+
actionId: entry.actionId,
|
|
1076
1206
|
opensWithRefresh: true,
|
|
1077
1207
|
attempt: priority => this.commitEntryOnce(entry, participants, cycle, priority),
|
|
1078
|
-
refresh: lastChance => this.refreshBetweenAttempts(participants.refreshers(), cycle, lastChance),
|
|
1208
|
+
refresh: (lastChance, tailFloor) => this.refreshBetweenAttempts(participants.refreshers(), cycle, lastChance, tailFloor),
|
|
1079
1209
|
nothingLeft: () => participants.size === 0,
|
|
1210
|
+
nextRevisions: () => participants.nextRevisions(),
|
|
1080
1211
|
};
|
|
1081
1212
|
try {
|
|
1082
1213
|
const written = await this.commitAttempts(driver, cycle, options);
|
|
@@ -1121,6 +1252,8 @@ export class TransactionCoordinator {
|
|
|
1121
1252
|
const context = {
|
|
1122
1253
|
actionId: entry.actionId,
|
|
1123
1254
|
timestamp: entry.timestamp,
|
|
1255
|
+
hlc: entry.hlc,
|
|
1256
|
+
lane: entry.lane,
|
|
1124
1257
|
reads: [],
|
|
1125
1258
|
priority,
|
|
1126
1259
|
...refuseSoloField(cycle.inFlight.refuseSolo),
|
|
@@ -1151,7 +1284,7 @@ export class TransactionCoordinator {
|
|
|
1151
1284
|
participants.refused(staged, failed);
|
|
1152
1285
|
if (coordResult.staleLoss) {
|
|
1153
1286
|
log('commitEntry:partial-retry entry=%s action=%s committed=%o failed=%o reason=%s', entry.entryId, entry.actionId, [...committed.keys()], failed, coordResult.error);
|
|
1154
|
-
throw new CoordinatorStaleLossError(failed, coordResult.error);
|
|
1287
|
+
throw new CoordinatorStaleLossError(failed, coordResult.error, coordResult.staleAt);
|
|
1155
1288
|
}
|
|
1156
1289
|
throw new CoordinatorPartialCommitError(committed, failed, coordResult.expired ?? coordResult.error);
|
|
1157
1290
|
}
|
|
@@ -1566,7 +1699,7 @@ export class TransactionCoordinator {
|
|
|
1566
1699
|
* Apply actions to a collection.
|
|
1567
1700
|
*
|
|
1568
1701
|
* This runs the action handlers, writes to the log, and collects transforms. The log entry is
|
|
1569
|
-
* written under `context`: its action id, its timestamp and its reads.
|
|
1702
|
+
* written under `context`: its action id, its timestamp, its HLC, its lane and its reads.
|
|
1570
1703
|
*/
|
|
1571
1704
|
async applyActionsToCollection(collectionActions, context, allCollectionIds) {
|
|
1572
1705
|
const collection = this.collections.get(collectionActions.collectionId);
|
|
@@ -1589,8 +1722,8 @@ export class TransactionCoordinator {
|
|
|
1589
1722
|
// same revision, and storage keys a saved block revision by `(action id, revision)`. So any
|
|
1590
1723
|
// data block the append mints takes the id an earlier attempt at this revision minted
|
|
1591
1724
|
// (Collection.logAppendBlockIds), and the entry is stamped with the transaction's own
|
|
1592
|
-
// timestamp
|
|
1593
|
-
// the time of the attempt.
|
|
1725
|
+
// timestamp and HLC — the timestamp fixed at BEGIN, the HLC minted once at commit, both
|
|
1726
|
+
// stable across attempts — rather than with the time of the attempt.
|
|
1594
1727
|
//
|
|
1595
1728
|
// NOTE: execute() reaches this method without marking its transaction in flight, so it mints
|
|
1596
1729
|
// afresh every time — correct today, because it re-runs the engine and re-stages, so a
|
|
@@ -1623,7 +1756,7 @@ export class TransactionCoordinator {
|
|
|
1623
1756
|
// correctness; a cross-collection invalidation cascade that treats it as "the definitive
|
|
1624
1757
|
// participant set of this transaction" must union it across the transaction's entries
|
|
1625
1758
|
// rather than trusting any single one.
|
|
1626
|
-
const addResult = await log.addActions(collectionActions.actions, actionId, newRev, () => blockIdsForTransforms(transforms), { collectionIds: allCollectionIds, reads: [...context.reads], timestamp: context.timestamp });
|
|
1759
|
+
const addResult = await log.addActions(collectionActions.actions, actionId, newRev, () => blockIdsForTransforms(transforms), { collectionIds: allCollectionIds, reads: [...context.reads], timestamp: context.timestamp, hlc: context.hlc, lane: context.lane });
|
|
1627
1760
|
// Return the transforms and log tail block ID
|
|
1628
1761
|
return {
|
|
1629
1762
|
success: true,
|
|
@@ -1696,6 +1829,7 @@ export class TransactionCoordinator {
|
|
|
1696
1829
|
committedCollections: commitResult.committedCollections,
|
|
1697
1830
|
failedCollections: commitResult.failedCollections,
|
|
1698
1831
|
staleLoss: commitResult.staleLoss,
|
|
1832
|
+
staleAt: commitResult.staleAt,
|
|
1699
1833
|
...(commitResult.expired === undefined ? {} : { expired: commitResult.expired }),
|
|
1700
1834
|
};
|
|
1701
1835
|
}
|
|
@@ -1869,7 +2003,7 @@ export class TransactionCoordinator {
|
|
|
1869
2003
|
return undefined;
|
|
1870
2004
|
return result.success
|
|
1871
2005
|
? { success: true, pendedBlockIds: blockIds }
|
|
1872
|
-
: pendFailure([pendRefusalError(Array.from(attempts.keys()), result)]);
|
|
2006
|
+
: pendFailure([pendRefusalError(Array.from(attempts.keys()), result, batchStaleAt(result.staleAt, blockIds))]);
|
|
1873
2007
|
}
|
|
1874
2008
|
/**
|
|
1875
2009
|
* Pend a single collection's transforms. Resolves with the collection id and its
|
|
@@ -1885,7 +2019,7 @@ export class TransactionCoordinator {
|
|
|
1885
2019
|
sole) {
|
|
1886
2020
|
const pendResult = await this.transactor.pend(this.pendRequestFor(context, validation, attempt, nominees, sole));
|
|
1887
2021
|
if (!pendResult.success) {
|
|
1888
|
-
throw pendRefusalError([collectionId], pendResult);
|
|
2022
|
+
throw pendRefusalError([collectionId], pendResult, ownStaleAt(collectionId, pendResult.staleAt));
|
|
1889
2023
|
}
|
|
1890
2024
|
return { collectionId, blockIds: pendResult.blockIds };
|
|
1891
2025
|
}
|
|
@@ -1907,10 +2041,15 @@ export class TransactionCoordinator {
|
|
|
1907
2041
|
// Collection.sync pend.
|
|
1908
2042
|
// The bases (`PendRequest.baseRevs`) were read from the same tracker the commit's declaration was
|
|
1909
2043
|
// taken from, when the attempt was described; omitted when empty.
|
|
2044
|
+
// The HLC and the lane are the write's (`WriteContext.hlc`, `WriteContext.lane`), the same
|
|
2045
|
+
// values the log entry inside `transforms` carries, so the pend and its validation payload
|
|
2046
|
+
// cannot disagree on them.
|
|
1910
2047
|
return {
|
|
1911
2048
|
actionId: context.actionId,
|
|
1912
2049
|
rev: attempt.rev,
|
|
1913
2050
|
transforms: attempt.transforms,
|
|
2051
|
+
hlc: context.hlc,
|
|
2052
|
+
lane: context.lane,
|
|
1914
2053
|
policy: 'r', // Return policy: fail but return pending actions
|
|
1915
2054
|
...(validation === undefined
|
|
1916
2055
|
? (context.priority > 0 ? { priority: context.priority } : {})
|
|
@@ -1947,6 +2086,7 @@ export class TransactionCoordinator {
|
|
|
1947
2086
|
const outcomes = together ?? await this.commitEach(context, attempts, pendedBlockIds);
|
|
1948
2087
|
const committedCollections = new Map();
|
|
1949
2088
|
const failedCollections = new Set();
|
|
2089
|
+
const staleAt = new Map();
|
|
1950
2090
|
const errors = new Set();
|
|
1951
2091
|
// Classify failures: a returned stale loss (someone committed a newer rev) is retryable after
|
|
1952
2092
|
// a re-read; a thrown/transient-exhausted or structural failure is not. staleLoss holds only
|
|
@@ -1967,6 +2107,8 @@ export class TransactionCoordinator {
|
|
|
1967
2107
|
anyStale = true;
|
|
1968
2108
|
else
|
|
1969
2109
|
anyHard = true;
|
|
2110
|
+
if (result.staleAt !== undefined)
|
|
2111
|
+
staleAt.set(result.collectionId, result.staleAt);
|
|
1970
2112
|
expired ??= result.expired;
|
|
1971
2113
|
}
|
|
1972
2114
|
}
|
|
@@ -1977,10 +2119,11 @@ export class TransactionCoordinator {
|
|
|
1977
2119
|
committedCollections,
|
|
1978
2120
|
failedCollections,
|
|
1979
2121
|
staleLoss: anyStale && !anyHard,
|
|
2122
|
+
staleAt,
|
|
1980
2123
|
...(expired === undefined ? {} : { expired }),
|
|
1981
2124
|
};
|
|
1982
2125
|
}
|
|
1983
|
-
return { success: true, committedCollections, failedCollections };
|
|
2126
|
+
return { success: true, committedCollections, failedCollections, staleAt };
|
|
1984
2127
|
}
|
|
1985
2128
|
/**
|
|
1986
2129
|
* Commits every collection as ONE request ({@link ITransactor.commitMany}): one consensus record,
|
|
@@ -2013,9 +2156,12 @@ export class TransactionCoordinator {
|
|
|
2013
2156
|
log('commitTogether: batched commit threw; committing each collection on its own actionId=%s: %o', actionId, e);
|
|
2014
2157
|
return undefined;
|
|
2015
2158
|
}
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2159
|
+
if (result === undefined)
|
|
2160
|
+
return undefined;
|
|
2161
|
+
// The batch's one confirmed revision belongs to the collection owning the block it names, and
|
|
2162
|
+
// to no other collection of the batch (see `batchStaleAt`).
|
|
2163
|
+
const attributed = batchStaleAt(result.success ? undefined : result.staleAt, pendedBlockIds);
|
|
2164
|
+
return requests.map(({ collectionId, request }) => collectionCommitOutcome(collectionId, request, result, attributed.get(collectionId)));
|
|
2019
2165
|
}
|
|
2020
2166
|
/** Commits each collection on its own, concurrently ({@link commitCollection}), one outcome per collection. */
|
|
2021
2167
|
async commitEach(context, attempts, pendedBlockIds) {
|
|
@@ -2052,7 +2198,9 @@ export class TransactionCoordinator {
|
|
|
2052
2198
|
let lastTransientError;
|
|
2053
2199
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
2054
2200
|
try {
|
|
2055
|
-
|
|
2201
|
+
const result = await this.transactor.commit(commitRequest);
|
|
2202
|
+
// A commit of one collection: its confirmed revision is this collection's own.
|
|
2203
|
+
return collectionCommitOutcome(collectionId, commitRequest, result, result.success ? undefined : result.staleAt);
|
|
2056
2204
|
}
|
|
2057
2205
|
catch (e) {
|
|
2058
2206
|
lastTransientError = e instanceof Error ? e.message : String(e);
|
|
@@ -2085,6 +2233,7 @@ export class TransactionCoordinator {
|
|
|
2085
2233
|
tailId: attempt.tailId,
|
|
2086
2234
|
rev: attempt.rev,
|
|
2087
2235
|
...blockDigestsField(attempt.blockDigests),
|
|
2236
|
+
lane: context.lane,
|
|
2088
2237
|
...refuseSoloField(context.refuseSolo)
|
|
2089
2238
|
};
|
|
2090
2239
|
}
|