@optimystic/db-core 1.22.0 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/dist/src/collection/action.d.ts +7 -4
- package/dist/src/collection/action.d.ts.map +1 -1
- package/dist/src/collection/action.js +13 -5
- package/dist/src/collection/action.js.map +1 -1
- package/dist/src/collection/collection.d.ts +134 -46
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +207 -70
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/struct.d.ts +38 -8
- package/dist/src/collection/struct.d.ts.map +1 -1
- package/dist/src/collection/struct.js.map +1 -1
- package/dist/src/collection/write-attempts.d.ts +138 -34
- package/dist/src/collection/write-attempts.d.ts.map +1 -1
- package/dist/src/collection/write-attempts.js +168 -101
- package/dist/src/collection/write-attempts.js.map +1 -1
- package/dist/src/collections/diary/diary.d.ts.map +1 -1
- package/dist/src/collections/diary/diary.js +5 -1
- package/dist/src/collections/diary/diary.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts +15 -9
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +6 -4
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/hlc/clock-store.d.ts +63 -0
- package/dist/src/hlc/clock-store.d.ts.map +1 -0
- package/dist/src/hlc/clock-store.js +49 -0
- package/dist/src/hlc/clock-store.js.map +1 -0
- package/dist/src/hlc/clock.d.ts +64 -0
- package/dist/src/hlc/clock.d.ts.map +1 -0
- package/dist/src/hlc/clock.js +171 -0
- package/dist/src/hlc/clock.js.map +1 -0
- package/dist/src/hlc/hlc.d.ts +38 -0
- package/dist/src/hlc/hlc.d.ts.map +1 -0
- package/dist/src/hlc/hlc.js +41 -0
- package/dist/src/hlc/hlc.js.map +1 -0
- package/dist/src/hlc/index.d.ts +5 -0
- package/dist/src/hlc/index.d.ts.map +1 -0
- package/dist/src/hlc/index.js +5 -0
- package/dist/src/hlc/index.js.map +1 -0
- package/dist/src/hlc/skew.d.ts +35 -0
- package/dist/src/hlc/skew.d.ts.map +1 -0
- package/dist/src/hlc/skew.js +35 -0
- package/dist/src/hlc/skew.js.map +1 -0
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/lane/index.d.ts +2 -0
- package/dist/src/lane/index.d.ts.map +1 -0
- package/dist/src/lane/index.js +2 -0
- package/dist/src/lane/index.js.map +1 -0
- package/dist/src/lane/lane.d.ts +46 -0
- package/dist/src/lane/lane.d.ts.map +1 -0
- package/dist/src/lane/lane.js +26 -0
- package/dist/src/lane/lane.js.map +1 -0
- package/dist/src/log/log.d.ts +50 -22
- package/dist/src/log/log.d.ts.map +1 -1
- package/dist/src/log/log.js +21 -13
- package/dist/src/log/log.js.map +1 -1
- package/dist/src/log/struct.d.ts +61 -1
- package/dist/src/log/struct.d.ts.map +1 -1
- package/dist/src/log/struct.js +24 -0
- package/dist/src/log/struct.js.map +1 -1
- package/dist/src/network/i-repo.d.ts +10 -0
- package/dist/src/network/i-repo.d.ts.map +1 -1
- package/dist/src/network/stale-failure.d.ts +20 -7
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +29 -7
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/network/struct.d.ts +35 -0
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/offline/entry.d.ts +14 -0
- package/dist/src/offline/entry.d.ts.map +1 -1
- package/dist/src/offline/entry.js +2 -0
- package/dist/src/offline/entry.js.map +1 -1
- package/dist/src/offline/journal.d.ts +10 -0
- package/dist/src/offline/journal.d.ts.map +1 -1
- package/dist/src/offline/queue.d.ts +10 -0
- package/dist/src/offline/queue.d.ts.map +1 -1
- package/dist/src/offline/queue.js +29 -3
- package/dist/src/offline/queue.js.map +1 -1
- package/dist/src/offline/status.d.ts +7 -0
- package/dist/src/offline/status.d.ts.map +1 -1
- package/dist/src/offline/status.js.map +1 -1
- package/dist/src/testing/test-transactor.d.ts +9 -1
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +6 -1
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/actions-engine.d.ts +3 -0
- package/dist/src/transaction/actions-engine.d.ts.map +1 -1
- package/dist/src/transaction/actions-engine.js +3 -0
- package/dist/src/transaction/actions-engine.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +10 -5
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +197 -48
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transaction/entry-commit.d.ts +45 -10
- package/dist/src/transaction/entry-commit.d.ts.map +1 -1
- package/dist/src/transaction/entry-commit.js +20 -7
- package/dist/src/transaction/entry-commit.js.map +1 -1
- package/dist/src/transaction/errors.d.ts +69 -1
- package/dist/src/transaction/errors.d.ts.map +1 -1
- package/dist/src/transaction/errors.js +70 -1
- package/dist/src/transaction/errors.js.map +1 -1
- package/dist/src/transaction/index.d.ts +1 -1
- package/dist/src/transaction/index.d.ts.map +1 -1
- package/dist/src/transaction/index.js +1 -1
- package/dist/src/transaction/index.js.map +1 -1
- package/dist/src/transaction/session.d.ts.map +1 -1
- package/dist/src/transaction/session.js +18 -5
- package/dist/src/transaction/session.js.map +1 -1
- package/dist/src/transaction/transaction.d.ts +74 -15
- package/dist/src/transaction/transaction.d.ts.map +1 -1
- package/dist/src/transaction/transaction.js +35 -15
- package/dist/src/transaction/transaction.js.map +1 -1
- package/dist/src/transaction/validator.d.ts +25 -4
- package/dist/src/transaction/validator.d.ts.map +1 -1
- package/dist/src/transaction/validator.js +45 -5
- package/dist/src/transaction/validator.js.map +1 -1
- package/dist/src/transactor/block-floors.d.ts +12 -1
- package/dist/src/transactor/block-floors.d.ts.map +1 -1
- package/dist/src/transactor/block-floors.js +14 -1
- package/dist/src/transactor/block-floors.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +8 -0
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +8 -4
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor-source.d.ts +11 -0
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +5 -1
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/dist/src/transactor/transactor.d.ts +8 -0
- package/dist/src/transactor/transactor.d.ts.map +1 -1
- package/dist/src/transform/cache-source.d.ts +7 -0
- package/dist/src/transform/cache-source.d.ts.map +1 -1
- package/dist/src/transform/cache-source.js +14 -18
- package/dist/src/transform/cache-source.js.map +1 -1
- package/package.json +1 -1
- package/src/collection/action.ts +13 -5
- package/src/collection/collection.ts +250 -76
- package/src/collection/struct.ts +38 -8
- package/src/collection/write-attempts.ts +213 -109
- package/src/collections/diary/diary.ts +5 -1
- package/src/collections/tree/tree.ts +18 -11
- package/src/hlc/clock-store.ts +68 -0
- package/src/hlc/clock.ts +196 -0
- package/src/hlc/hlc.ts +56 -0
- package/src/hlc/index.ts +4 -0
- package/src/hlc/skew.ts +55 -0
- package/src/index.ts +2 -0
- package/src/lane/index.ts +1 -0
- package/src/lane/lane.ts +54 -0
- package/src/log/log.ts +341 -310
- package/src/log/struct.ts +62 -1
- package/src/network/i-repo.ts +11 -1
- package/src/network/stale-failure.ts +31 -7
- package/src/network/struct.ts +805 -770
- package/src/offline/entry.ts +16 -0
- package/src/offline/journal.ts +10 -0
- package/src/offline/queue.ts +38 -4
- package/src/offline/status.ts +7 -0
- package/src/testing/test-transactor.ts +1065 -1053
- package/src/transaction/actions-engine.ts +3 -0
- package/src/transaction/coordinator.ts +229 -54
- package/src/transaction/entry-commit.ts +56 -14
- package/src/transaction/errors.ts +60 -0
- package/src/transaction/index.ts +1 -1
- package/src/transaction/session.ts +18 -5
- package/src/transaction/transaction.ts +90 -17
- package/src/transaction/validator.ts +46 -5
- package/src/transactor/block-floors.ts +15 -1
- package/src/transactor/network-transactor.ts +15 -4
- package/src/transactor/transactor-source.ts +15 -1
- package/src/transactor/transactor.ts +9 -0
- package/src/transform/cache-source.ts +15 -17
|
@@ -1,17 +1,144 @@
|
|
|
1
1
|
import type { IBlock, BlockId, Action, ActionId, BlockContentDigests, BlockStore, Transforms, WriteDurability, WriteFailure } from "../index.js";
|
|
2
|
-
import type { BlockBaseRevs } from "../network/struct.js";
|
|
2
|
+
import type { BlockBaseRevs, StaleFailure } from "../network/struct.js";
|
|
3
|
+
import type { Hlc } from "../hlc/hlc.js";
|
|
4
|
+
import type { Lane } from "../lane/lane.js";
|
|
3
5
|
import { Tracker } from "../transform/tracker.js";
|
|
4
6
|
import type { BasePins } from "../transform/base-pins.js";
|
|
5
7
|
import type { CacheSource } from "../transform/cache-source.js";
|
|
6
8
|
import type { CollectionId, InFlightAttempt, SyncOptions } from "./struct.js";
|
|
7
9
|
import type { RefreshReport } from "./collection.js";
|
|
8
|
-
/**
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
/** Default consecutive stalled refreshes — ones that moved this collection's revision NOWHERE
|
|
11
|
+
* while a responder CONFIRMED a revision at or above the one being requested — before a write
|
|
12
|
+
* gives up. Two, not one, so a single transiently-lagging read is absorbed.
|
|
13
|
+
*
|
|
14
|
+
* NOTE: two is a judgement call, not a measurement — nothing here counts how often a legitimate
|
|
15
|
+
* loser reads a view that moves nowhere for two consecutive rounds. The check only strikes when the
|
|
16
|
+
* refresh made NO progress at all, so a client merely catching up is already excluded; if a
|
|
17
|
+
* spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
|
|
18
|
+
export declare const DefaultMaxStalledAttempts = 2;
|
|
19
|
+
/** A revision a responder CONFIRMED is committed under some other action — `StaleFailure.staleAt`
|
|
20
|
+
* when it is present. A producer sets it only after reading the revision out of its own storage, so
|
|
21
|
+
* it is evidence, where an absent `staleAt` is merely "no confirmed number". */
|
|
22
|
+
export type ConfirmedRevision = NonNullable<StaleFailure['staleAt']>;
|
|
23
|
+
/** What {@link RevisionStallWatch.check} says about the attempt about to run: `clear` (progress, no
|
|
24
|
+
* evidence, or no news this round), a `strike` counted toward the limit, or `stalled` — the strike
|
|
25
|
+
* that reaches it, after which the write path throws its own stalled error. */
|
|
26
|
+
export type StallVerdict = {
|
|
27
|
+
kind: 'clear';
|
|
28
|
+
} | {
|
|
29
|
+
kind: 'strike';
|
|
30
|
+
strike: number;
|
|
31
|
+
of: number;
|
|
32
|
+
staleAt: ConfirmedRevision;
|
|
33
|
+
} | {
|
|
34
|
+
kind: 'stalled';
|
|
35
|
+
strike: number;
|
|
36
|
+
of: number;
|
|
37
|
+
staleAt: ConfirmedRevision;
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* The evidence one collection's write gathers from its refusals, and the two rules that read it: the
|
|
41
|
+
* stall check ({@link check}) and the slot hold ({@link holdsSlot}). One instance per collection per
|
|
42
|
+
* write. Both retry loops keep one — {@link AttemptCycle} for `Collection.sync` and `commitHead`,
|
|
43
|
+
* `TransactionCoordinator.commitAttempts` one per participant — so the rule exists once and the two
|
|
44
|
+
* loops cannot drift (docs/internals.md, "Monotonicity is also what makes a wedged view detectable").
|
|
45
|
+
*
|
|
46
|
+
* Revisions are per-collection counters, so a confirmed revision is only ever compared against the
|
|
47
|
+
* counter of the collection that owns it; a caller holding a refusal that covers several collections
|
|
48
|
+
* attributes the number before noting it (`staleAtOwner` in network/stale-failure.ts).
|
|
49
|
+
*/
|
|
50
|
+
export declare class RevisionStallWatch {
|
|
51
|
+
private readonly maxStalledAttempts;
|
|
52
|
+
private highest;
|
|
53
|
+
private latestRefusalConfirmed;
|
|
54
|
+
private latestLossConfirmed;
|
|
55
|
+
private confirmedLosses;
|
|
56
|
+
private consecutiveStalls;
|
|
57
|
+
private previousRequestedRev;
|
|
58
|
+
constructor(maxStalledAttempts: number);
|
|
59
|
+
/** The highest confirmed revision noted so far, for the write path's error. */
|
|
60
|
+
get staleAt(): ConfirmedRevision | undefined;
|
|
61
|
+
/** A refused attempt of this collection: its own confirmed revision, or `undefined` when the
|
|
62
|
+
* refusal carried none — a bare refusal, or a refusal of a sibling's batch whose number belongs
|
|
63
|
+
* to another collection. */
|
|
64
|
+
noteRefusal(staleAt: ConfirmedRevision | undefined): void;
|
|
65
|
+
/** A refresh round refused for a cause that can clear (a `completion-refused` `TornActionError`):
|
|
66
|
+
* the latest loss no longer confirms a committed rival, so the next round sleeps. The stall
|
|
67
|
+
* evidence is unchanged — the attempt's own refusal still said what it said. */
|
|
68
|
+
noteRefusedRound(): void;
|
|
69
|
+
/** Forward progress — an attempt committed, or a refresh finished the write's own entry: the
|
|
70
|
+
* stall evidence starts over. `confirmedLosses` is NOT reset: a member counts per action for the
|
|
71
|
+
* action's whole life. */
|
|
72
|
+
noteCommit(): void;
|
|
73
|
+
/**
|
|
74
|
+
* The check before an attempt that would request `requestedRev`. It asks: can the attempt about
|
|
75
|
+
* to run possibly differ from the one that just failed? Only when BOTH of these hold is the
|
|
76
|
+
* answer provably no:
|
|
77
|
+
*
|
|
78
|
+
* - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
|
|
79
|
+
* only after reading that revision as durably held by someone else out of its own storage,
|
|
80
|
+
* revisions are one per-collection counter that every commit touches, and a confirmed
|
|
81
|
+
* revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
|
|
82
|
+
* lost before it is sent.
|
|
83
|
+
* - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
|
|
84
|
+
* held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
|
|
85
|
+
* refresh that moved forward WITHOUT clearing the confirmed number is a collection still
|
|
86
|
+
* climbing (it read a replica that lags the holder — the same partial catch-up
|
|
87
|
+
* `reportShortfall` exists to report), and its next attempt is a genuinely different request
|
|
88
|
+
* that may yet win.
|
|
89
|
+
*
|
|
90
|
+
* This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
|
|
91
|
+
* sole rule for that, untouched. It only ever stops a loop that rule had already decided to
|
|
92
|
+
* continue. Called once before EVERY attempt, the first included, so the previous requested
|
|
93
|
+
* revision is always the one the attempt just refused asked for.
|
|
94
|
+
*/
|
|
95
|
+
check(requestedRev: number): StallVerdict;
|
|
96
|
+
/**
|
|
97
|
+
* Whether the members that refused this write hold the next slot for it, so its retry should
|
|
98
|
+
* follow with no sleep. The jittered backoff exists to separate a herd of writers that lost the
|
|
99
|
+
* SAME race, so they do not re-collide on the next tick. A loss to a rival that had already
|
|
100
|
+
* committed (the refusal carried a confirmed `staleAt`) is not a herd, and once this write has
|
|
101
|
+
* taken `SlotHoldAfterLosses` such losses (`confirmedLosses`, counted over the whole write as a
|
|
102
|
+
* member counts its stale refusals of one action) the members that refused it hold the next slot
|
|
103
|
+
* for it — so sleeping only spends that hold. Both sides count the same refusals against the same
|
|
104
|
+
* constant, so the writer stops sleeping on the loss after which a member holds, with nothing
|
|
105
|
+
* said over the wire. A loss the latest refusal did not confirm (a race against something still
|
|
106
|
+
* in flight) keeps the jitter.
|
|
107
|
+
*
|
|
108
|
+
* NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member
|
|
109
|
+
* the refused pends did not reach) the write retries without sleeping and so spends the same
|
|
110
|
+
* attempt budget sooner, at about twice one writer's pend rate (measured in the plan stage of
|
|
111
|
+
* `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it
|
|
112
|
+
* only delayed the same give-up.
|
|
113
|
+
*/
|
|
114
|
+
holdsSlot(): boolean;
|
|
115
|
+
/** The highest confirmed revision, as the floor of the refresh's log-tail read, so a machine
|
|
116
|
+
* that has not caught up to it is not the last one asked (see `Collection.updateInternal`). */
|
|
117
|
+
tailFloor(): number | undefined;
|
|
118
|
+
}
|
|
119
|
+
/** What a write is identified as on the wire: the action id every attempt of it reuses, the two
|
|
120
|
+
* per-write time values its log entry carries, the HLC and the wall-clock timestamp, and its lane.
|
|
121
|
+
* All four are minted once per WRITE, never per attempt, so a retry at the same revision appends the
|
|
122
|
+
* same entry (docs/internals.md, "A write's log append is a fixed function of the write and the
|
|
123
|
+
* revision it is requesting"). */
|
|
12
124
|
export type WriteIdentity = {
|
|
13
125
|
readonly actionId: ActionId;
|
|
126
|
+
/** The wall-clock time the write was minted at (`LogEntry.timestamp`). */
|
|
14
127
|
readonly timestamp: number;
|
|
128
|
+
/** The HLC the write was minted with (`LogEntry.hlc`), from the writer's clock
|
|
129
|
+
* (`clockOf(transactor).now()`), and carried on every pend of it (`PendRequest.hlc`).
|
|
130
|
+
* NOTE: a write refused as stale is retried over a refresh that may have observed newer HLCs,
|
|
131
|
+
* and the retry keeps the HLC it was first minted with — the identity rule above — so its entry
|
|
132
|
+
* can carry an HLC below a rival it was re-staged over. Nothing is wrong today: in the strong
|
|
133
|
+
* lane the revision is the order and the HLC is observational. The replay stage (backlog
|
|
134
|
+
* `feat-replay-mode-collections-merge-divergent-histories`) commits locally without a log-tail
|
|
135
|
+
* retry, so it never meets the case. */
|
|
136
|
+
readonly hlc: Hlc;
|
|
137
|
+
/** The write's lane (`ActionEntry.lane`, `PendRequest.lane`, `CommitRequest.lane`): the caller's
|
|
138
|
+
* classification when it gave one (`SyncOptions.lane`), else the collection's own declaration
|
|
139
|
+
* over what the write stages (`Collection.laneOf`); the queue's for a head commit
|
|
140
|
+
* (`HeadCommitOptions.lane`). */
|
|
141
|
+
readonly lane: Lane;
|
|
15
142
|
};
|
|
16
143
|
/** What one attempt is built from: the actions its log entry records, the transforms they produced
|
|
17
144
|
* over the committed state, and the bases those transforms were computed against. For a sync these
|
|
@@ -46,8 +173,8 @@ export type StagedAttempt = {
|
|
|
46
173
|
* — but if an attempt ever grows a per-attempt footprint beyond the log blocks, reset the abandoned
|
|
47
174
|
* tracker on the stale-failure branch instead of letting the residue ride to the end of the write.
|
|
48
175
|
*
|
|
49
|
-
* The append is a fixed function of the write and `rev`: the timestamp
|
|
50
|
-
* data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
|
|
176
|
+
* The append is a fixed function of the write and `rev`: the HLC, timestamp and lane are the identity's, and
|
|
177
|
+
* any data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
|
|
51
178
|
* `Collection.logAppendBlockIds`, which replays the ids an earlier attempt at this revision minted.
|
|
52
179
|
*
|
|
53
180
|
* The declarations are computed AFTER the append so the log tail and header transforms the attempt
|
|
@@ -118,7 +245,6 @@ export declare class AttemptCycle {
|
|
|
118
245
|
private readonly collectionId;
|
|
119
246
|
private readonly instanceTag;
|
|
120
247
|
private readonly maxAttempts;
|
|
121
|
-
private readonly maxStalledAttempts;
|
|
122
248
|
private readonly backoff;
|
|
123
249
|
private readonly deadlineMs;
|
|
124
250
|
private readonly signal;
|
|
@@ -126,11 +252,7 @@ export declare class AttemptCycle {
|
|
|
126
252
|
private readonly startedAt;
|
|
127
253
|
private consecutiveFailures;
|
|
128
254
|
private lastReason;
|
|
129
|
-
private
|
|
130
|
-
private lastFailureConfirmedStaleAt;
|
|
131
|
-
private confirmedLosses;
|
|
132
|
-
private consecutiveStalls;
|
|
133
|
-
private previousRequestedRev;
|
|
255
|
+
private readonly watch;
|
|
134
256
|
/** Who holds what the write has committed so far. Stays `undefined` while nothing has been
|
|
135
257
|
* committed, which is also the answer when no attempt runs at all — a write with nothing staged
|
|
136
258
|
* writes nothing, and there is no durability to fabricate for it. Refused attempts never
|
|
@@ -147,27 +269,9 @@ export declare class AttemptCycle {
|
|
|
147
269
|
priority(): number;
|
|
148
270
|
/**
|
|
149
271
|
* The checks that run before an attempt is built: the abort signal, the wall-clock deadline, and
|
|
150
|
-
* the stall check. The deadline comes before the stall check
|
|
151
|
-
* outer bound, so a write that is both past it and stalled
|
|
152
|
-
*
|
|
153
|
-
* The stall check asks: can the attempt about to run possibly differ from the one that just
|
|
154
|
-
* failed? Only when BOTH of these hold is the answer provably no:
|
|
155
|
-
*
|
|
156
|
-
* - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
|
|
157
|
-
* only after reading that revision as durably held by someone else out of its own storage,
|
|
158
|
-
* revisions are one per-collection counter that every commit touches, and a confirmed
|
|
159
|
-
* revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
|
|
160
|
-
* lost before it is sent.
|
|
161
|
-
* - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
|
|
162
|
-
* held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
|
|
163
|
-
* refresh that moved forward WITHOUT clearing the confirmed number is a collection still
|
|
164
|
-
* climbing (it read a replica that lags the holder — the same partial catch-up
|
|
165
|
-
* `reportShortfall` exists to report), and its next attempt is a genuinely different request
|
|
166
|
-
* that may yet win.
|
|
167
|
-
*
|
|
168
|
-
* This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
|
|
169
|
-
* sole rule for that, untouched. It only ever stops a loop that rule had already decided to
|
|
170
|
-
* continue.
|
|
272
|
+
* the stall check ({@link RevisionStallWatch.check}). The deadline comes before the stall check
|
|
273
|
+
* deliberately: it is the documented outer bound, so a write that is both past it and stalled
|
|
274
|
+
* reports the deadline.
|
|
171
275
|
*
|
|
172
276
|
* @param requestedRev the revision the attempt would request (`Collection.getNextRev`).
|
|
173
277
|
* @param heldRev the revision the collection holds, for the stall report.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"write-attempts.d.ts","sourceRoot":"","sources":["../../../src/collection/write-attempts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,mBAAmB,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACjJ,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"write-attempts.d.ts","sourceRoot":"","sources":["../../../src/collection/write-attempts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,mBAAmB,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACjJ,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAUhE,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC9E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA4BrD;;;;;;;iGAOiG;AACjG,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C;;gFAEgF;AAChF,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;AAErE;;+EAE+E;AAC/E,MAAM,MAAM,YAAY,GACrB;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,GACjB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,GAC1E;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,CAAC;AAE/E;;;;;;;;;;GAUG;AACH,qBAAa,kBAAkB;IAyBlB,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IArB/C,OAAO,CAAC,OAAO,CAAgC;IAI/C,OAAO,CAAC,sBAAsB,CAAS;IAKvC,OAAO,CAAC,mBAAmB,CAAS;IAIpC,OAAO,CAAC,eAAe,CAAK;IAG5B,OAAO,CAAC,iBAAiB,CAAK;IAG9B,OAAO,CAAC,oBAAoB,CAAqB;gBAEpB,kBAAkB,EAAE,MAAM;IAEvD,+EAA+E;IAC/E,IAAI,OAAO,IAAI,iBAAiB,GAAG,SAAS,CAE3C;IAED;;gCAE4B;IAC5B,WAAW,CAAC,OAAO,EAAE,iBAAiB,GAAG,SAAS,GAAG,IAAI;IAUzD;;oFAEgF;IAChF,gBAAgB,IAAI,IAAI;IAIxB;;8BAE0B;IAC1B,UAAU,IAAI,IAAI;IAOlB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,YAAY,EAAE,MAAM,GAAG,YAAY;IAyBzC;;;;;;;;;;;;;;;;;OAiBG;IACH,SAAS,IAAI,OAAO;IAIpB;mGAC+F;IAC/F,SAAS,IAAI,MAAM,GAAG,SAAS;CAG/B;AAWD;;;;kCAIkC;AAClC,MAAM,MAAM,aAAa,GAAG;IAC3B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;;;4CAOwC;IACxC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC;IAClB;;;qCAGiC;IACjC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;CACpB,CAAC;AAEF;;;8CAG8C;AAC9C,MAAM,MAAM,aAAa,CAAC,OAAO,IAAI;IACpC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;CACxB,CAAC;AAEF;+FAC+F;AAC/F,MAAM,MAAM,aAAa,GAAG;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAClC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,mBAAmB,GAAG,SAAS,CAAC;IACvD,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;CACjC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;0EA2B0E;AAC1E,wBAAsB,YAAY,CAAC,OAAO,EACzC,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,EAC1B,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,EAC9B,QAAQ,EAAE,aAAa,EACvB,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,MAAM,OAAO,GAC1D,OAAO,CAAC,aAAa,CAAC,CAoBxB;AAED;;;;;;;;sDAQsD;AACtD,wBAAgB,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,GAAG,eAAe,CAS1F;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAWvI;AAED;;;iEAGiE;AACjE,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnH;;;;;;GAMG;AACH,qBAAa,YAAY;IA0BvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,WAAW;IA1B7B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAqB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IAKxC,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,UAAU,CAAqB;IAEvC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAE3C;;;;;;wFAMoF;IACpF,UAAU,EAAE,eAAe,GAAG,SAAS,CAAC;gBAGtB,YAAY,EAAE,YAAY,EAC1B,WAAW,EAAE,MAAM,EACpC,OAAO,CAAC,EAAE,WAAW;IAatB;;;mCAG+B;IAC/B,QAAQ,IAAI,MAAM;IAIlB;;;;;;;;2GAQuG;IACvG,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAsBtE;;;;;;;;;;;;;;;;;;;;;;;;;;;gHA2B4G;IAC5G,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAkB1C;;qFAEiF;IACjF,YAAY,CAAC,UAAU,EAAE,eAAe,GAAG,IAAI;IAO/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qFAgCiF;IAC3E,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IA+B7D,OAAO,CAAC,YAAY;CAGpB"}
|
|
@@ -40,27 +40,155 @@ const DefaultMaxBackoffMs = 5000;
|
|
|
40
40
|
* loser reads a view that moves nowhere for two consecutive rounds. The check only strikes when the
|
|
41
41
|
* refresh made NO progress at all, so a client merely catching up is already excluded; if a
|
|
42
42
|
* spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
|
|
43
|
-
const DefaultMaxStalledAttempts = 2;
|
|
43
|
+
export const DefaultMaxStalledAttempts = 2;
|
|
44
44
|
/**
|
|
45
|
-
* The
|
|
45
|
+
* The evidence one collection's write gathers from its refusals, and the two rules that read it: the
|
|
46
|
+
* stall check ({@link check}) and the slot hold ({@link holdsSlot}). One instance per collection per
|
|
47
|
+
* write. Both retry loops keep one — {@link AttemptCycle} for `Collection.sync` and `commitHead`,
|
|
48
|
+
* `TransactionCoordinator.commitAttempts` one per participant — so the rule exists once and the two
|
|
49
|
+
* loops cannot drift (docs/internals.md, "Monotonicity is also what makes a wedged view detectable").
|
|
46
50
|
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* (`confirmedLosses`, counted over the whole write as a member counts its stale refusals of one
|
|
51
|
-
* action) the members that refused it hold the next slot for it — so sleeping only spends that hold.
|
|
52
|
-
* Both sides count the same refusals against the same constant, so the writer stops sleeping on the
|
|
53
|
-
* loss after which a member holds, with nothing said over the wire. A loss the latest refusal did not
|
|
54
|
-
* confirm (`latestConfirmed` false — a race against something still in flight) keeps the jitter.
|
|
55
|
-
*
|
|
56
|
-
* NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member the
|
|
57
|
-
* refused pends did not reach) this retries without sleeping and so spends the same attempt budget
|
|
58
|
-
* sooner, at about twice one writer's pend rate (measured in the plan stage of
|
|
59
|
-
* `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it only
|
|
60
|
-
* delayed the same give-up.
|
|
51
|
+
* Revisions are per-collection counters, so a confirmed revision is only ever compared against the
|
|
52
|
+
* counter of the collection that owns it; a caller holding a refusal that covers several collections
|
|
53
|
+
* attributes the number before noting it (`staleAtOwner` in network/stale-failure.ts).
|
|
61
54
|
*/
|
|
62
|
-
|
|
63
|
-
|
|
55
|
+
export class RevisionStallWatch {
|
|
56
|
+
maxStalledAttempts;
|
|
57
|
+
// Highest confirmed revision any responder has reported holding, accumulated with the
|
|
58
|
+
// codebase's single rule for picking among several candidates. Reported on the error, and the
|
|
59
|
+
// evidence the stall check reasons from.
|
|
60
|
+
highest;
|
|
61
|
+
// Whether the refusal most recently noted carried its OWN staleAt. A strike needs the responder
|
|
62
|
+
// to have re-confirmed the number this round, not merely an older observation left standing in
|
|
63
|
+
// `highest`.
|
|
64
|
+
latestRefusalConfirmed = false;
|
|
65
|
+
// Whether the failure most recently handled — the refusal, or a refresh round refused since —
|
|
66
|
+
// was a loss to a rival already durably committed. Only the attempt's own refusal can say so; a
|
|
67
|
+
// round refused for a cause that can clear is a race against something still in flight. Read by
|
|
68
|
+
// the slot hold alone, where `latestRefusalConfirmed` is the stall check's.
|
|
69
|
+
latestLossConfirmed = false;
|
|
70
|
+
// Refused attempts of this write that carried their own staleAt. Never reset: a member counts
|
|
71
|
+
// its stale refusals per action for the action's whole life, and this is the writer's view of
|
|
72
|
+
// that count (see holdsSlot).
|
|
73
|
+
confirmedLosses = 0;
|
|
74
|
+
// Consecutive refreshes that moved the requested revision nowhere at all while a confirmed
|
|
75
|
+
// revision stood at or above it.
|
|
76
|
+
consecutiveStalls = 0;
|
|
77
|
+
// The revision the PREVIOUS attempt would have requested, so the stall check can tell a refresh
|
|
78
|
+
// that moved nowhere from one that is still climbing toward the confirmed number.
|
|
79
|
+
previousRequestedRev;
|
|
80
|
+
constructor(maxStalledAttempts) {
|
|
81
|
+
this.maxStalledAttempts = maxStalledAttempts;
|
|
82
|
+
}
|
|
83
|
+
/** The highest confirmed revision noted so far, for the write path's error. */
|
|
84
|
+
get staleAt() {
|
|
85
|
+
return this.highest;
|
|
86
|
+
}
|
|
87
|
+
/** A refused attempt of this collection: its own confirmed revision, or `undefined` when the
|
|
88
|
+
* refusal carried none — a bare refusal, or a refusal of a sibling's batch whose number belongs
|
|
89
|
+
* to another collection. */
|
|
90
|
+
noteRefusal(staleAt) {
|
|
91
|
+
// Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
|
|
92
|
+
// responder reporting a LOWER number understates the binding constraint. Same rule the
|
|
93
|
+
// producers and the transactor's aggregation already use.
|
|
94
|
+
this.highest = highestStaleAt([this.highest, staleAt]);
|
|
95
|
+
this.latestRefusalConfirmed = staleAt !== undefined;
|
|
96
|
+
this.latestLossConfirmed = this.latestRefusalConfirmed;
|
|
97
|
+
if (this.latestRefusalConfirmed)
|
|
98
|
+
this.confirmedLosses++;
|
|
99
|
+
}
|
|
100
|
+
/** A refresh round refused for a cause that can clear (a `completion-refused` `TornActionError`):
|
|
101
|
+
* the latest loss no longer confirms a committed rival, so the next round sleeps. The stall
|
|
102
|
+
* evidence is unchanged — the attempt's own refusal still said what it said. */
|
|
103
|
+
noteRefusedRound() {
|
|
104
|
+
this.latestLossConfirmed = false;
|
|
105
|
+
}
|
|
106
|
+
/** Forward progress — an attempt committed, or a refresh finished the write's own entry: the
|
|
107
|
+
* stall evidence starts over. `confirmedLosses` is NOT reset: a member counts per action for the
|
|
108
|
+
* action's whole life. */
|
|
109
|
+
noteCommit() {
|
|
110
|
+
this.highest = undefined;
|
|
111
|
+
this.latestRefusalConfirmed = false;
|
|
112
|
+
this.latestLossConfirmed = false;
|
|
113
|
+
this.consecutiveStalls = 0;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* The check before an attempt that would request `requestedRev`. It asks: can the attempt about
|
|
117
|
+
* to run possibly differ from the one that just failed? Only when BOTH of these hold is the
|
|
118
|
+
* answer provably no:
|
|
119
|
+
*
|
|
120
|
+
* - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
|
|
121
|
+
* only after reading that revision as durably held by someone else out of its own storage,
|
|
122
|
+
* revisions are one per-collection counter that every commit touches, and a confirmed
|
|
123
|
+
* revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
|
|
124
|
+
* lost before it is sent.
|
|
125
|
+
* - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
|
|
126
|
+
* held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
|
|
127
|
+
* refresh that moved forward WITHOUT clearing the confirmed number is a collection still
|
|
128
|
+
* climbing (it read a replica that lags the holder — the same partial catch-up
|
|
129
|
+
* `reportShortfall` exists to report), and its next attempt is a genuinely different request
|
|
130
|
+
* that may yet win.
|
|
131
|
+
*
|
|
132
|
+
* This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
|
|
133
|
+
* sole rule for that, untouched. It only ever stops a loop that rule had already decided to
|
|
134
|
+
* continue. Called once before EVERY attempt, the first included, so the previous requested
|
|
135
|
+
* revision is always the one the attempt just refused asked for.
|
|
136
|
+
*/
|
|
137
|
+
check(requestedRev) {
|
|
138
|
+
const refreshMoved = this.previousRequestedRev !== undefined && requestedRev > this.previousRequestedRev;
|
|
139
|
+
this.previousRequestedRev = requestedRev;
|
|
140
|
+
if (this.highest === undefined) {
|
|
141
|
+
return { kind: 'clear' };
|
|
142
|
+
}
|
|
143
|
+
if (requestedRev > this.highest.rev || refreshMoved) {
|
|
144
|
+
// Either the refresh adopted a revision above the confirmed one — ordinary contention,
|
|
145
|
+
// where the rival's commit is exactly what we just read — or it made partial forward
|
|
146
|
+
// progress. Both mean the next attempt differs. Not a stall.
|
|
147
|
+
this.consecutiveStalls = 0;
|
|
148
|
+
return { kind: 'clear' };
|
|
149
|
+
}
|
|
150
|
+
if (!this.latestRefusalConfirmed) {
|
|
151
|
+
// The responder told us nothing new this round. No strike, and no reset either — the
|
|
152
|
+
// budget stays bounded by maxAttempts.
|
|
153
|
+
return { kind: 'clear' };
|
|
154
|
+
}
|
|
155
|
+
this.consecutiveStalls++;
|
|
156
|
+
// Two strikes, not one: a legitimate loser can transiently read a view that has not yet
|
|
157
|
+
// caught up with the rival's commit, which looks identical for one round.
|
|
158
|
+
const kind = this.consecutiveStalls >= this.maxStalledAttempts ? 'stalled' : 'strike';
|
|
159
|
+
return { kind, strike: this.consecutiveStalls, of: this.maxStalledAttempts, staleAt: this.highest };
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Whether the members that refused this write hold the next slot for it, so its retry should
|
|
163
|
+
* follow with no sleep. The jittered backoff exists to separate a herd of writers that lost the
|
|
164
|
+
* SAME race, so they do not re-collide on the next tick. A loss to a rival that had already
|
|
165
|
+
* committed (the refusal carried a confirmed `staleAt`) is not a herd, and once this write has
|
|
166
|
+
* taken `SlotHoldAfterLosses` such losses (`confirmedLosses`, counted over the whole write as a
|
|
167
|
+
* member counts its stale refusals of one action) the members that refused it hold the next slot
|
|
168
|
+
* for it — so sleeping only spends that hold. Both sides count the same refusals against the same
|
|
169
|
+
* constant, so the writer stops sleeping on the loss after which a member holds, with nothing
|
|
170
|
+
* said over the wire. A loss the latest refusal did not confirm (a race against something still
|
|
171
|
+
* in flight) keeps the jitter.
|
|
172
|
+
*
|
|
173
|
+
* NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member
|
|
174
|
+
* the refused pends did not reach) the write retries without sleeping and so spends the same
|
|
175
|
+
* attempt budget sooner, at about twice one writer's pend rate (measured in the plan stage of
|
|
176
|
+
* `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it
|
|
177
|
+
* only delayed the same give-up.
|
|
178
|
+
*/
|
|
179
|
+
holdsSlot() {
|
|
180
|
+
return this.latestLossConfirmed && this.confirmedLosses >= SlotHoldAfterLosses;
|
|
181
|
+
}
|
|
182
|
+
/** The highest confirmed revision, as the floor of the refresh's log-tail read, so a machine
|
|
183
|
+
* that has not caught up to it is not the last one asked (see `Collection.updateInternal`). */
|
|
184
|
+
tailFloor() {
|
|
185
|
+
return this.highest?.rev;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
/** The sleep before the refresh that follows a write's `failures`-th consecutive loss: none while a
|
|
189
|
+
* member holds the next slot for this write ({@link RevisionStallWatch.holdsSlot}), else jittered. */
|
|
190
|
+
function retryBackoffMs(failures, watch, config, rand) {
|
|
191
|
+
if (watch.holdsSlot()) {
|
|
64
192
|
return 0;
|
|
65
193
|
}
|
|
66
194
|
return jitteredBackoffMs(failures - 1, config, rand);
|
|
@@ -79,8 +207,8 @@ function retryBackoffMs(failures, confirmedLosses, latestConfirmed, config, rand
|
|
|
79
207
|
* — but if an attempt ever grows a per-attempt footprint beyond the log blocks, reset the abandoned
|
|
80
208
|
* tracker on the stale-failure branch instead of letting the residue ride to the end of the write.
|
|
81
209
|
*
|
|
82
|
-
* The append is a fixed function of the write and `rev`: the timestamp
|
|
83
|
-
* data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
|
|
210
|
+
* The append is a fixed function of the write and `rev`: the HLC, timestamp and lane are the identity's, and
|
|
211
|
+
* any data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
|
|
84
212
|
* `Collection.logAppendBlockIds`, which replays the ids an earlier attempt at this revision minted.
|
|
85
213
|
*
|
|
86
214
|
* The declarations are computed AFTER the append so the log tail and header transforms the attempt
|
|
@@ -99,7 +227,7 @@ export async function stageAttempt(collectionId, cache, target, identity, rev, n
|
|
|
99
227
|
if (!collectionLog) {
|
|
100
228
|
throw new Error(`Log not found for collection ${collectionId}`);
|
|
101
229
|
}
|
|
102
|
-
const added = await collectionLog.addActions([...target.actions], identity.actionId, rev, () => tracker.transformedBlockIds(), { timestamp: identity.timestamp });
|
|
230
|
+
const added = await collectionLog.addActions([...target.actions], identity.actionId, rev, () => tracker.transformedBlockIds(), { timestamp: identity.timestamp, hlc: identity.hlc, lane: identity.lane });
|
|
103
231
|
const touched = await describeTouchedBlocks(tracker, tracker.transformedBlockIds());
|
|
104
232
|
const tooLarge = oversizedBlock(touched.bytes);
|
|
105
233
|
if (tooLarge)
|
|
@@ -186,7 +314,6 @@ export class AttemptCycle {
|
|
|
186
314
|
collectionId;
|
|
187
315
|
instanceTag;
|
|
188
316
|
maxAttempts;
|
|
189
|
-
maxStalledAttempts;
|
|
190
317
|
backoff;
|
|
191
318
|
deadlineMs;
|
|
192
319
|
signal;
|
|
@@ -197,24 +324,8 @@ export class AttemptCycle {
|
|
|
197
324
|
// sync (which iterates many times committing progress) never trips it.
|
|
198
325
|
consecutiveFailures = 0;
|
|
199
326
|
lastReason;
|
|
200
|
-
//
|
|
201
|
-
|
|
202
|
-
// the evidence the stall check reasons from.
|
|
203
|
-
lastStaleAt;
|
|
204
|
-
// Whether the failure most recently handled carried its OWN staleAt. A strike needs the
|
|
205
|
-
// responder to have re-confirmed the number this round, not merely an older observation
|
|
206
|
-
// left standing in `lastStaleAt`.
|
|
207
|
-
lastFailureConfirmedStaleAt = false;
|
|
208
|
-
// Refused attempts of this write that carried their own staleAt. Never reset: a member counts
|
|
209
|
-
// its stale refusals per action for the action's whole life, and this is the writer's view of
|
|
210
|
-
// that count (see retryBackoffMs).
|
|
211
|
-
confirmedLosses = 0;
|
|
212
|
-
// Consecutive refreshes that moved the requested revision nowhere at all while a confirmed
|
|
213
|
-
// revision stood at or above it.
|
|
214
|
-
consecutiveStalls = 0;
|
|
215
|
-
// The revision the PREVIOUS attempt would have requested, so the stall check can tell a
|
|
216
|
-
// refresh that moved nowhere from one that is still climbing toward the confirmed number.
|
|
217
|
-
previousRequestedRev;
|
|
327
|
+
// What the responders confirmed, and the two rules that read it (the stall check, the slot hold).
|
|
328
|
+
watch;
|
|
218
329
|
/** Who holds what the write has committed so far. Stays `undefined` while nothing has been
|
|
219
330
|
* committed, which is also the answer when no attempt runs at all — a write with nothing staged
|
|
220
331
|
* writes nothing, and there is no durability to fabricate for it. Refused attempts never
|
|
@@ -227,7 +338,7 @@ export class AttemptCycle {
|
|
|
227
338
|
this.collectionId = collectionId;
|
|
228
339
|
this.instanceTag = instanceTag;
|
|
229
340
|
this.maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
|
|
230
|
-
this.
|
|
341
|
+
this.watch = new RevisionStallWatch(options?.maxStalledAttempts ?? DefaultMaxStalledAttempts);
|
|
231
342
|
this.backoff = {
|
|
232
343
|
baseMs: options?.baseBackoffMs ?? PendingRetryDelayMs,
|
|
233
344
|
capMs: options?.maxBackoffMs ?? DefaultMaxBackoffMs,
|
|
@@ -245,27 +356,9 @@ export class AttemptCycle {
|
|
|
245
356
|
}
|
|
246
357
|
/**
|
|
247
358
|
* The checks that run before an attempt is built: the abort signal, the wall-clock deadline, and
|
|
248
|
-
* the stall check. The deadline comes before the stall check
|
|
249
|
-
* outer bound, so a write that is both past it and stalled
|
|
250
|
-
*
|
|
251
|
-
* The stall check asks: can the attempt about to run possibly differ from the one that just
|
|
252
|
-
* failed? Only when BOTH of these hold is the answer provably no:
|
|
253
|
-
*
|
|
254
|
-
* - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
|
|
255
|
-
* only after reading that revision as durably held by someone else out of its own storage,
|
|
256
|
-
* revisions are one per-collection counter that every commit touches, and a confirmed
|
|
257
|
-
* revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
|
|
258
|
-
* lost before it is sent.
|
|
259
|
-
* - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
|
|
260
|
-
* held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
|
|
261
|
-
* refresh that moved forward WITHOUT clearing the confirmed number is a collection still
|
|
262
|
-
* climbing (it read a replica that lags the holder — the same partial catch-up
|
|
263
|
-
* `reportShortfall` exists to report), and its next attempt is a genuinely different request
|
|
264
|
-
* that may yet win.
|
|
265
|
-
*
|
|
266
|
-
* This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
|
|
267
|
-
* sole rule for that, untouched. It only ever stops a loop that rule had already decided to
|
|
268
|
-
* continue.
|
|
359
|
+
* the stall check ({@link RevisionStallWatch.check}). The deadline comes before the stall check
|
|
360
|
+
* deliberately: it is the documented outer bound, so a write that is both past it and stalled
|
|
361
|
+
* reports the deadline.
|
|
269
362
|
*
|
|
270
363
|
* @param requestedRev the revision the attempt would request (`Collection.getNextRev`).
|
|
271
364
|
* @param heldRev the revision the collection holds, for the stall report.
|
|
@@ -275,32 +368,18 @@ export class AttemptCycle {
|
|
|
275
368
|
throw makeAbortError(this.signal);
|
|
276
369
|
}
|
|
277
370
|
if (this.pastDeadline()) {
|
|
278
|
-
throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason ?? 'deadline exceeded', this.
|
|
371
|
+
throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason ?? 'deadline exceeded', this.watch.staleAt);
|
|
279
372
|
}
|
|
280
|
-
const
|
|
281
|
-
|
|
282
|
-
if (this.lastStaleAt === undefined) {
|
|
373
|
+
const verdict = this.watch.check(requestedRev);
|
|
374
|
+
if (verdict.kind === 'clear') {
|
|
283
375
|
return;
|
|
284
376
|
}
|
|
285
|
-
if (
|
|
286
|
-
|
|
287
|
-
// where the rival's commit is exactly what we just read — or it made partial forward
|
|
288
|
-
// progress. Both mean the next attempt differs. Not a stall.
|
|
289
|
-
this.consecutiveStalls = 0;
|
|
377
|
+
if (log.enabled) {
|
|
378
|
+
log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d', this.collectionId, this.instanceTag, heldRev ?? 'none', requestedRev, verdict.staleAt.blockId, verdict.staleAt.rev, verdict.strike, verdict.of);
|
|
290
379
|
}
|
|
291
|
-
|
|
292
|
-
this.
|
|
293
|
-
if (log.enabled) {
|
|
294
|
-
log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d', this.collectionId, this.instanceTag, heldRev ?? 'none', requestedRev, this.lastStaleAt.blockId, this.lastStaleAt.rev, this.consecutiveStalls, this.maxStalledAttempts);
|
|
295
|
-
}
|
|
296
|
-
// Two strikes, not one: a legitimate loser can transiently read a view that has not yet
|
|
297
|
-
// caught up with the rival's commit, which looks identical for one round.
|
|
298
|
-
if (this.consecutiveStalls >= this.maxStalledAttempts) {
|
|
299
|
-
throw new SyncRevisionStalledError(this.collectionId, this.consecutiveFailures, this.lastStaleAt, requestedRev, heldRev, this.lastReason);
|
|
300
|
-
}
|
|
380
|
+
if (verdict.kind === 'stalled') {
|
|
381
|
+
throw new SyncRevisionStalledError(this.collectionId, this.consecutiveFailures, verdict.staleAt, requestedRev, heldRev, this.lastReason);
|
|
301
382
|
}
|
|
302
|
-
// Else: the responder told us nothing new this round. No strike, and no reset either — the
|
|
303
|
-
// budget stays bounded by maxAttempts.
|
|
304
383
|
}
|
|
305
384
|
/**
|
|
306
385
|
* Account for a refused attempt. The three refusals a retry can never get past are thrown by
|
|
@@ -342,15 +421,9 @@ export class AttemptCycle {
|
|
|
342
421
|
}
|
|
343
422
|
this.consecutiveFailures++;
|
|
344
423
|
this.lastReason = refusal.reason ?? this.lastReason;
|
|
345
|
-
|
|
346
|
-
// responder reporting a LOWER number understates the binding constraint. Same rule the
|
|
347
|
-
// producers and the transactor's aggregation already use.
|
|
348
|
-
this.lastStaleAt = highestStaleAt([this.lastStaleAt, refusal.staleAt]);
|
|
349
|
-
this.lastFailureConfirmedStaleAt = refusal.staleAt !== undefined;
|
|
350
|
-
if (this.lastFailureConfirmedStaleAt)
|
|
351
|
-
this.confirmedLosses++;
|
|
424
|
+
this.watch.noteRefusal(refusal.staleAt);
|
|
352
425
|
if (this.consecutiveFailures >= this.maxAttempts) {
|
|
353
|
-
throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason, this.
|
|
426
|
+
throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason, this.watch.staleAt);
|
|
354
427
|
}
|
|
355
428
|
}
|
|
356
429
|
/** Account for a commit — by an attempt, or by a refresh that finished the write's own entry,
|
|
@@ -360,9 +433,7 @@ export class AttemptCycle {
|
|
|
360
433
|
this.durability = this.durability === undefined ? durability : mergeDurability([this.durability, durability]);
|
|
361
434
|
this.consecutiveFailures = 0;
|
|
362
435
|
this.lastReason = undefined;
|
|
363
|
-
this.
|
|
364
|
-
this.lastFailureConfirmedStaleAt = false;
|
|
365
|
-
this.consecutiveStalls = 0;
|
|
436
|
+
this.watch.noteCommit();
|
|
366
437
|
}
|
|
367
438
|
/**
|
|
368
439
|
* Back off, refresh, and keep refreshing while the refresh reports that this write's own
|
|
@@ -398,18 +469,14 @@ export class AttemptCycle {
|
|
|
398
469
|
* @throws TornActionError with any reason but an unsettled `completion-refused`, and that one too
|
|
399
470
|
* once the budget or the deadline is spent; whatever else the refresh throws. */
|
|
400
471
|
async refreshUntilSettled(refresh) {
|
|
401
|
-
// Whether the failure just handled was a loss to a rival already durably committed. Only the
|
|
402
|
-
// attempt's own refusal can say so; a refresh refused for a cause that can clear (below) is a
|
|
403
|
-
// race against something still in flight, and clears it.
|
|
404
|
-
let latestConfirmed = this.lastFailureConfirmedStaleAt;
|
|
405
472
|
for (;;) {
|
|
406
473
|
const delay = this.consecutiveFailures === 0
|
|
407
474
|
? 0
|
|
408
|
-
: retryBackoffMs(this.consecutiveFailures, this.
|
|
475
|
+
: retryBackoffMs(this.consecutiveFailures, this.watch, this.backoff, this.rand);
|
|
409
476
|
await abortableDelay(delay, this.signal);
|
|
410
477
|
try {
|
|
411
478
|
const report = {};
|
|
412
|
-
await refresh(report, this.consecutiveFailures + 1 >= this.maxAttempts, this.
|
|
479
|
+
await refresh(report, this.consecutiveFailures + 1 >= this.maxAttempts, this.watch.tailFloor());
|
|
413
480
|
const completed = report.ownEntryFinished?.durability;
|
|
414
481
|
if (completed !== undefined) {
|
|
415
482
|
this.recordCommit(completed);
|
|
@@ -425,7 +492,7 @@ export class AttemptCycle {
|
|
|
425
492
|
// the log holds an entry for a write that was not saved.
|
|
426
493
|
this.consecutiveFailures++;
|
|
427
494
|
this.lastReason = err.detail;
|
|
428
|
-
|
|
495
|
+
this.watch.noteRefusedRound();
|
|
429
496
|
if (this.consecutiveFailures >= this.maxAttempts || this.pastDeadline()) {
|
|
430
497
|
throw err;
|
|
431
498
|
}
|