@optimystic/db-core 1.22.0 → 1.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -4
  2. package/dist/src/collection/action.d.ts +7 -4
  3. package/dist/src/collection/action.d.ts.map +1 -1
  4. package/dist/src/collection/action.js +13 -5
  5. package/dist/src/collection/action.js.map +1 -1
  6. package/dist/src/collection/collection.d.ts +134 -46
  7. package/dist/src/collection/collection.d.ts.map +1 -1
  8. package/dist/src/collection/collection.js +207 -70
  9. package/dist/src/collection/collection.js.map +1 -1
  10. package/dist/src/collection/struct.d.ts +38 -8
  11. package/dist/src/collection/struct.d.ts.map +1 -1
  12. package/dist/src/collection/struct.js.map +1 -1
  13. package/dist/src/collection/write-attempts.d.ts +138 -34
  14. package/dist/src/collection/write-attempts.d.ts.map +1 -1
  15. package/dist/src/collection/write-attempts.js +168 -101
  16. package/dist/src/collection/write-attempts.js.map +1 -1
  17. package/dist/src/collections/diary/diary.d.ts.map +1 -1
  18. package/dist/src/collections/diary/diary.js +5 -1
  19. package/dist/src/collections/diary/diary.js.map +1 -1
  20. package/dist/src/collections/tree/tree.d.ts +15 -9
  21. package/dist/src/collections/tree/tree.d.ts.map +1 -1
  22. package/dist/src/collections/tree/tree.js +6 -4
  23. package/dist/src/collections/tree/tree.js.map +1 -1
  24. package/dist/src/hlc/clock-store.d.ts +63 -0
  25. package/dist/src/hlc/clock-store.d.ts.map +1 -0
  26. package/dist/src/hlc/clock-store.js +49 -0
  27. package/dist/src/hlc/clock-store.js.map +1 -0
  28. package/dist/src/hlc/clock.d.ts +64 -0
  29. package/dist/src/hlc/clock.d.ts.map +1 -0
  30. package/dist/src/hlc/clock.js +171 -0
  31. package/dist/src/hlc/clock.js.map +1 -0
  32. package/dist/src/hlc/hlc.d.ts +38 -0
  33. package/dist/src/hlc/hlc.d.ts.map +1 -0
  34. package/dist/src/hlc/hlc.js +41 -0
  35. package/dist/src/hlc/hlc.js.map +1 -0
  36. package/dist/src/hlc/index.d.ts +5 -0
  37. package/dist/src/hlc/index.d.ts.map +1 -0
  38. package/dist/src/hlc/index.js +5 -0
  39. package/dist/src/hlc/index.js.map +1 -0
  40. package/dist/src/hlc/skew.d.ts +35 -0
  41. package/dist/src/hlc/skew.d.ts.map +1 -0
  42. package/dist/src/hlc/skew.js +35 -0
  43. package/dist/src/hlc/skew.js.map +1 -0
  44. package/dist/src/index.d.ts +2 -0
  45. package/dist/src/index.d.ts.map +1 -1
  46. package/dist/src/index.js +2 -0
  47. package/dist/src/index.js.map +1 -1
  48. package/dist/src/lane/index.d.ts +2 -0
  49. package/dist/src/lane/index.d.ts.map +1 -0
  50. package/dist/src/lane/index.js +2 -0
  51. package/dist/src/lane/index.js.map +1 -0
  52. package/dist/src/lane/lane.d.ts +46 -0
  53. package/dist/src/lane/lane.d.ts.map +1 -0
  54. package/dist/src/lane/lane.js +26 -0
  55. package/dist/src/lane/lane.js.map +1 -0
  56. package/dist/src/log/log.d.ts +50 -22
  57. package/dist/src/log/log.d.ts.map +1 -1
  58. package/dist/src/log/log.js +21 -13
  59. package/dist/src/log/log.js.map +1 -1
  60. package/dist/src/log/struct.d.ts +61 -1
  61. package/dist/src/log/struct.d.ts.map +1 -1
  62. package/dist/src/log/struct.js +24 -0
  63. package/dist/src/log/struct.js.map +1 -1
  64. package/dist/src/network/i-repo.d.ts +10 -0
  65. package/dist/src/network/i-repo.d.ts.map +1 -1
  66. package/dist/src/network/stale-failure.d.ts +20 -7
  67. package/dist/src/network/stale-failure.d.ts.map +1 -1
  68. package/dist/src/network/stale-failure.js +29 -7
  69. package/dist/src/network/stale-failure.js.map +1 -1
  70. package/dist/src/network/struct.d.ts +35 -0
  71. package/dist/src/network/struct.d.ts.map +1 -1
  72. package/dist/src/network/struct.js.map +1 -1
  73. package/dist/src/offline/entry.d.ts +14 -0
  74. package/dist/src/offline/entry.d.ts.map +1 -1
  75. package/dist/src/offline/entry.js +2 -0
  76. package/dist/src/offline/entry.js.map +1 -1
  77. package/dist/src/offline/journal.d.ts +10 -0
  78. package/dist/src/offline/journal.d.ts.map +1 -1
  79. package/dist/src/offline/queue.d.ts +10 -0
  80. package/dist/src/offline/queue.d.ts.map +1 -1
  81. package/dist/src/offline/queue.js +29 -3
  82. package/dist/src/offline/queue.js.map +1 -1
  83. package/dist/src/offline/status.d.ts +7 -0
  84. package/dist/src/offline/status.d.ts.map +1 -1
  85. package/dist/src/offline/status.js.map +1 -1
  86. package/dist/src/testing/test-transactor.d.ts +9 -1
  87. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  88. package/dist/src/testing/test-transactor.js +6 -1
  89. package/dist/src/testing/test-transactor.js.map +1 -1
  90. package/dist/src/transaction/actions-engine.d.ts +3 -0
  91. package/dist/src/transaction/actions-engine.d.ts.map +1 -1
  92. package/dist/src/transaction/actions-engine.js +3 -0
  93. package/dist/src/transaction/actions-engine.js.map +1 -1
  94. package/dist/src/transaction/coordinator.d.ts +10 -5
  95. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  96. package/dist/src/transaction/coordinator.js +197 -48
  97. package/dist/src/transaction/coordinator.js.map +1 -1
  98. package/dist/src/transaction/entry-commit.d.ts +45 -10
  99. package/dist/src/transaction/entry-commit.d.ts.map +1 -1
  100. package/dist/src/transaction/entry-commit.js +20 -7
  101. package/dist/src/transaction/entry-commit.js.map +1 -1
  102. package/dist/src/transaction/errors.d.ts +69 -1
  103. package/dist/src/transaction/errors.d.ts.map +1 -1
  104. package/dist/src/transaction/errors.js +70 -1
  105. package/dist/src/transaction/errors.js.map +1 -1
  106. package/dist/src/transaction/index.d.ts +1 -1
  107. package/dist/src/transaction/index.d.ts.map +1 -1
  108. package/dist/src/transaction/index.js +1 -1
  109. package/dist/src/transaction/index.js.map +1 -1
  110. package/dist/src/transaction/session.d.ts.map +1 -1
  111. package/dist/src/transaction/session.js +18 -5
  112. package/dist/src/transaction/session.js.map +1 -1
  113. package/dist/src/transaction/transaction.d.ts +74 -15
  114. package/dist/src/transaction/transaction.d.ts.map +1 -1
  115. package/dist/src/transaction/transaction.js +35 -15
  116. package/dist/src/transaction/transaction.js.map +1 -1
  117. package/dist/src/transaction/validator.d.ts +25 -4
  118. package/dist/src/transaction/validator.d.ts.map +1 -1
  119. package/dist/src/transaction/validator.js +45 -5
  120. package/dist/src/transaction/validator.js.map +1 -1
  121. package/dist/src/transactor/block-floors.d.ts +12 -1
  122. package/dist/src/transactor/block-floors.d.ts.map +1 -1
  123. package/dist/src/transactor/block-floors.js +14 -1
  124. package/dist/src/transactor/block-floors.js.map +1 -1
  125. package/dist/src/transactor/network-transactor.d.ts +8 -0
  126. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  127. package/dist/src/transactor/network-transactor.js +8 -4
  128. package/dist/src/transactor/network-transactor.js.map +1 -1
  129. package/dist/src/transactor/transactor-source.d.ts +11 -0
  130. package/dist/src/transactor/transactor-source.d.ts.map +1 -1
  131. package/dist/src/transactor/transactor-source.js +5 -1
  132. package/dist/src/transactor/transactor-source.js.map +1 -1
  133. package/dist/src/transactor/transactor.d.ts +8 -0
  134. package/dist/src/transactor/transactor.d.ts.map +1 -1
  135. package/dist/src/transform/cache-source.d.ts +7 -0
  136. package/dist/src/transform/cache-source.d.ts.map +1 -1
  137. package/dist/src/transform/cache-source.js +14 -18
  138. package/dist/src/transform/cache-source.js.map +1 -1
  139. package/package.json +1 -1
  140. package/src/collection/action.ts +13 -5
  141. package/src/collection/collection.ts +250 -76
  142. package/src/collection/struct.ts +38 -8
  143. package/src/collection/write-attempts.ts +213 -109
  144. package/src/collections/diary/diary.ts +5 -1
  145. package/src/collections/tree/tree.ts +18 -11
  146. package/src/hlc/clock-store.ts +68 -0
  147. package/src/hlc/clock.ts +196 -0
  148. package/src/hlc/hlc.ts +56 -0
  149. package/src/hlc/index.ts +4 -0
  150. package/src/hlc/skew.ts +55 -0
  151. package/src/index.ts +2 -0
  152. package/src/lane/index.ts +1 -0
  153. package/src/lane/lane.ts +54 -0
  154. package/src/log/log.ts +341 -310
  155. package/src/log/struct.ts +62 -1
  156. package/src/network/i-repo.ts +11 -1
  157. package/src/network/stale-failure.ts +31 -7
  158. package/src/network/struct.ts +805 -770
  159. package/src/offline/entry.ts +16 -0
  160. package/src/offline/journal.ts +10 -0
  161. package/src/offline/queue.ts +38 -4
  162. package/src/offline/status.ts +7 -0
  163. package/src/testing/test-transactor.ts +1065 -1053
  164. package/src/transaction/actions-engine.ts +3 -0
  165. package/src/transaction/coordinator.ts +229 -54
  166. package/src/transaction/entry-commit.ts +56 -14
  167. package/src/transaction/errors.ts +60 -0
  168. package/src/transaction/index.ts +1 -1
  169. package/src/transaction/session.ts +18 -5
  170. package/src/transaction/transaction.ts +90 -17
  171. package/src/transaction/validator.ts +46 -5
  172. package/src/transactor/block-floors.ts +15 -1
  173. package/src/transactor/network-transactor.ts +15 -4
  174. package/src/transactor/transactor-source.ts +15 -1
  175. package/src/transactor/transactor.ts +9 -0
  176. package/src/transform/cache-source.ts +15 -17
@@ -1,17 +1,144 @@
1
1
  import type { IBlock, BlockId, Action, ActionId, BlockContentDigests, BlockStore, Transforms, WriteDurability, WriteFailure } from "../index.js";
2
- import type { BlockBaseRevs } from "../network/struct.js";
2
+ import type { BlockBaseRevs, StaleFailure } from "../network/struct.js";
3
+ import type { Hlc } from "../hlc/hlc.js";
4
+ import type { Lane } from "../lane/lane.js";
3
5
  import { Tracker } from "../transform/tracker.js";
4
6
  import type { BasePins } from "../transform/base-pins.js";
5
7
  import type { CacheSource } from "../transform/cache-source.js";
6
8
  import type { CollectionId, InFlightAttempt, SyncOptions } from "./struct.js";
7
9
  import type { RefreshReport } from "./collection.js";
8
- /** What a write is identified as on the wire: the action id every attempt of it reuses, and the
9
- * log entry's timestamp. Both are minted once per WRITE, never per attempt, so a retry at the same
10
- * revision appends the same entry (docs/internals.md, "A write's log append is a fixed function of
11
- * the write and the revision it is requesting"). */
10
+ /** Default consecutive stalled refreshes — ones that moved this collection's revision NOWHERE
11
+ * while a responder CONFIRMED a revision at or above the one being requested — before a write
12
+ * gives up. Two, not one, so a single transiently-lagging read is absorbed.
13
+ *
14
+ * NOTE: two is a judgement call, not a measurement — nothing here counts how often a legitimate
15
+ * loser reads a view that moves nowhere for two consecutive rounds. The check only strikes when the
16
+ * refresh made NO progress at all, so a client merely catching up is already excluded; if a
17
+ * spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
18
+ export declare const DefaultMaxStalledAttempts = 2;
19
+ /** A revision a responder CONFIRMED is committed under some other action — `StaleFailure.staleAt`
20
+ * when it is present. A producer sets it only after reading the revision out of its own storage, so
21
+ * it is evidence, where an absent `staleAt` is merely "no confirmed number". */
22
+ export type ConfirmedRevision = NonNullable<StaleFailure['staleAt']>;
23
+ /** What {@link RevisionStallWatch.check} says about the attempt about to run: `clear` (progress, no
24
+ * evidence, or no news this round), a `strike` counted toward the limit, or `stalled` — the strike
25
+ * that reaches it, after which the write path throws its own stalled error. */
26
+ export type StallVerdict = {
27
+ kind: 'clear';
28
+ } | {
29
+ kind: 'strike';
30
+ strike: number;
31
+ of: number;
32
+ staleAt: ConfirmedRevision;
33
+ } | {
34
+ kind: 'stalled';
35
+ strike: number;
36
+ of: number;
37
+ staleAt: ConfirmedRevision;
38
+ };
39
+ /**
40
+ * The evidence one collection's write gathers from its refusals, and the two rules that read it: the
41
+ * stall check ({@link check}) and the slot hold ({@link holdsSlot}). One instance per collection per
42
+ * write. Both retry loops keep one — {@link AttemptCycle} for `Collection.sync` and `commitHead`,
43
+ * `TransactionCoordinator.commitAttempts` one per participant — so the rule exists once and the two
44
+ * loops cannot drift (docs/internals.md, "Monotonicity is also what makes a wedged view detectable").
45
+ *
46
+ * Revisions are per-collection counters, so a confirmed revision is only ever compared against the
47
+ * counter of the collection that owns it; a caller holding a refusal that covers several collections
48
+ * attributes the number before noting it (`staleAtOwner` in network/stale-failure.ts).
49
+ */
50
+ export declare class RevisionStallWatch {
51
+ private readonly maxStalledAttempts;
52
+ private highest;
53
+ private latestRefusalConfirmed;
54
+ private latestLossConfirmed;
55
+ private confirmedLosses;
56
+ private consecutiveStalls;
57
+ private previousRequestedRev;
58
+ constructor(maxStalledAttempts: number);
59
+ /** The highest confirmed revision noted so far, for the write path's error. */
60
+ get staleAt(): ConfirmedRevision | undefined;
61
+ /** A refused attempt of this collection: its own confirmed revision, or `undefined` when the
62
+ * refusal carried none — a bare refusal, or a refusal of a sibling's batch whose number belongs
63
+ * to another collection. */
64
+ noteRefusal(staleAt: ConfirmedRevision | undefined): void;
65
+ /** A refresh round refused for a cause that can clear (a `completion-refused` `TornActionError`):
66
+ * the latest loss no longer confirms a committed rival, so the next round sleeps. The stall
67
+ * evidence is unchanged — the attempt's own refusal still said what it said. */
68
+ noteRefusedRound(): void;
69
+ /** Forward progress — an attempt committed, or a refresh finished the write's own entry: the
70
+ * stall evidence starts over. `confirmedLosses` is NOT reset: a member counts per action for the
71
+ * action's whole life. */
72
+ noteCommit(): void;
73
+ /**
74
+ * The check before an attempt that would request `requestedRev`. It asks: can the attempt about
75
+ * to run possibly differ from the one that just failed? Only when BOTH of these hold is the
76
+ * answer provably no:
77
+ *
78
+ * - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
79
+ * only after reading that revision as durably held by someone else out of its own storage,
80
+ * revisions are one per-collection counter that every commit touches, and a confirmed
81
+ * revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
82
+ * lost before it is sent.
83
+ * - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
84
+ * held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
85
+ * refresh that moved forward WITHOUT clearing the confirmed number is a collection still
86
+ * climbing (it read a replica that lags the holder — the same partial catch-up
87
+ * `reportShortfall` exists to report), and its next attempt is a genuinely different request
88
+ * that may yet win.
89
+ *
90
+ * This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
91
+ * sole rule for that, untouched. It only ever stops a loop that rule had already decided to
92
+ * continue. Called once before EVERY attempt, the first included, so the previous requested
93
+ * revision is always the one the attempt just refused asked for.
94
+ */
95
+ check(requestedRev: number): StallVerdict;
96
+ /**
97
+ * Whether the members that refused this write hold the next slot for it, so its retry should
98
+ * follow with no sleep. The jittered backoff exists to separate a herd of writers that lost the
99
+ * SAME race, so they do not re-collide on the next tick. A loss to a rival that had already
100
+ * committed (the refusal carried a confirmed `staleAt`) is not a herd, and once this write has
101
+ * taken `SlotHoldAfterLosses` such losses (`confirmedLosses`, counted over the whole write as a
102
+ * member counts its stale refusals of one action) the members that refused it hold the next slot
103
+ * for it — so sleeping only spends that hold. Both sides count the same refusals against the same
104
+ * constant, so the writer stops sleeping on the loss after which a member holds, with nothing
105
+ * said over the wire. A loss the latest refusal did not confirm (a race against something still
106
+ * in flight) keeps the jitter.
107
+ *
108
+ * NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member
109
+ * the refused pends did not reach) the write retries without sleeping and so spends the same
110
+ * attempt budget sooner, at about twice one writer's pend rate (measured in the plan stage of
111
+ * `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it
112
+ * only delayed the same give-up.
113
+ */
114
+ holdsSlot(): boolean;
115
+ /** The highest confirmed revision, as the floor of the refresh's log-tail read, so a machine
116
+ * that has not caught up to it is not the last one asked (see `Collection.updateInternal`). */
117
+ tailFloor(): number | undefined;
118
+ }
119
+ /** What a write is identified as on the wire: the action id every attempt of it reuses, the two
120
+ * per-write time values its log entry carries, the HLC and the wall-clock timestamp, and its lane.
121
+ * All four are minted once per WRITE, never per attempt, so a retry at the same revision appends the
122
+ * same entry (docs/internals.md, "A write's log append is a fixed function of the write and the
123
+ * revision it is requesting"). */
12
124
  export type WriteIdentity = {
13
125
  readonly actionId: ActionId;
126
+ /** The wall-clock time the write was minted at (`LogEntry.timestamp`). */
14
127
  readonly timestamp: number;
128
+ /** The HLC the write was minted with (`LogEntry.hlc`), from the writer's clock
129
+ * (`clockOf(transactor).now()`), and carried on every pend of it (`PendRequest.hlc`).
130
+ * NOTE: a write refused as stale is retried over a refresh that may have observed newer HLCs,
131
+ * and the retry keeps the HLC it was first minted with — the identity rule above — so its entry
132
+ * can carry an HLC below a rival it was re-staged over. Nothing is wrong today: in the strong
133
+ * lane the revision is the order and the HLC is observational. The replay stage (backlog
134
+ * `feat-replay-mode-collections-merge-divergent-histories`) commits locally without a log-tail
135
+ * retry, so it never meets the case. */
136
+ readonly hlc: Hlc;
137
+ /** The write's lane (`ActionEntry.lane`, `PendRequest.lane`, `CommitRequest.lane`): the caller's
138
+ * classification when it gave one (`SyncOptions.lane`), else the collection's own declaration
139
+ * over what the write stages (`Collection.laneOf`); the queue's for a head commit
140
+ * (`HeadCommitOptions.lane`). */
141
+ readonly lane: Lane;
15
142
  };
16
143
  /** What one attempt is built from: the actions its log entry records, the transforms they produced
17
144
  * over the committed state, and the bases those transforms were computed against. For a sync these
@@ -46,8 +173,8 @@ export type StagedAttempt = {
46
173
  * — but if an attempt ever grows a per-attempt footprint beyond the log blocks, reset the abandoned
47
174
  * tracker on the stale-failure branch instead of letting the residue ride to the end of the write.
48
175
  *
49
- * The append is a fixed function of the write and `rev`: the timestamp is the identity's, and any
50
- * data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
176
+ * The append is a fixed function of the write and `rev`: the HLC, timestamp and lane are the identity's, and
177
+ * any data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
51
178
  * `Collection.logAppendBlockIds`, which replays the ids an earlier attempt at this revision minted.
52
179
  *
53
180
  * The declarations are computed AFTER the append so the log tail and header transforms the attempt
@@ -118,7 +245,6 @@ export declare class AttemptCycle {
118
245
  private readonly collectionId;
119
246
  private readonly instanceTag;
120
247
  private readonly maxAttempts;
121
- private readonly maxStalledAttempts;
122
248
  private readonly backoff;
123
249
  private readonly deadlineMs;
124
250
  private readonly signal;
@@ -126,11 +252,7 @@ export declare class AttemptCycle {
126
252
  private readonly startedAt;
127
253
  private consecutiveFailures;
128
254
  private lastReason;
129
- private lastStaleAt;
130
- private lastFailureConfirmedStaleAt;
131
- private confirmedLosses;
132
- private consecutiveStalls;
133
- private previousRequestedRev;
255
+ private readonly watch;
134
256
  /** Who holds what the write has committed so far. Stays `undefined` while nothing has been
135
257
  * committed, which is also the answer when no attempt runs at all — a write with nothing staged
136
258
  * writes nothing, and there is no durability to fabricate for it. Refused attempts never
@@ -147,27 +269,9 @@ export declare class AttemptCycle {
147
269
  priority(): number;
148
270
  /**
149
271
  * The checks that run before an attempt is built: the abort signal, the wall-clock deadline, and
150
- * the stall check. The deadline comes before the stall check deliberately: it is the documented
151
- * outer bound, so a write that is both past it and stalled reports the deadline.
152
- *
153
- * The stall check asks: can the attempt about to run possibly differ from the one that just
154
- * failed? Only when BOTH of these hold is the answer provably no:
155
- *
156
- * - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
157
- * only after reading that revision as durably held by someone else out of its own storage,
158
- * revisions are one per-collection counter that every commit touches, and a confirmed
159
- * revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
160
- * lost before it is sent.
161
- * - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
162
- * held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
163
- * refresh that moved forward WITHOUT clearing the confirmed number is a collection still
164
- * climbing (it read a replica that lags the holder — the same partial catch-up
165
- * `reportShortfall` exists to report), and its next attempt is a genuinely different request
166
- * that may yet win.
167
- *
168
- * This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
169
- * sole rule for that, untouched. It only ever stops a loop that rule had already decided to
170
- * continue.
272
+ * the stall check ({@link RevisionStallWatch.check}). The deadline comes before the stall check
273
+ * deliberately: it is the documented outer bound, so a write that is both past it and stalled
274
+ * reports the deadline.
171
275
  *
172
276
  * @param requestedRev the revision the attempt would request (`Collection.getNextRev`).
173
277
  * @param heldRev the revision the collection holds, for the stall report.
@@ -1 +1 @@
1
- {"version":3,"file":"write-attempts.d.ts","sourceRoot":"","sources":["../../../src/collection/write-attempts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,mBAAmB,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACjJ,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAUhE,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC9E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAgErD;;;oDAGoD;AACpD,MAAM,MAAM,aAAa,GAAG;IAC3B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF;;;8CAG8C;AAC9C,MAAM,MAAM,aAAa,CAAC,OAAO,IAAI;IACpC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;CACxB,CAAC;AAEF;+FAC+F;AAC/F,MAAM,MAAM,aAAa,GAAG;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAClC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,mBAAmB,GAAG,SAAS,CAAC;IACvD,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;CACjC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;0EA2B0E;AAC1E,wBAAsB,YAAY,CAAC,OAAO,EACzC,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,EAC1B,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,EAC9B,QAAQ,EAAE,aAAa,EACvB,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,MAAM,OAAO,GAC1D,OAAO,CAAC,aAAa,CAAC,CAoBxB;AAED;;;;;;;;sDAQsD;AACtD,wBAAgB,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,GAAG,eAAe,CAS1F;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAWvI;AAED;;;iEAGiE;AACjE,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnH;;;;;;GAMG;AACH,qBAAa,YAAY;IA2CvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,WAAW;IA3C7B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAqB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IAKxC,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,UAAU,CAAqB;IAIvC,OAAO,CAAC,WAAW,CAAgD;IAInE,OAAO,CAAC,2BAA2B,CAAS;IAI5C,OAAO,CAAC,eAAe,CAAK;IAG5B,OAAO,CAAC,iBAAiB,CAAK;IAG9B,OAAO,CAAC,oBAAoB,CAAqB;IAEjD;;;;;;wFAMoF;IACpF,UAAU,EAAE,eAAe,GAAG,SAAS,CAAC;gBAGtB,YAAY,EAAE,YAAY,EAC1B,WAAW,EAAE,MAAM,EACpC,OAAO,CAAC,EAAE,WAAW;IAatB;;;mCAG+B;IAC/B,QAAQ,IAAI,MAAM;IAIlB;;;;;;;;;;;;;;;;;;;;;;;;;;2GA0BuG;IACvG,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAmCtE;;;;;;;;;;;;;;;;;;;;;;;;;;;gHA2B4G;IAC5G,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAuB1C;;qFAEiF;IACjF,YAAY,CAAC,UAAU,EAAE,eAAe,GAAG,IAAI;IAS/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qFAgCiF;IAC3E,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAmC7D,OAAO,CAAC,YAAY;CAGpB"}
1
+ {"version":3,"file":"write-attempts.d.ts","sourceRoot":"","sources":["../../../src/collection/write-attempts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,mBAAmB,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACjJ,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAUhE,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC9E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA4BrD;;;;;;;iGAOiG;AACjG,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C;;gFAEgF;AAChF,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;AAErE;;+EAE+E;AAC/E,MAAM,MAAM,YAAY,GACrB;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,GACjB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,GAC1E;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,CAAC;AAE/E;;;;;;;;;;GAUG;AACH,qBAAa,kBAAkB;IAyBlB,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IArB/C,OAAO,CAAC,OAAO,CAAgC;IAI/C,OAAO,CAAC,sBAAsB,CAAS;IAKvC,OAAO,CAAC,mBAAmB,CAAS;IAIpC,OAAO,CAAC,eAAe,CAAK;IAG5B,OAAO,CAAC,iBAAiB,CAAK;IAG9B,OAAO,CAAC,oBAAoB,CAAqB;gBAEpB,kBAAkB,EAAE,MAAM;IAEvD,+EAA+E;IAC/E,IAAI,OAAO,IAAI,iBAAiB,GAAG,SAAS,CAE3C;IAED;;gCAE4B;IAC5B,WAAW,CAAC,OAAO,EAAE,iBAAiB,GAAG,SAAS,GAAG,IAAI;IAUzD;;oFAEgF;IAChF,gBAAgB,IAAI,IAAI;IAIxB;;8BAE0B;IAC1B,UAAU,IAAI,IAAI;IAOlB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,YAAY,EAAE,MAAM,GAAG,YAAY;IAyBzC;;;;;;;;;;;;;;;;;OAiBG;IACH,SAAS,IAAI,OAAO;IAIpB;mGAC+F;IAC/F,SAAS,IAAI,MAAM,GAAG,SAAS;CAG/B;AAWD;;;;kCAIkC;AAClC,MAAM,MAAM,aAAa,GAAG;IAC3B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;;;4CAOwC;IACxC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC;IAClB;;;qCAGiC;IACjC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;CACpB,CAAC;AAEF;;;8CAG8C;AAC9C,MAAM,MAAM,aAAa,CAAC,OAAO,IAAI;IACpC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;CACxB,CAAC;AAEF;+FAC+F;AAC/F,MAAM,MAAM,aAAa,GAAG;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAClC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,mBAAmB,GAAG,SAAS,CAAC;IACvD,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;CACjC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;0EA2B0E;AAC1E,wBAAsB,YAAY,CAAC,OAAO,EACzC,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,EAC1B,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,EAC9B,QAAQ,EAAE,aAAa,EACvB,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,MAAM,OAAO,GAC1D,OAAO,CAAC,aAAa,CAAC,CAoBxB;AAED;;;;;;;;sDAQsD;AACtD,wBAAgB,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,GAAG,eAAe,CAS1F;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAWvI;AAED;;;iEAGiE;AACjE,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnH;;;;;;GAMG;AACH,qBAAa,YAAY;IA0BvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,WAAW;IA1B7B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAqB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IAKxC,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,UAAU,CAAqB;IAEvC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAE3C;;;;;;wFAMoF;IACpF,UAAU,EAAE,eAAe,GAAG,SAAS,CAAC;gBAGtB,YAAY,EAAE,YAAY,EAC1B,WAAW,EAAE,MAAM,EACpC,OAAO,CAAC,EAAE,WAAW;IAatB;;;mCAG+B;IAC/B,QAAQ,IAAI,MAAM;IAIlB;;;;;;;;2GAQuG;IACvG,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAsBtE;;;;;;;;;;;;;;;;;;;;;;;;;;;gHA2B4G;IAC5G,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAkB1C;;qFAEiF;IACjF,YAAY,CAAC,UAAU,EAAE,eAAe,GAAG,IAAI;IAO/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qFAgCiF;IAC3E,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IA+B7D,OAAO,CAAC,YAAY;CAGpB"}
@@ -40,27 +40,155 @@ const DefaultMaxBackoffMs = 5000;
40
40
  * loser reads a view that moves nowhere for two consecutive rounds. The check only strikes when the
41
41
  * refresh made NO progress at all, so a client merely catching up is already excluded; if a
42
42
  * spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
43
- const DefaultMaxStalledAttempts = 2;
43
+ export const DefaultMaxStalledAttempts = 2;
44
44
  /**
45
- * The sleep before the refresh that follows a write's `failures`-th consecutive loss.
45
+ * The evidence one collection's write gathers from its refusals, and the two rules that read it: the
46
+ * stall check ({@link check}) and the slot hold ({@link holdsSlot}). One instance per collection per
47
+ * write. Both retry loops keep one — {@link AttemptCycle} for `Collection.sync` and `commitHead`,
48
+ * `TransactionCoordinator.commitAttempts` one per participant — so the rule exists once and the two
49
+ * loops cannot drift (docs/internals.md, "Monotonicity is also what makes a wedged view detectable").
46
50
  *
47
- * The jittered backoff exists to separate a herd of writers that lost the SAME race, so they do not
48
- * re-collide on the next tick. A loss to a rival that had already committed (the refusal carried a
49
- * confirmed `staleAt`) is not a herd, and once this write has taken `SlotHoldAfterLosses` such losses
50
- * (`confirmedLosses`, counted over the whole write as a member counts its stale refusals of one
51
- * action) the members that refused it hold the next slot for it — so sleeping only spends that hold.
52
- * Both sides count the same refusals against the same constant, so the writer stops sleeping on the
53
- * loss after which a member holds, with nothing said over the wire. A loss the latest refusal did not
54
- * confirm (`latestConfirmed` false — a race against something still in flight) keeps the jitter.
55
- *
56
- * NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member the
57
- * refused pends did not reach) this retries without sleeping and so spends the same attempt budget
58
- * sooner, at about twice one writer's pend rate (measured in the plan stage of
59
- * `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it only
60
- * delayed the same give-up.
51
+ * Revisions are per-collection counters, so a confirmed revision is only ever compared against the
52
+ * counter of the collection that owns it; a caller holding a refusal that covers several collections
53
+ * attributes the number before noting it (`staleAtOwner` in network/stale-failure.ts).
61
54
  */
62
- function retryBackoffMs(failures, confirmedLosses, latestConfirmed, config, rand) {
63
- if (latestConfirmed && confirmedLosses >= SlotHoldAfterLosses) {
55
+ export class RevisionStallWatch {
56
+ maxStalledAttempts;
57
+ // Highest confirmed revision any responder has reported holding, accumulated with the
58
+ // codebase's single rule for picking among several candidates. Reported on the error, and the
59
+ // evidence the stall check reasons from.
60
+ highest;
61
+ // Whether the refusal most recently noted carried its OWN staleAt. A strike needs the responder
62
+ // to have re-confirmed the number this round, not merely an older observation left standing in
63
+ // `highest`.
64
+ latestRefusalConfirmed = false;
65
+ // Whether the failure most recently handled — the refusal, or a refresh round refused since —
66
+ // was a loss to a rival already durably committed. Only the attempt's own refusal can say so; a
67
+ // round refused for a cause that can clear is a race against something still in flight. Read by
68
+ // the slot hold alone, where `latestRefusalConfirmed` is the stall check's.
69
+ latestLossConfirmed = false;
70
+ // Refused attempts of this write that carried their own staleAt. Never reset: a member counts
71
+ // its stale refusals per action for the action's whole life, and this is the writer's view of
72
+ // that count (see holdsSlot).
73
+ confirmedLosses = 0;
74
+ // Consecutive refreshes that moved the requested revision nowhere at all while a confirmed
75
+ // revision stood at or above it.
76
+ consecutiveStalls = 0;
77
+ // The revision the PREVIOUS attempt would have requested, so the stall check can tell a refresh
78
+ // that moved nowhere from one that is still climbing toward the confirmed number.
79
+ previousRequestedRev;
80
+ constructor(maxStalledAttempts) {
81
+ this.maxStalledAttempts = maxStalledAttempts;
82
+ }
83
+ /** The highest confirmed revision noted so far, for the write path's error. */
84
+ get staleAt() {
85
+ return this.highest;
86
+ }
87
+ /** A refused attempt of this collection: its own confirmed revision, or `undefined` when the
88
+ * refusal carried none — a bare refusal, or a refusal of a sibling's batch whose number belongs
89
+ * to another collection. */
90
+ noteRefusal(staleAt) {
91
+ // Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
92
+ // responder reporting a LOWER number understates the binding constraint. Same rule the
93
+ // producers and the transactor's aggregation already use.
94
+ this.highest = highestStaleAt([this.highest, staleAt]);
95
+ this.latestRefusalConfirmed = staleAt !== undefined;
96
+ this.latestLossConfirmed = this.latestRefusalConfirmed;
97
+ if (this.latestRefusalConfirmed)
98
+ this.confirmedLosses++;
99
+ }
100
+ /** A refresh round refused for a cause that can clear (a `completion-refused` `TornActionError`):
101
+ * the latest loss no longer confirms a committed rival, so the next round sleeps. The stall
102
+ * evidence is unchanged — the attempt's own refusal still said what it said. */
103
+ noteRefusedRound() {
104
+ this.latestLossConfirmed = false;
105
+ }
106
+ /** Forward progress — an attempt committed, or a refresh finished the write's own entry: the
107
+ * stall evidence starts over. `confirmedLosses` is NOT reset: a member counts per action for the
108
+ * action's whole life. */
109
+ noteCommit() {
110
+ this.highest = undefined;
111
+ this.latestRefusalConfirmed = false;
112
+ this.latestLossConfirmed = false;
113
+ this.consecutiveStalls = 0;
114
+ }
115
+ /**
116
+ * The check before an attempt that would request `requestedRev`. It asks: can the attempt about
117
+ * to run possibly differ from the one that just failed? Only when BOTH of these hold is the
118
+ * answer provably no:
119
+ *
120
+ * - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
121
+ * only after reading that revision as durably held by someone else out of its own storage,
122
+ * revisions are one per-collection counter that every commit touches, and a confirmed
123
+ * revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
124
+ * lost before it is sent.
125
+ * - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
126
+ * held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
127
+ * refresh that moved forward WITHOUT clearing the confirmed number is a collection still
128
+ * climbing (it read a replica that lags the holder — the same partial catch-up
129
+ * `reportShortfall` exists to report), and its next attempt is a genuinely different request
130
+ * that may yet win.
131
+ *
132
+ * This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
133
+ * sole rule for that, untouched. It only ever stops a loop that rule had already decided to
134
+ * continue. Called once before EVERY attempt, the first included, so the previous requested
135
+ * revision is always the one the attempt just refused asked for.
136
+ */
137
+ check(requestedRev) {
138
+ const refreshMoved = this.previousRequestedRev !== undefined && requestedRev > this.previousRequestedRev;
139
+ this.previousRequestedRev = requestedRev;
140
+ if (this.highest === undefined) {
141
+ return { kind: 'clear' };
142
+ }
143
+ if (requestedRev > this.highest.rev || refreshMoved) {
144
+ // Either the refresh adopted a revision above the confirmed one — ordinary contention,
145
+ // where the rival's commit is exactly what we just read — or it made partial forward
146
+ // progress. Both mean the next attempt differs. Not a stall.
147
+ this.consecutiveStalls = 0;
148
+ return { kind: 'clear' };
149
+ }
150
+ if (!this.latestRefusalConfirmed) {
151
+ // The responder told us nothing new this round. No strike, and no reset either — the
152
+ // budget stays bounded by maxAttempts.
153
+ return { kind: 'clear' };
154
+ }
155
+ this.consecutiveStalls++;
156
+ // Two strikes, not one: a legitimate loser can transiently read a view that has not yet
157
+ // caught up with the rival's commit, which looks identical for one round.
158
+ const kind = this.consecutiveStalls >= this.maxStalledAttempts ? 'stalled' : 'strike';
159
+ return { kind, strike: this.consecutiveStalls, of: this.maxStalledAttempts, staleAt: this.highest };
160
+ }
161
+ /**
162
+ * Whether the members that refused this write hold the next slot for it, so its retry should
163
+ * follow with no sleep. The jittered backoff exists to separate a herd of writers that lost the
164
+ * SAME race, so they do not re-collide on the next tick. A loss to a rival that had already
165
+ * committed (the refusal carried a confirmed `staleAt`) is not a herd, and once this write has
166
+ * taken `SlotHoldAfterLosses` such losses (`confirmedLosses`, counted over the whole write as a
167
+ * member counts its stale refusals of one action) the members that refused it hold the next slot
168
+ * for it — so sleeping only spends that hold. Both sides count the same refusals against the same
169
+ * constant, so the writer stops sleeping on the loss after which a member holds, with nothing
170
+ * said over the wire. A loss the latest refusal did not confirm (a race against something still
171
+ * in flight) keeps the jitter.
172
+ *
173
+ * NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member
174
+ * the refused pends did not reach) the write retries without sleeping and so spends the same
175
+ * attempt budget sooner, at about twice one writer's pend rate (measured in the plan stage of
176
+ * `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it
177
+ * only delayed the same give-up.
178
+ */
179
+ holdsSlot() {
180
+ return this.latestLossConfirmed && this.confirmedLosses >= SlotHoldAfterLosses;
181
+ }
182
+ /** The highest confirmed revision, as the floor of the refresh's log-tail read, so a machine
183
+ * that has not caught up to it is not the last one asked (see `Collection.updateInternal`). */
184
+ tailFloor() {
185
+ return this.highest?.rev;
186
+ }
187
+ }
188
+ /** The sleep before the refresh that follows a write's `failures`-th consecutive loss: none while a
189
+ * member holds the next slot for this write ({@link RevisionStallWatch.holdsSlot}), else jittered. */
190
+ function retryBackoffMs(failures, watch, config, rand) {
191
+ if (watch.holdsSlot()) {
64
192
  return 0;
65
193
  }
66
194
  return jitteredBackoffMs(failures - 1, config, rand);
@@ -79,8 +207,8 @@ function retryBackoffMs(failures, confirmedLosses, latestConfirmed, config, rand
79
207
  * — but if an attempt ever grows a per-attempt footprint beyond the log blocks, reset the abandoned
80
208
  * tracker on the stale-failure branch instead of letting the residue ride to the end of the write.
81
209
  *
82
- * The append is a fixed function of the write and `rev`: the timestamp is the identity's, and any
83
- * data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
210
+ * The append is a fixed function of the write and `rev`: the HLC, timestamp and lane are the identity's, and
211
+ * any data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
84
212
  * `Collection.logAppendBlockIds`, which replays the ids an earlier attempt at this revision minted.
85
213
  *
86
214
  * The declarations are computed AFTER the append so the log tail and header transforms the attempt
@@ -99,7 +227,7 @@ export async function stageAttempt(collectionId, cache, target, identity, rev, n
99
227
  if (!collectionLog) {
100
228
  throw new Error(`Log not found for collection ${collectionId}`);
101
229
  }
102
- const added = await collectionLog.addActions([...target.actions], identity.actionId, rev, () => tracker.transformedBlockIds(), { timestamp: identity.timestamp });
230
+ const added = await collectionLog.addActions([...target.actions], identity.actionId, rev, () => tracker.transformedBlockIds(), { timestamp: identity.timestamp, hlc: identity.hlc, lane: identity.lane });
103
231
  const touched = await describeTouchedBlocks(tracker, tracker.transformedBlockIds());
104
232
  const tooLarge = oversizedBlock(touched.bytes);
105
233
  if (tooLarge)
@@ -186,7 +314,6 @@ export class AttemptCycle {
186
314
  collectionId;
187
315
  instanceTag;
188
316
  maxAttempts;
189
- maxStalledAttempts;
190
317
  backoff;
191
318
  deadlineMs;
192
319
  signal;
@@ -197,24 +324,8 @@ export class AttemptCycle {
197
324
  // sync (which iterates many times committing progress) never trips it.
198
325
  consecutiveFailures = 0;
199
326
  lastReason;
200
- // Highest confirmed revision any responder has reported holding, accumulated with the
201
- // codebase's single rule for picking among several candidates. Reported on the error, and
202
- // the evidence the stall check reasons from.
203
- lastStaleAt;
204
- // Whether the failure most recently handled carried its OWN staleAt. A strike needs the
205
- // responder to have re-confirmed the number this round, not merely an older observation
206
- // left standing in `lastStaleAt`.
207
- lastFailureConfirmedStaleAt = false;
208
- // Refused attempts of this write that carried their own staleAt. Never reset: a member counts
209
- // its stale refusals per action for the action's whole life, and this is the writer's view of
210
- // that count (see retryBackoffMs).
211
- confirmedLosses = 0;
212
- // Consecutive refreshes that moved the requested revision nowhere at all while a confirmed
213
- // revision stood at or above it.
214
- consecutiveStalls = 0;
215
- // The revision the PREVIOUS attempt would have requested, so the stall check can tell a
216
- // refresh that moved nowhere from one that is still climbing toward the confirmed number.
217
- previousRequestedRev;
327
+ // What the responders confirmed, and the two rules that read it (the stall check, the slot hold).
328
+ watch;
218
329
  /** Who holds what the write has committed so far. Stays `undefined` while nothing has been
219
330
  * committed, which is also the answer when no attempt runs at all — a write with nothing staged
220
331
  * writes nothing, and there is no durability to fabricate for it. Refused attempts never
@@ -227,7 +338,7 @@ export class AttemptCycle {
227
338
  this.collectionId = collectionId;
228
339
  this.instanceTag = instanceTag;
229
340
  this.maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
230
- this.maxStalledAttempts = options?.maxStalledAttempts ?? DefaultMaxStalledAttempts;
341
+ this.watch = new RevisionStallWatch(options?.maxStalledAttempts ?? DefaultMaxStalledAttempts);
231
342
  this.backoff = {
232
343
  baseMs: options?.baseBackoffMs ?? PendingRetryDelayMs,
233
344
  capMs: options?.maxBackoffMs ?? DefaultMaxBackoffMs,
@@ -245,27 +356,9 @@ export class AttemptCycle {
245
356
  }
246
357
  /**
247
358
  * The checks that run before an attempt is built: the abort signal, the wall-clock deadline, and
248
- * the stall check. The deadline comes before the stall check deliberately: it is the documented
249
- * outer bound, so a write that is both past it and stalled reports the deadline.
250
- *
251
- * The stall check asks: can the attempt about to run possibly differ from the one that just
252
- * failed? Only when BOTH of these hold is the answer provably no:
253
- *
254
- * - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
255
- * only after reading that revision as durably held by someone else out of its own storage,
256
- * revisions are one per-collection counter that every commit touches, and a confirmed
257
- * revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
258
- * lost before it is sent.
259
- * - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
260
- * held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
261
- * refresh that moved forward WITHOUT clearing the confirmed number is a collection still
262
- * climbing (it read a replica that lags the holder — the same partial catch-up
263
- * `reportShortfall` exists to report), and its next attempt is a genuinely different request
264
- * that may yet win.
265
- *
266
- * This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
267
- * sole rule for that, untouched. It only ever stops a loop that rule had already decided to
268
- * continue.
359
+ * the stall check ({@link RevisionStallWatch.check}). The deadline comes before the stall check
360
+ * deliberately: it is the documented outer bound, so a write that is both past it and stalled
361
+ * reports the deadline.
269
362
  *
270
363
  * @param requestedRev the revision the attempt would request (`Collection.getNextRev`).
271
364
  * @param heldRev the revision the collection holds, for the stall report.
@@ -275,32 +368,18 @@ export class AttemptCycle {
275
368
  throw makeAbortError(this.signal);
276
369
  }
277
370
  if (this.pastDeadline()) {
278
- throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason ?? 'deadline exceeded', this.lastStaleAt);
371
+ throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason ?? 'deadline exceeded', this.watch.staleAt);
279
372
  }
280
- const refreshMoved = this.previousRequestedRev !== undefined && requestedRev > this.previousRequestedRev;
281
- this.previousRequestedRev = requestedRev;
282
- if (this.lastStaleAt === undefined) {
373
+ const verdict = this.watch.check(requestedRev);
374
+ if (verdict.kind === 'clear') {
283
375
  return;
284
376
  }
285
- if (requestedRev > this.lastStaleAt.rev || refreshMoved) {
286
- // Either the refresh adopted a revision above the confirmed one — ordinary contention,
287
- // where the rival's commit is exactly what we just read — or it made partial forward
288
- // progress. Both mean the next attempt differs. Not a stall.
289
- this.consecutiveStalls = 0;
377
+ if (log.enabled) {
378
+ log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d', this.collectionId, this.instanceTag, heldRev ?? 'none', requestedRev, verdict.staleAt.blockId, verdict.staleAt.rev, verdict.strike, verdict.of);
290
379
  }
291
- else if (this.lastFailureConfirmedStaleAt) {
292
- this.consecutiveStalls++;
293
- if (log.enabled) {
294
- log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d', this.collectionId, this.instanceTag, heldRev ?? 'none', requestedRev, this.lastStaleAt.blockId, this.lastStaleAt.rev, this.consecutiveStalls, this.maxStalledAttempts);
295
- }
296
- // Two strikes, not one: a legitimate loser can transiently read a view that has not yet
297
- // caught up with the rival's commit, which looks identical for one round.
298
- if (this.consecutiveStalls >= this.maxStalledAttempts) {
299
- throw new SyncRevisionStalledError(this.collectionId, this.consecutiveFailures, this.lastStaleAt, requestedRev, heldRev, this.lastReason);
300
- }
380
+ if (verdict.kind === 'stalled') {
381
+ throw new SyncRevisionStalledError(this.collectionId, this.consecutiveFailures, verdict.staleAt, requestedRev, heldRev, this.lastReason);
301
382
  }
302
- // Else: the responder told us nothing new this round. No strike, and no reset either — the
303
- // budget stays bounded by maxAttempts.
304
383
  }
305
384
  /**
306
385
  * Account for a refused attempt. The three refusals a retry can never get past are thrown by
@@ -342,15 +421,9 @@ export class AttemptCycle {
342
421
  }
343
422
  this.consecutiveFailures++;
344
423
  this.lastReason = refusal.reason ?? this.lastReason;
345
- // Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
346
- // responder reporting a LOWER number understates the binding constraint. Same rule the
347
- // producers and the transactor's aggregation already use.
348
- this.lastStaleAt = highestStaleAt([this.lastStaleAt, refusal.staleAt]);
349
- this.lastFailureConfirmedStaleAt = refusal.staleAt !== undefined;
350
- if (this.lastFailureConfirmedStaleAt)
351
- this.confirmedLosses++;
424
+ this.watch.noteRefusal(refusal.staleAt);
352
425
  if (this.consecutiveFailures >= this.maxAttempts) {
353
- throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason, this.lastStaleAt);
426
+ throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason, this.watch.staleAt);
354
427
  }
355
428
  }
356
429
  /** Account for a commit — by an attempt, or by a refresh that finished the write's own entry,
@@ -360,9 +433,7 @@ export class AttemptCycle {
360
433
  this.durability = this.durability === undefined ? durability : mergeDurability([this.durability, durability]);
361
434
  this.consecutiveFailures = 0;
362
435
  this.lastReason = undefined;
363
- this.lastStaleAt = undefined;
364
- this.lastFailureConfirmedStaleAt = false;
365
- this.consecutiveStalls = 0;
436
+ this.watch.noteCommit();
366
437
  }
367
438
  /**
368
439
  * Back off, refresh, and keep refreshing while the refresh reports that this write's own
@@ -398,18 +469,14 @@ export class AttemptCycle {
398
469
  * @throws TornActionError with any reason but an unsettled `completion-refused`, and that one too
399
470
  * once the budget or the deadline is spent; whatever else the refresh throws. */
400
471
  async refreshUntilSettled(refresh) {
401
- // Whether the failure just handled was a loss to a rival already durably committed. Only the
402
- // attempt's own refusal can say so; a refresh refused for a cause that can clear (below) is a
403
- // race against something still in flight, and clears it.
404
- let latestConfirmed = this.lastFailureConfirmedStaleAt;
405
472
  for (;;) {
406
473
  const delay = this.consecutiveFailures === 0
407
474
  ? 0
408
- : retryBackoffMs(this.consecutiveFailures, this.confirmedLosses, latestConfirmed, this.backoff, this.rand);
475
+ : retryBackoffMs(this.consecutiveFailures, this.watch, this.backoff, this.rand);
409
476
  await abortableDelay(delay, this.signal);
410
477
  try {
411
478
  const report = {};
412
- await refresh(report, this.consecutiveFailures + 1 >= this.maxAttempts, this.lastStaleAt?.rev);
479
+ await refresh(report, this.consecutiveFailures + 1 >= this.maxAttempts, this.watch.tailFloor());
413
480
  const completed = report.ownEntryFinished?.durability;
414
481
  if (completed !== undefined) {
415
482
  this.recordCommit(completed);
@@ -425,7 +492,7 @@ export class AttemptCycle {
425
492
  // the log holds an entry for a write that was not saved.
426
493
  this.consecutiveFailures++;
427
494
  this.lastReason = err.detail;
428
- latestConfirmed = false;
495
+ this.watch.noteRefusedRound();
429
496
  if (this.consecutiveFailures >= this.maxAttempts || this.pastDeadline()) {
430
497
  throw err;
431
498
  }