@optimystic/db-core 1.21.0 → 1.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/README.md +8 -4
  2. package/dist/src/btree/btree.d.ts +36 -4
  3. package/dist/src/btree/btree.d.ts.map +1 -1
  4. package/dist/src/btree/btree.js +86 -18
  5. package/dist/src/btree/btree.js.map +1 -1
  6. package/dist/src/collection/action.d.ts +7 -4
  7. package/dist/src/collection/action.d.ts.map +1 -1
  8. package/dist/src/collection/action.js +13 -5
  9. package/dist/src/collection/action.js.map +1 -1
  10. package/dist/src/collection/collection.d.ts +177 -51
  11. package/dist/src/collection/collection.d.ts.map +1 -1
  12. package/dist/src/collection/collection.js +265 -81
  13. package/dist/src/collection/collection.js.map +1 -1
  14. package/dist/src/collection/struct.d.ts +44 -8
  15. package/dist/src/collection/struct.d.ts.map +1 -1
  16. package/dist/src/collection/struct.js.map +1 -1
  17. package/dist/src/collection/write-attempts.d.ts +138 -34
  18. package/dist/src/collection/write-attempts.d.ts.map +1 -1
  19. package/dist/src/collection/write-attempts.js +168 -101
  20. package/dist/src/collection/write-attempts.js.map +1 -1
  21. package/dist/src/collections/diary/diary.d.ts.map +1 -1
  22. package/dist/src/collections/diary/diary.js +5 -1
  23. package/dist/src/collections/diary/diary.js.map +1 -1
  24. package/dist/src/collections/tree/tree.d.ts +18 -10
  25. package/dist/src/collections/tree/tree.d.ts.map +1 -1
  26. package/dist/src/collections/tree/tree.js +17 -9
  27. package/dist/src/collections/tree/tree.js.map +1 -1
  28. package/dist/src/hlc/clock-store.d.ts +63 -0
  29. package/dist/src/hlc/clock-store.d.ts.map +1 -0
  30. package/dist/src/hlc/clock-store.js +49 -0
  31. package/dist/src/hlc/clock-store.js.map +1 -0
  32. package/dist/src/hlc/clock.d.ts +64 -0
  33. package/dist/src/hlc/clock.d.ts.map +1 -0
  34. package/dist/src/hlc/clock.js +171 -0
  35. package/dist/src/hlc/clock.js.map +1 -0
  36. package/dist/src/hlc/hlc.d.ts +38 -0
  37. package/dist/src/hlc/hlc.d.ts.map +1 -0
  38. package/dist/src/hlc/hlc.js +41 -0
  39. package/dist/src/hlc/hlc.js.map +1 -0
  40. package/dist/src/hlc/index.d.ts +5 -0
  41. package/dist/src/hlc/index.d.ts.map +1 -0
  42. package/dist/src/hlc/index.js +5 -0
  43. package/dist/src/hlc/index.js.map +1 -0
  44. package/dist/src/hlc/skew.d.ts +35 -0
  45. package/dist/src/hlc/skew.d.ts.map +1 -0
  46. package/dist/src/hlc/skew.js +35 -0
  47. package/dist/src/hlc/skew.js.map +1 -0
  48. package/dist/src/index.d.ts +2 -0
  49. package/dist/src/index.d.ts.map +1 -1
  50. package/dist/src/index.js +2 -0
  51. package/dist/src/index.js.map +1 -1
  52. package/dist/src/lane/index.d.ts +2 -0
  53. package/dist/src/lane/index.d.ts.map +1 -0
  54. package/dist/src/lane/index.js +2 -0
  55. package/dist/src/lane/index.js.map +1 -0
  56. package/dist/src/lane/lane.d.ts +46 -0
  57. package/dist/src/lane/lane.d.ts.map +1 -0
  58. package/dist/src/lane/lane.js +26 -0
  59. package/dist/src/lane/lane.js.map +1 -0
  60. package/dist/src/log/log.d.ts +50 -22
  61. package/dist/src/log/log.d.ts.map +1 -1
  62. package/dist/src/log/log.js +21 -13
  63. package/dist/src/log/log.js.map +1 -1
  64. package/dist/src/log/struct.d.ts +61 -1
  65. package/dist/src/log/struct.d.ts.map +1 -1
  66. package/dist/src/log/struct.js +24 -0
  67. package/dist/src/log/struct.js.map +1 -1
  68. package/dist/src/network/i-repo.d.ts +10 -0
  69. package/dist/src/network/i-repo.d.ts.map +1 -1
  70. package/dist/src/network/stale-failure.d.ts +20 -7
  71. package/dist/src/network/stale-failure.d.ts.map +1 -1
  72. package/dist/src/network/stale-failure.js +29 -7
  73. package/dist/src/network/stale-failure.js.map +1 -1
  74. package/dist/src/network/struct.d.ts +50 -0
  75. package/dist/src/network/struct.d.ts.map +1 -1
  76. package/dist/src/network/struct.js.map +1 -1
  77. package/dist/src/offline/entry.d.ts +14 -0
  78. package/dist/src/offline/entry.d.ts.map +1 -1
  79. package/dist/src/offline/entry.js +2 -0
  80. package/dist/src/offline/entry.js.map +1 -1
  81. package/dist/src/offline/journal.d.ts +10 -0
  82. package/dist/src/offline/journal.d.ts.map +1 -1
  83. package/dist/src/offline/queue.d.ts +10 -0
  84. package/dist/src/offline/queue.d.ts.map +1 -1
  85. package/dist/src/offline/queue.js +29 -3
  86. package/dist/src/offline/queue.js.map +1 -1
  87. package/dist/src/offline/status.d.ts +7 -0
  88. package/dist/src/offline/status.d.ts.map +1 -1
  89. package/dist/src/offline/status.js.map +1 -1
  90. package/dist/src/testing/test-transactor.d.ts +9 -1
  91. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  92. package/dist/src/testing/test-transactor.js +12 -3
  93. package/dist/src/testing/test-transactor.js.map +1 -1
  94. package/dist/src/transaction/actions-engine.d.ts +3 -0
  95. package/dist/src/transaction/actions-engine.d.ts.map +1 -1
  96. package/dist/src/transaction/actions-engine.js +3 -0
  97. package/dist/src/transaction/actions-engine.js.map +1 -1
  98. package/dist/src/transaction/coordinator.d.ts +16 -5
  99. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  100. package/dist/src/transaction/coordinator.js +203 -48
  101. package/dist/src/transaction/coordinator.js.map +1 -1
  102. package/dist/src/transaction/entry-commit.d.ts +45 -10
  103. package/dist/src/transaction/entry-commit.d.ts.map +1 -1
  104. package/dist/src/transaction/entry-commit.js +20 -7
  105. package/dist/src/transaction/entry-commit.js.map +1 -1
  106. package/dist/src/transaction/errors.d.ts +69 -1
  107. package/dist/src/transaction/errors.d.ts.map +1 -1
  108. package/dist/src/transaction/errors.js +70 -1
  109. package/dist/src/transaction/errors.js.map +1 -1
  110. package/dist/src/transaction/index.d.ts +1 -1
  111. package/dist/src/transaction/index.d.ts.map +1 -1
  112. package/dist/src/transaction/index.js +1 -1
  113. package/dist/src/transaction/index.js.map +1 -1
  114. package/dist/src/transaction/session.d.ts.map +1 -1
  115. package/dist/src/transaction/session.js +18 -5
  116. package/dist/src/transaction/session.js.map +1 -1
  117. package/dist/src/transaction/transaction.d.ts +74 -15
  118. package/dist/src/transaction/transaction.d.ts.map +1 -1
  119. package/dist/src/transaction/transaction.js +35 -15
  120. package/dist/src/transaction/transaction.js.map +1 -1
  121. package/dist/src/transaction/validator.d.ts +25 -4
  122. package/dist/src/transaction/validator.d.ts.map +1 -1
  123. package/dist/src/transaction/validator.js +45 -5
  124. package/dist/src/transaction/validator.js.map +1 -1
  125. package/dist/src/transactor/block-floors.d.ts +12 -1
  126. package/dist/src/transactor/block-floors.d.ts.map +1 -1
  127. package/dist/src/transactor/block-floors.js +14 -1
  128. package/dist/src/transactor/block-floors.js.map +1 -1
  129. package/dist/src/transactor/network-transactor.d.ts +8 -0
  130. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  131. package/dist/src/transactor/network-transactor.js +9 -4
  132. package/dist/src/transactor/network-transactor.js.map +1 -1
  133. package/dist/src/transactor/transactor-source.d.ts +11 -0
  134. package/dist/src/transactor/transactor-source.d.ts.map +1 -1
  135. package/dist/src/transactor/transactor-source.js +5 -1
  136. package/dist/src/transactor/transactor-source.js.map +1 -1
  137. package/dist/src/transactor/transactor.d.ts +8 -0
  138. package/dist/src/transactor/transactor.d.ts.map +1 -1
  139. package/dist/src/transform/cache-source.d.ts +7 -0
  140. package/dist/src/transform/cache-source.d.ts.map +1 -1
  141. package/dist/src/transform/cache-source.js +14 -18
  142. package/dist/src/transform/cache-source.js.map +1 -1
  143. package/package.json +1 -1
  144. package/src/btree/btree.ts +92 -18
  145. package/src/collection/action.ts +13 -5
  146. package/src/collection/collection.ts +3272 -3039
  147. package/src/collection/struct.ts +493 -456
  148. package/src/collection/write-attempts.ts +213 -109
  149. package/src/collections/diary/diary.ts +5 -1
  150. package/src/collections/tree/tree.ts +33 -18
  151. package/src/hlc/clock-store.ts +68 -0
  152. package/src/hlc/clock.ts +196 -0
  153. package/src/hlc/hlc.ts +56 -0
  154. package/src/hlc/index.ts +4 -0
  155. package/src/hlc/skew.ts +55 -0
  156. package/src/index.ts +2 -0
  157. package/src/lane/index.ts +1 -0
  158. package/src/lane/lane.ts +54 -0
  159. package/src/log/log.ts +341 -310
  160. package/src/log/struct.ts +62 -1
  161. package/src/network/i-repo.ts +11 -1
  162. package/src/network/stale-failure.ts +31 -7
  163. package/src/network/struct.ts +50 -0
  164. package/src/offline/entry.ts +16 -0
  165. package/src/offline/journal.ts +10 -0
  166. package/src/offline/queue.ts +38 -4
  167. package/src/offline/status.ts +7 -0
  168. package/src/testing/test-transactor.ts +1065 -1050
  169. package/src/transaction/actions-engine.ts +3 -0
  170. package/src/transaction/coordinator.ts +235 -54
  171. package/src/transaction/entry-commit.ts +56 -14
  172. package/src/transaction/errors.ts +60 -0
  173. package/src/transaction/index.ts +1 -1
  174. package/src/transaction/session.ts +18 -5
  175. package/src/transaction/transaction.ts +90 -17
  176. package/src/transaction/validator.ts +46 -5
  177. package/src/transactor/block-floors.ts +15 -1
  178. package/src/transactor/network-transactor.ts +1908 -1896
  179. package/src/transactor/transactor-source.ts +15 -1
  180. package/src/transactor/transactor.ts +9 -0
  181. package/src/transform/cache-source.ts +15 -17
@@ -1,5 +1,7 @@
1
1
  import type { IBlock, BlockId, Action, ActionId, BlockContentDigests, BlockStore, Transforms, WriteDurability, WriteFailure } from "../index.js";
2
- import type { BlockBaseRevs } from "../network/struct.js";
2
+ import type { BlockBaseRevs, StaleFailure } from "../network/struct.js";
3
+ import type { Hlc } from "../hlc/hlc.js";
4
+ import type { Lane } from "../lane/lane.js";
3
5
  import { Log } from "../log/log.js";
4
6
  import { Tracker } from "../transform/tracker.js";
5
7
  import type { BasePins } from "../transform/base-pins.js";
@@ -50,41 +52,202 @@ const DefaultMaxBackoffMs = 5000;
50
52
  * loser reads a view that moves nowhere for two consecutive rounds. The check only strikes when the
51
53
  * refresh made NO progress at all, so a client merely catching up is already excluded; if a
52
54
  * spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
53
- const DefaultMaxStalledAttempts = 2;
55
+ export const DefaultMaxStalledAttempts = 2;
56
+
57
+ /** A revision a responder CONFIRMED is committed under some other action — `StaleFailure.staleAt`
58
+ * when it is present. A producer sets it only after reading the revision out of its own storage, so
59
+ * it is evidence, where an absent `staleAt` is merely "no confirmed number". */
60
+ export type ConfirmedRevision = NonNullable<StaleFailure['staleAt']>;
61
+
62
+ /** What {@link RevisionStallWatch.check} says about the attempt about to run: `clear` (progress, no
63
+ * evidence, or no news this round), a `strike` counted toward the limit, or `stalled` — the strike
64
+ * that reaches it, after which the write path throws its own stalled error. */
65
+ export type StallVerdict =
66
+ | { kind: 'clear' }
67
+ | { kind: 'strike'; strike: number; of: number; staleAt: ConfirmedRevision }
68
+ | { kind: 'stalled'; strike: number; of: number; staleAt: ConfirmedRevision };
54
69
 
55
70
  /**
56
- * The sleep before the refresh that follows a write's `failures`-th consecutive loss.
57
- *
58
- * The jittered backoff exists to separate a herd of writers that lost the SAME race, so they do not
59
- * re-collide on the next tick. A loss to a rival that had already committed (the refusal carried a
60
- * confirmed `staleAt`) is not a herd, and once this write has taken `SlotHoldAfterLosses` such losses
61
- * (`confirmedLosses`, counted over the whole write as a member counts its stale refusals of one
62
- * action) the members that refused it hold the next slot for it — so sleeping only spends that hold.
63
- * Both sides count the same refusals against the same constant, so the writer stops sleeping on the
64
- * loss after which a member holds, with nothing said over the wire. A loss the latest refusal did not
65
- * confirm (`latestConfirmed` false — a race against something still in flight) keeps the jitter.
71
+ * The evidence one collection's write gathers from its refusals, and the two rules that read it: the
72
+ * stall check ({@link check}) and the slot hold ({@link holdsSlot}). One instance per collection per
73
+ * write. Both retry loops keep one — {@link AttemptCycle} for `Collection.sync` and `commitHead`,
74
+ * `TransactionCoordinator.commitAttempts` one per participant — so the rule exists once and the two
75
+ * loops cannot drift (docs/internals.md, "Monotonicity is also what makes a wedged view detectable").
66
76
  *
67
- * NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member the
68
- * refused pends did not reach) this retries without sleeping and so spends the same attempt budget
69
- * sooner, at about twice one writer's pend rate (measured in the plan stage of
70
- * `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it only
71
- * delayed the same give-up.
77
+ * Revisions are per-collection counters, so a confirmed revision is only ever compared against the
78
+ * counter of the collection that owns it; a caller holding a refusal that covers several collections
79
+ * attributes the number before noting it (`staleAtOwner` in network/stale-failure.ts).
72
80
  */
73
- function retryBackoffMs(failures: number, confirmedLosses: number, latestConfirmed: boolean,
74
- config: JitteredBackoffConfig, rand?: RandFn): number {
75
- if (latestConfirmed && confirmedLosses >= SlotHoldAfterLosses) {
81
+ export class RevisionStallWatch {
82
+ // Highest confirmed revision any responder has reported holding, accumulated with the
83
+ // codebase's single rule for picking among several candidates. Reported on the error, and the
84
+ // evidence the stall check reasons from.
85
+ private highest: ConfirmedRevision | undefined;
86
+ // Whether the refusal most recently noted carried its OWN staleAt. A strike needs the responder
87
+ // to have re-confirmed the number this round, not merely an older observation left standing in
88
+ // `highest`.
89
+ private latestRefusalConfirmed = false;
90
+ // Whether the failure most recently handled — the refusal, or a refresh round refused since —
91
+ // was a loss to a rival already durably committed. Only the attempt's own refusal can say so; a
92
+ // round refused for a cause that can clear is a race against something still in flight. Read by
93
+ // the slot hold alone, where `latestRefusalConfirmed` is the stall check's.
94
+ private latestLossConfirmed = false;
95
+ // Refused attempts of this write that carried their own staleAt. Never reset: a member counts
96
+ // its stale refusals per action for the action's whole life, and this is the writer's view of
97
+ // that count (see holdsSlot).
98
+ private confirmedLosses = 0;
99
+ // Consecutive refreshes that moved the requested revision nowhere at all while a confirmed
100
+ // revision stood at or above it.
101
+ private consecutiveStalls = 0;
102
+ // The revision the PREVIOUS attempt would have requested, so the stall check can tell a refresh
103
+ // that moved nowhere from one that is still climbing toward the confirmed number.
104
+ private previousRequestedRev: number | undefined;
105
+
106
+ constructor(private readonly maxStalledAttempts: number) {}
107
+
108
+ /** The highest confirmed revision noted so far, for the write path's error. */
109
+ get staleAt(): ConfirmedRevision | undefined {
110
+ return this.highest;
111
+ }
112
+
113
+ /** A refused attempt of this collection: its own confirmed revision, or `undefined` when the
114
+ * refusal carried none — a bare refusal, or a refusal of a sibling's batch whose number belongs
115
+ * to another collection. */
116
+ noteRefusal(staleAt: ConfirmedRevision | undefined): void {
117
+ // Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
118
+ // responder reporting a LOWER number understates the binding constraint. Same rule the
119
+ // producers and the transactor's aggregation already use.
120
+ this.highest = highestStaleAt([this.highest, staleAt]);
121
+ this.latestRefusalConfirmed = staleAt !== undefined;
122
+ this.latestLossConfirmed = this.latestRefusalConfirmed;
123
+ if (this.latestRefusalConfirmed) this.confirmedLosses++;
124
+ }
125
+
126
+ /** A refresh round refused for a cause that can clear (a `completion-refused` `TornActionError`):
127
+ * the latest loss no longer confirms a committed rival, so the next round sleeps. The stall
128
+ * evidence is unchanged — the attempt's own refusal still said what it said. */
129
+ noteRefusedRound(): void {
130
+ this.latestLossConfirmed = false;
131
+ }
132
+
133
+ /** Forward progress — an attempt committed, or a refresh finished the write's own entry: the
134
+ * stall evidence starts over. `confirmedLosses` is NOT reset: a member counts per action for the
135
+ * action's whole life. */
136
+ noteCommit(): void {
137
+ this.highest = undefined;
138
+ this.latestRefusalConfirmed = false;
139
+ this.latestLossConfirmed = false;
140
+ this.consecutiveStalls = 0;
141
+ }
142
+
143
+ /**
144
+ * The check before an attempt that would request `requestedRev`. It asks: can the attempt about
145
+ * to run possibly differ from the one that just failed? Only when BOTH of these hold is the
146
+ * answer provably no:
147
+ *
148
+ * - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
149
+ * only after reading that revision as durably held by someone else out of its own storage,
150
+ * revisions are one per-collection counter that every commit touches, and a confirmed
151
+ * revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
152
+ * lost before it is sent.
153
+ * - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
154
+ * held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
155
+ * refresh that moved forward WITHOUT clearing the confirmed number is a collection still
156
+ * climbing (it read a replica that lags the holder — the same partial catch-up
157
+ * `reportShortfall` exists to report), and its next attempt is a genuinely different request
158
+ * that may yet win.
159
+ *
160
+ * This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
161
+ * sole rule for that, untouched. It only ever stops a loop that rule had already decided to
162
+ * continue. Called once before EVERY attempt, the first included, so the previous requested
163
+ * revision is always the one the attempt just refused asked for.
164
+ */
165
+ check(requestedRev: number): StallVerdict {
166
+ const refreshMoved = this.previousRequestedRev !== undefined && requestedRev > this.previousRequestedRev;
167
+ this.previousRequestedRev = requestedRev;
168
+ if (this.highest === undefined) {
169
+ return { kind: 'clear' };
170
+ }
171
+ if (requestedRev > this.highest.rev || refreshMoved) {
172
+ // Either the refresh adopted a revision above the confirmed one — ordinary contention,
173
+ // where the rival's commit is exactly what we just read — or it made partial forward
174
+ // progress. Both mean the next attempt differs. Not a stall.
175
+ this.consecutiveStalls = 0;
176
+ return { kind: 'clear' };
177
+ }
178
+ if (!this.latestRefusalConfirmed) {
179
+ // The responder told us nothing new this round. No strike, and no reset either — the
180
+ // budget stays bounded by maxAttempts.
181
+ return { kind: 'clear' };
182
+ }
183
+ this.consecutiveStalls++;
184
+ // Two strikes, not one: a legitimate loser can transiently read a view that has not yet
185
+ // caught up with the rival's commit, which looks identical for one round.
186
+ const kind = this.consecutiveStalls >= this.maxStalledAttempts ? 'stalled' : 'strike';
187
+ return { kind, strike: this.consecutiveStalls, of: this.maxStalledAttempts, staleAt: this.highest };
188
+ }
189
+
190
+ /**
191
+ * Whether the members that refused this write hold the next slot for it, so its retry should
192
+ * follow with no sleep. The jittered backoff exists to separate a herd of writers that lost the
193
+ * SAME race, so they do not re-collide on the next tick. A loss to a rival that had already
194
+ * committed (the refusal carried a confirmed `staleAt`) is not a herd, and once this write has
195
+ * taken `SlotHoldAfterLosses` such losses (`confirmedLosses`, counted over the whole write as a
196
+ * member counts its stale refusals of one action) the members that refused it hold the next slot
197
+ * for it — so sleeping only spends that hold. Both sides count the same refusals against the same
198
+ * constant, so the writer stops sleeping on the loss after which a member holds, with nothing
199
+ * said over the wire. A loss the latest refusal did not confirm (a race against something still
200
+ * in flight) keeps the jitter.
201
+ *
202
+ * NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member
203
+ * the refused pends did not reach) the write retries without sleeping and so spends the same
204
+ * attempt budget sooner, at about twice one writer's pend rate (measured in the plan stage of
205
+ * `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it
206
+ * only delayed the same give-up.
207
+ */
208
+ holdsSlot(): boolean {
209
+ return this.latestLossConfirmed && this.confirmedLosses >= SlotHoldAfterLosses;
210
+ }
211
+
212
+ /** The highest confirmed revision, as the floor of the refresh's log-tail read, so a machine
213
+ * that has not caught up to it is not the last one asked (see `Collection.updateInternal`). */
214
+ tailFloor(): number | undefined {
215
+ return this.highest?.rev;
216
+ }
217
+ }
218
+
219
+ /** The sleep before the refresh that follows a write's `failures`-th consecutive loss: none while a
220
+ * member holds the next slot for this write ({@link RevisionStallWatch.holdsSlot}), else jittered. */
221
+ function retryBackoffMs(failures: number, watch: RevisionStallWatch, config: JitteredBackoffConfig, rand?: RandFn): number {
222
+ if (watch.holdsSlot()) {
76
223
  return 0;
77
224
  }
78
225
  return jitteredBackoffMs(failures - 1, config, rand);
79
226
  }
80
227
 
81
- /** What a write is identified as on the wire: the action id every attempt of it reuses, and the
82
- * log entry's timestamp. Both are minted once per WRITE, never per attempt, so a retry at the same
83
- * revision appends the same entry (docs/internals.md, "A write's log append is a fixed function of
84
- * the write and the revision it is requesting"). */
228
+ /** What a write is identified as on the wire: the action id every attempt of it reuses, the two
229
+ * per-write time values its log entry carries, the HLC and the wall-clock timestamp, and its lane.
230
+ * All four are minted once per WRITE, never per attempt, so a retry at the same revision appends the
231
+ * same entry (docs/internals.md, "A write's log append is a fixed function of the write and the
232
+ * revision it is requesting"). */
85
233
  export type WriteIdentity = {
86
234
  readonly actionId: ActionId;
235
+ /** The wall-clock time the write was minted at (`LogEntry.timestamp`). */
87
236
  readonly timestamp: number;
237
+ /** The HLC the write was minted with (`LogEntry.hlc`), from the writer's clock
238
+ * (`clockOf(transactor).now()`), and carried on every pend of it (`PendRequest.hlc`).
239
+ * NOTE: a write refused as stale is retried over a refresh that may have observed newer HLCs,
240
+ * and the retry keeps the HLC it was first minted with — the identity rule above — so its entry
241
+ * can carry an HLC below a rival it was re-staged over. Nothing is wrong today: in the strong
242
+ * lane the revision is the order and the HLC is observational. The replay stage (backlog
243
+ * `feat-replay-mode-collections-merge-divergent-histories`) commits locally without a log-tail
244
+ * retry, so it never meets the case. */
245
+ readonly hlc: Hlc;
246
+ /** The write's lane (`ActionEntry.lane`, `PendRequest.lane`, `CommitRequest.lane`): the caller's
247
+ * classification when it gave one (`SyncOptions.lane`), else the collection's own declaration
248
+ * over what the write stages (`Collection.laneOf`); the queue's for a head commit
249
+ * (`HeadCommitOptions.lane`). */
250
+ readonly lane: Lane;
88
251
  };
89
252
 
90
253
  /** What one attempt is built from: the actions its log entry records, the transforms they produced
@@ -122,8 +285,8 @@ export type StagedAttempt = {
122
285
  * — but if an attempt ever grows a per-attempt footprint beyond the log blocks, reset the abandoned
123
286
  * tracker on the stale-failure branch instead of letting the residue ride to the end of the write.
124
287
  *
125
- * The append is a fixed function of the write and `rev`: the timestamp is the identity's, and any
126
- * data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
288
+ * The append is a fixed function of the write and `rev`: the HLC, timestamp and lane are the identity's, and
289
+ * any data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
127
290
  * `Collection.logAppendBlockIds`, which replays the ids an earlier attempt at this revision minted.
128
291
  *
129
292
  * The declarations are computed AFTER the append so the log tail and header transforms the attempt
@@ -150,7 +313,7 @@ export async function stageAttempt<TAction>(
150
313
  throw new Error(`Log not found for collection ${collectionId}`);
151
314
  }
152
315
  const added = await collectionLog.addActions([...target.actions], identity.actionId, rev,
153
- () => tracker.transformedBlockIds(), { timestamp: identity.timestamp });
316
+ () => tracker.transformedBlockIds(), { timestamp: identity.timestamp, hlc: identity.hlc, lane: identity.lane });
154
317
  const touched = await describeTouchedBlocks(tracker, tracker.transformedBlockIds());
155
318
  const tooLarge = oversizedBlock(touched.bytes);
156
319
  if (tooLarge) throw tooLarge;
@@ -243,7 +406,6 @@ export type Refresh = (report: RefreshReport, lastChance: boolean, tailFloor: nu
243
406
  */
244
407
  export class AttemptCycle {
245
408
  private readonly maxAttempts: number;
246
- private readonly maxStalledAttempts: number;
247
409
  private readonly backoff: JitteredBackoffConfig;
248
410
  private readonly deadlineMs: number | undefined;
249
411
  private readonly signal: AbortSignal | undefined;
@@ -255,24 +417,8 @@ export class AttemptCycle {
255
417
  // sync (which iterates many times committing progress) never trips it.
256
418
  private consecutiveFailures = 0;
257
419
  private lastReason: string | undefined;
258
- // Highest confirmed revision any responder has reported holding, accumulated with the
259
- // codebase's single rule for picking among several candidates. Reported on the error, and
260
- // the evidence the stall check reasons from.
261
- private lastStaleAt: { blockId: BlockId; rev: number } | undefined;
262
- // Whether the failure most recently handled carried its OWN staleAt. A strike needs the
263
- // responder to have re-confirmed the number this round, not merely an older observation
264
- // left standing in `lastStaleAt`.
265
- private lastFailureConfirmedStaleAt = false;
266
- // Refused attempts of this write that carried their own staleAt. Never reset: a member counts
267
- // its stale refusals per action for the action's whole life, and this is the writer's view of
268
- // that count (see retryBackoffMs).
269
- private confirmedLosses = 0;
270
- // Consecutive refreshes that moved the requested revision nowhere at all while a confirmed
271
- // revision stood at or above it.
272
- private consecutiveStalls = 0;
273
- // The revision the PREVIOUS attempt would have requested, so the stall check can tell a
274
- // refresh that moved nowhere from one that is still climbing toward the confirmed number.
275
- private previousRequestedRev: number | undefined;
420
+ // What the responders confirmed, and the two rules that read it (the stall check, the slot hold).
421
+ private readonly watch: RevisionStallWatch;
276
422
 
277
423
  /** Who holds what the write has committed so far. Stays `undefined` while nothing has been
278
424
  * committed, which is also the answer when no attempt runs at all — a write with nothing staged
@@ -289,7 +435,7 @@ export class AttemptCycle {
289
435
  options?: SyncOptions,
290
436
  ) {
291
437
  this.maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
292
- this.maxStalledAttempts = options?.maxStalledAttempts ?? DefaultMaxStalledAttempts;
438
+ this.watch = new RevisionStallWatch(options?.maxStalledAttempts ?? DefaultMaxStalledAttempts);
293
439
  this.backoff = {
294
440
  baseMs: options?.baseBackoffMs ?? PendingRetryDelayMs,
295
441
  capMs: options?.maxBackoffMs ?? DefaultMaxBackoffMs,
@@ -309,27 +455,9 @@ export class AttemptCycle {
309
455
 
310
456
  /**
311
457
  * The checks that run before an attempt is built: the abort signal, the wall-clock deadline, and
312
- * the stall check. The deadline comes before the stall check deliberately: it is the documented
313
- * outer bound, so a write that is both past it and stalled reports the deadline.
314
- *
315
- * The stall check asks: can the attempt about to run possibly differ from the one that just
316
- * failed? Only when BOTH of these hold is the answer provably no:
317
- *
318
- * - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
319
- * only after reading that revision as durably held by someone else out of its own storage,
320
- * revisions are one per-collection counter that every commit touches, and a confirmed
321
- * revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
322
- * lost before it is sent.
323
- * - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
324
- * held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
325
- * refresh that moved forward WITHOUT clearing the confirmed number is a collection still
326
- * climbing (it read a replica that lags the holder — the same partial catch-up
327
- * `reportShortfall` exists to report), and its next attempt is a genuinely different request
328
- * that may yet win.
329
- *
330
- * This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
331
- * sole rule for that, untouched. It only ever stops a loop that rule had already decided to
332
- * continue.
458
+ * the stall check ({@link RevisionStallWatch.check}). The deadline comes before the stall check
459
+ * deliberately: it is the documented outer bound, so a write that is both past it and stalled
460
+ * reports the deadline.
333
461
  *
334
462
  * @param requestedRev the revision the attempt would request (`Collection.getNextRev`).
335
463
  * @param heldRev the revision the collection holds, for the stall report.
@@ -339,34 +467,21 @@ export class AttemptCycle {
339
467
  throw makeAbortError(this.signal);
340
468
  }
341
469
  if (this.pastDeadline()) {
342
- throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason ?? 'deadline exceeded', this.lastStaleAt);
470
+ throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason ?? 'deadline exceeded', this.watch.staleAt);
343
471
  }
344
- const refreshMoved = this.previousRequestedRev !== undefined && requestedRev > this.previousRequestedRev;
345
- this.previousRequestedRev = requestedRev;
346
- if (this.lastStaleAt === undefined) {
472
+ const verdict = this.watch.check(requestedRev);
473
+ if (verdict.kind === 'clear') {
347
474
  return;
348
475
  }
349
- if (requestedRev > this.lastStaleAt.rev || refreshMoved) {
350
- // Either the refresh adopted a revision above the confirmed one — ordinary contention,
351
- // where the rival's commit is exactly what we just read — or it made partial forward
352
- // progress. Both mean the next attempt differs. Not a stall.
353
- this.consecutiveStalls = 0;
354
- } else if (this.lastFailureConfirmedStaleAt) {
355
- this.consecutiveStalls++;
356
- if (log.enabled) {
357
- log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d',
358
- this.collectionId, this.instanceTag, heldRev ?? 'none', requestedRev,
359
- this.lastStaleAt.blockId, this.lastStaleAt.rev, this.consecutiveStalls, this.maxStalledAttempts);
360
- }
361
- // Two strikes, not one: a legitimate loser can transiently read a view that has not yet
362
- // caught up with the rival's commit, which looks identical for one round.
363
- if (this.consecutiveStalls >= this.maxStalledAttempts) {
364
- throw new SyncRevisionStalledError(this.collectionId, this.consecutiveFailures, this.lastStaleAt,
365
- requestedRev, heldRev, this.lastReason);
366
- }
476
+ if (log.enabled) {
477
+ log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d',
478
+ this.collectionId, this.instanceTag, heldRev ?? 'none', requestedRev,
479
+ verdict.staleAt.blockId, verdict.staleAt.rev, verdict.strike, verdict.of);
480
+ }
481
+ if (verdict.kind === 'stalled') {
482
+ throw new SyncRevisionStalledError(this.collectionId, this.consecutiveFailures, verdict.staleAt,
483
+ requestedRev, heldRev, this.lastReason);
367
484
  }
368
- // Else: the responder told us nothing new this round. No strike, and no reset either — the
369
- // budget stays bounded by maxAttempts.
370
485
  }
371
486
 
372
487
  /**
@@ -409,14 +524,9 @@ export class AttemptCycle {
409
524
  }
410
525
  this.consecutiveFailures++;
411
526
  this.lastReason = refusal.reason ?? this.lastReason;
412
- // Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
413
- // responder reporting a LOWER number understates the binding constraint. Same rule the
414
- // producers and the transactor's aggregation already use.
415
- this.lastStaleAt = highestStaleAt([this.lastStaleAt, refusal.staleAt]);
416
- this.lastFailureConfirmedStaleAt = refusal.staleAt !== undefined;
417
- if (this.lastFailureConfirmedStaleAt) this.confirmedLosses++;
527
+ this.watch.noteRefusal(refusal.staleAt);
418
528
  if (this.consecutiveFailures >= this.maxAttempts) {
419
- throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason, this.lastStaleAt);
529
+ throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason, this.watch.staleAt);
420
530
  }
421
531
  }
422
532
 
@@ -427,9 +537,7 @@ export class AttemptCycle {
427
537
  this.durability = this.durability === undefined ? durability : mergeDurability([this.durability, durability]);
428
538
  this.consecutiveFailures = 0;
429
539
  this.lastReason = undefined;
430
- this.lastStaleAt = undefined;
431
- this.lastFailureConfirmedStaleAt = false;
432
- this.consecutiveStalls = 0;
540
+ this.watch.noteCommit();
433
541
  }
434
542
 
435
543
  /**
@@ -466,18 +574,14 @@ export class AttemptCycle {
466
574
  * @throws TornActionError with any reason but an unsettled `completion-refused`, and that one too
467
575
  * once the budget or the deadline is spent; whatever else the refresh throws. */
468
576
  async refreshUntilSettled(refresh: Refresh): Promise<boolean> {
469
- // Whether the failure just handled was a loss to a rival already durably committed. Only the
470
- // attempt's own refusal can say so; a refresh refused for a cause that can clear (below) is a
471
- // race against something still in flight, and clears it.
472
- let latestConfirmed = this.lastFailureConfirmedStaleAt;
473
577
  for (;;) {
474
578
  const delay = this.consecutiveFailures === 0
475
579
  ? 0
476
- : retryBackoffMs(this.consecutiveFailures, this.confirmedLosses, latestConfirmed, this.backoff, this.rand);
580
+ : retryBackoffMs(this.consecutiveFailures, this.watch, this.backoff, this.rand);
477
581
  await abortableDelay(delay, this.signal);
478
582
  try {
479
583
  const report: RefreshReport = {};
480
- await refresh(report, this.consecutiveFailures + 1 >= this.maxAttempts, this.lastStaleAt?.rev);
584
+ await refresh(report, this.consecutiveFailures + 1 >= this.maxAttempts, this.watch.tailFloor());
481
585
  const completed = report.ownEntryFinished?.durability;
482
586
  if (completed !== undefined) {
483
587
  this.recordCommit(completed);
@@ -492,7 +596,7 @@ export class AttemptCycle {
492
596
  // the log holds an entry for a write that was not saved.
493
597
  this.consecutiveFailures++;
494
598
  this.lastReason = err.detail;
495
- latestConfirmed = false;
599
+ this.watch.noteRefusedRound();
496
600
  if (this.consecutiveFailures >= this.maxAttempts || this.pastDeadline()) {
497
601
  throw err;
498
602
  }
@@ -15,7 +15,11 @@ function diaryInit<TEntry>(): CollectionInitOptions<TEntry> {
15
15
  },
16
16
  createHeaderBlock: (id: BlockId, store: BlockStore<IBlock>) => ({
17
17
  header: store.createBlockHeader(DiaryHeaderBlockType, id)
18
- })
18
+ }),
19
+ // Append-only is the first natural convergent case: two machines' concurrent appends merge as
20
+ // their union (docs/partition-healing.md §I-Confluence Classification). The handler touches no
21
+ // block, so there is nothing to rebase and no `filterConflict`.
22
+ merge: { laneOf: () => 'convergent' },
19
23
  };
20
24
  }
21
25
 
@@ -1,4 +1,4 @@
1
- import { Collection, type CollectionInitOptions, type CollectionId, type CollectionSnapshot, type ReadViewOptions } from "../../collection/index.js";
1
+ import { Collection, type CollectionInitOptions, type CollectionId, type CollectionSnapshot, type ReadViewOptions, type SyncOptions } from "../../collection/index.js";
2
2
  import type { ITransactor, BlockId, BlockStore, IBlock, ActionId, WriteDurability } from "../../index.js";
3
3
  import { BTree, type Path, type KeyRange } from "../../btree/index.js";
4
4
  import { CollectionTrunk } from "./collection-trunk.js";
@@ -173,21 +173,27 @@ export interface TreeReadView<TKey, TEntry> {
173
173
  get(key: TKey): Promise<TEntry | undefined>;
174
174
  at(path: Path<TKey, TEntry>): TEntry | undefined;
175
175
  range(range: KeyRange<TKey>): AsyncIterableIterator<Path<TKey, TEntry>>;
176
+ entriesIn(range: KeyRange<TKey>): Promise<TEntry[]>;
176
177
  ascending(path: Path<TKey, TEntry>): AsyncIterableIterator<Path<TKey, TEntry>>;
177
178
  isValid(path: Path<TKey, TEntry>): boolean;
178
179
  }
179
180
 
180
181
  /** Carries the read {@link BTree} from wherever it gets built (the `createHeaderBlock`
181
- * callback on the create path, {@link Tree.attach} on the open path) to the `replace`
182
- * handler, which needs the live instance to invalidate outstanding paths. */
182
+ * callback on the create path, {@link Tree.attach} on the open path) to the collection's
183
+ * `onViewChanged` callback, which needs the live instance to invalidate outstanding paths. */
183
184
  interface BTreeHolder<TKey, TEntry> {
184
185
  btree?: BTree<TKey, TEntry>;
185
186
  }
186
187
 
187
188
  /** The collection options a tree's opener may set — the ones that are not the tree's own to
188
- * decide. Today that is the sealed-segment refusal callback the offline-sync queue installs
189
- * (see {@link CollectionInitOptions.onSealedRefused}). */
190
- export type TreeCollectionOptions = Pick<CollectionInitOptions<unknown>, 'onSealedRefused'>;
189
+ * decide: the sealed-segment refusal callback the offline-sync queue installs (see
190
+ * {@link CollectionInitOptions.onSealedRefused}), and the convergent-lane merge declaration
191
+ * ({@link CollectionInitOptions.merge}), which a raw tree user may make. `Tree` itself declares none:
192
+ * a tree is strong unless its opener says otherwise, and the SQL table over a tree never declares
193
+ * one here, because it classifies per statement and passes the lane on each write
194
+ * (`SyncOptions.lane`). */
195
+ export type TreeCollectionOptions<TKey = unknown, TEntry = unknown> =
196
+ Pick<CollectionInitOptions<TreeReplaceAction<TKey, TEntry>>, 'onSealedRefused' | 'merge'>;
191
197
 
192
198
  export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
193
199
 
@@ -212,7 +218,7 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
212
218
  /** See {@link Tree.createOrOpen}'s `nodeCapacity`. */
213
219
  nodeCapacity?: number,
214
220
  /** See {@link Tree.createOrOpen}'s `options`. */
215
- options?: TreeCollectionOptions,
221
+ options?: TreeCollectionOptions<TKey, TEntry>,
216
222
  ): Promise<Tree<TKey, TEntry> | undefined> {
217
223
  const held: BTreeHolder<TKey, TEntry> = {};
218
224
  const init = Tree.buildInit(id, keyFromEntry, compare, nodeCapacity, held, options);
@@ -238,8 +244,9 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
238
244
  * the header and read it back on reopen rather than trusting the caller to re-supply it. */
239
245
  nodeCapacity?: number,
240
246
  /** Collection options the opener sets rather than the tree: the sealed-segment refusal
241
- * callback (see {@link TreeCollectionOptions}). Omit for a tree nothing seals. */
242
- options?: TreeCollectionOptions,
247
+ * callback and the merge declaration (see {@link TreeCollectionOptions}). Omit for a strong
248
+ * tree nothing seals. */
249
+ options?: TreeCollectionOptions<TKey, TEntry>,
243
250
  ): Promise<Tree<TKey, TEntry>> {
244
251
  const held: BTreeHolder<TKey, TEntry> = {};
245
252
  const init = Tree.buildInit(id, keyFromEntry, compare, nodeCapacity, held, options);
@@ -255,7 +262,7 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
255
262
  compare: (a: TKey, b: TKey) => number,
256
263
  nodeCapacity: number | undefined,
257
264
  held: BTreeHolder<TKey, TEntry>,
258
- options: TreeCollectionOptions | undefined,
265
+ options: TreeCollectionOptions<TKey, TEntry> | undefined,
259
266
  ): CollectionInitOptions<TreeReplaceAction<TKey, TEntry>> {
260
267
  return {
261
268
  ...options,
@@ -318,13 +325,16 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
318
325
  await actionTree.deleteAt(path);
319
326
  }
320
327
  }
321
- // Mutations landed in `trx`, not the read btree, so its version counter never moved.
322
- // Bump it to invalidate any Path a caller still holds — preserving the path-invalidation
323
- // the previous in-place handler gave for free.
324
- held.btree?.invalidatePaths();
325
328
  return tally.result();
326
329
  }
327
330
  },
331
+ // Mutations land in `trx` and in the collection's cache, never through the read btree, so
332
+ // its version counter would never move on its own. The collection says when what its reads
333
+ // see changes — a staged action landing, a replay, a restore, a refresh, a commit folded in
334
+ // — and the bump invalidates every Path handed out before, and tells a point read or a
335
+ // range scan's setup part-way through a descent to read again (docs/btree.md §Reads under
336
+ // a concurrent change).
337
+ onViewChanged: () => held.btree?.invalidatePaths(),
328
338
  createHeaderBlock: (hid: BlockId, store: BlockStore<IBlock>) => { // Only called if the collection does not exist
329
339
  // Tricky bootstrapping here:
330
340
  // We need the root id to initialize the collection header, so we create the btree here.
@@ -344,7 +354,7 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
344
354
  /** Bind an opened collection to its read btree. On the create path `createHeaderBlock` already
345
355
  * built one (it needed the root id for the header); on the open path it never ran, so build it
346
356
  * over the collection's existing tracker. Either way the result is written back into `held` so
347
- * the `replace` handler's path-invalidation targets the very btree reads go through. */
357
+ * the collection's path-invalidation (`onViewChanged`) targets the very btree reads go through. */
348
358
  private static attach<TKey, TEntry>(
349
359
  collection: Collection<TreeReplaceAction<TKey, TEntry>>,
350
360
  held: BTreeHolder<TKey, TEntry>,
@@ -382,9 +392,10 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
382
392
  }
383
393
 
384
394
  /** Flush all staged (and any other pending) changes to the transactor.
385
- * Equivalent to the flush half of {@link replace}. */
386
- async sync(): Promise<WriteDurability | undefined> {
387
- return await this.collection.updateAndSync();
395
+ * Equivalent to the flush half of {@link replace}. `options` is the write's, {@link SyncOptions.lane}
396
+ * included: the SQL bridge classifies per statement and passes the fold here. */
397
+ async sync(options?: SyncOptions): Promise<WriteDurability | undefined> {
398
+ return await this.collection.updateAndSync(options);
388
399
  }
389
400
 
390
401
  /** Whether {@link sync} has anything to push — see
@@ -526,6 +537,10 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
526
537
  return this.btree.range(range);
527
538
  }
528
539
 
540
+ async entriesIn(range: KeyRange<TKey>): Promise<TEntry[]> {
541
+ return await this.btree.entriesIn(range);
542
+ }
543
+
529
544
  ascending(path: Path<TKey, TEntry>): AsyncIterableIterator<Path<TKey, TEntry>> {
530
545
  return this.btree.ascending(path);
531
546
  }
@@ -0,0 +1,68 @@
1
+ import type { Hlc } from './hlc.js';
2
+
3
+ /**
4
+ * Where an `HlcClock` keeps its high-water mark across restarts, so a node whose wall clock is
5
+ * behind at restart does not issue an HLC below ones it issued before.
6
+ *
7
+ * A binding only stores and returns the value. When to save, and what to make of the loaded value,
8
+ * is the clock's policy, so every binding behaves the same way: the clock saves on a jump and at
9
+ * most once per {@link DEFAULT_HLC_PERSIST_INTERVAL_MS} otherwise, never awaits a save on its write
10
+ * path, and resumes from {@link restartFloor}.
11
+ */
12
+ export interface HlcClockStore {
13
+ /** The mark last saved, or `undefined` when none was ever saved. */
14
+ load(): Promise<Hlc | undefined>;
15
+ /** Replace the mark. The clock has at most one call in flight; a rejection is logged by the clock and retried on its next qualifying advance. */
16
+ save(hlc: Hlc): Promise<void>;
17
+ }
18
+
19
+ /**
20
+ * How far `wallMs` must have moved past the last saved mark before the clock saves again: a node
21
+ * issuing on its own wall clock writes at most once a second under load and nothing while idle,
22
+ * while an observed HLC far ahead of local time moves it past this at once and is saved at once.
23
+ * It is also the larger part of what {@link DEFAULT_HLC_RESTART_SLACK_MS} has to cover.
24
+ */
25
+ export const DEFAULT_HLC_PERSIST_INTERVAL_MS = 1000;
26
+
27
+ /**
28
+ * How far above the stored mark a restarted clock resumes when its local clock has not already
29
+ * passed that point. It covers the HLCs issued after the stored mark: at most
30
+ * {@link DEFAULT_HLC_PERSIST_INTERVAL_MS} of wall time before the next save, plus whatever the
31
+ * binding's store had not yet made durable when the process stopped.
32
+ *
33
+ * The second part, for a db-p2p node: the clock never flushes, so its mark is durable once some
34
+ * flush of the store covers it. The longest any of a node's stores may defer that is the relaxed
35
+ * flush window's cap, `maxWindowMs`, 5000 ms by default (`resolveDurabilityWindowPolicy` in
36
+ * `packages/db-p2p/src/storage/durability-window.ts`, packages/db-p2p/docs/storage.md §6); a write
37
+ * no flush covers is synced by the write group's idle timer, one second after it. So at the defaults
38
+ * the durable mark trails what the clock issued by at most 1000 + 5000 = 6000 ms, and 10 s leaves
39
+ * 4 s of margin. A deployment that raises `maxWindowMs` past 9 s should raise this with it
40
+ * (`HlcClockStartOptions.restartSlackMs`).
41
+ *
42
+ * In the ordinary case a real restart takes longer than this and the local clock wins, so the floor
43
+ * costs nothing; it matters for a fast restart or a clock that went backwards, where the node's HLCs
44
+ * then sit up to this far ahead of its wall clock until wall time catches up.
45
+ *
46
+ * NOTE: the slack covers an unsaved stretch only while the mark tracked wall time. A mark that
47
+ * jumped — to an observed HLC far ahead, or to the restart floor itself — is saved at once but not
48
+ * flushed, so a crash inside the store's durability window loses the jump, and the next start can
49
+ * issue HLCs at or below ones the lost run issued (a crash loop faster than that window repeats one
50
+ * floor). Harmless while HLCs are observational (Stage 1 of docs/crdt-sync.md); once anything orders
51
+ * by them, the save after a jump must be durable before the clock issues past it.
52
+ *
53
+ * NOTE: the floor is saved, so each restart that comes faster than the slack resumes a further
54
+ * slack above the last one: a crash loop restarting every few seconds moves the node's HLCs ahead
55
+ * of its wall clock by about the slack less the restart interval, per restart, without bound. Fine
56
+ * while HLCs are observational; if HLCs start deciding order (Stage 2), a remedy is to wait out the
57
+ * floor at start rather than resume above it.
58
+ */
59
+ export const DEFAULT_HLC_RESTART_SLACK_MS = 10_000;
60
+
61
+ /**
62
+ * The HLC a clock restarted over `stored` treats as already observed: `slackMs` above the stored
63
+ * `wallMs`, logical 0, in this node's name. The clock only ever raises its mark, so a local clock
64
+ * already past this point simply wins.
65
+ */
66
+ export function restartFloor(stored: Hlc, slackMs: number, nodeId: string): Hlc {
67
+ return { wallMs: stored.wallMs + slackMs, logical: 0, nodeId };
68
+ }