@optimystic/db-core 1.21.0 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/dist/src/btree/btree.d.ts +36 -4
- package/dist/src/btree/btree.d.ts.map +1 -1
- package/dist/src/btree/btree.js +86 -18
- package/dist/src/btree/btree.js.map +1 -1
- package/dist/src/collection/action.d.ts +7 -4
- package/dist/src/collection/action.d.ts.map +1 -1
- package/dist/src/collection/action.js +13 -5
- package/dist/src/collection/action.js.map +1 -1
- package/dist/src/collection/collection.d.ts +177 -51
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +265 -81
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/struct.d.ts +44 -8
- package/dist/src/collection/struct.d.ts.map +1 -1
- package/dist/src/collection/struct.js.map +1 -1
- package/dist/src/collection/write-attempts.d.ts +138 -34
- package/dist/src/collection/write-attempts.d.ts.map +1 -1
- package/dist/src/collection/write-attempts.js +168 -101
- package/dist/src/collection/write-attempts.js.map +1 -1
- package/dist/src/collections/diary/diary.d.ts.map +1 -1
- package/dist/src/collections/diary/diary.js +5 -1
- package/dist/src/collections/diary/diary.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts +18 -10
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +17 -9
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/hlc/clock-store.d.ts +63 -0
- package/dist/src/hlc/clock-store.d.ts.map +1 -0
- package/dist/src/hlc/clock-store.js +49 -0
- package/dist/src/hlc/clock-store.js.map +1 -0
- package/dist/src/hlc/clock.d.ts +64 -0
- package/dist/src/hlc/clock.d.ts.map +1 -0
- package/dist/src/hlc/clock.js +171 -0
- package/dist/src/hlc/clock.js.map +1 -0
- package/dist/src/hlc/hlc.d.ts +38 -0
- package/dist/src/hlc/hlc.d.ts.map +1 -0
- package/dist/src/hlc/hlc.js +41 -0
- package/dist/src/hlc/hlc.js.map +1 -0
- package/dist/src/hlc/index.d.ts +5 -0
- package/dist/src/hlc/index.d.ts.map +1 -0
- package/dist/src/hlc/index.js +5 -0
- package/dist/src/hlc/index.js.map +1 -0
- package/dist/src/hlc/skew.d.ts +35 -0
- package/dist/src/hlc/skew.d.ts.map +1 -0
- package/dist/src/hlc/skew.js +35 -0
- package/dist/src/hlc/skew.js.map +1 -0
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/lane/index.d.ts +2 -0
- package/dist/src/lane/index.d.ts.map +1 -0
- package/dist/src/lane/index.js +2 -0
- package/dist/src/lane/index.js.map +1 -0
- package/dist/src/lane/lane.d.ts +46 -0
- package/dist/src/lane/lane.d.ts.map +1 -0
- package/dist/src/lane/lane.js +26 -0
- package/dist/src/lane/lane.js.map +1 -0
- package/dist/src/log/log.d.ts +50 -22
- package/dist/src/log/log.d.ts.map +1 -1
- package/dist/src/log/log.js +21 -13
- package/dist/src/log/log.js.map +1 -1
- package/dist/src/log/struct.d.ts +61 -1
- package/dist/src/log/struct.d.ts.map +1 -1
- package/dist/src/log/struct.js +24 -0
- package/dist/src/log/struct.js.map +1 -1
- package/dist/src/network/i-repo.d.ts +10 -0
- package/dist/src/network/i-repo.d.ts.map +1 -1
- package/dist/src/network/stale-failure.d.ts +20 -7
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +29 -7
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/network/struct.d.ts +50 -0
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/offline/entry.d.ts +14 -0
- package/dist/src/offline/entry.d.ts.map +1 -1
- package/dist/src/offline/entry.js +2 -0
- package/dist/src/offline/entry.js.map +1 -1
- package/dist/src/offline/journal.d.ts +10 -0
- package/dist/src/offline/journal.d.ts.map +1 -1
- package/dist/src/offline/queue.d.ts +10 -0
- package/dist/src/offline/queue.d.ts.map +1 -1
- package/dist/src/offline/queue.js +29 -3
- package/dist/src/offline/queue.js.map +1 -1
- package/dist/src/offline/status.d.ts +7 -0
- package/dist/src/offline/status.d.ts.map +1 -1
- package/dist/src/offline/status.js.map +1 -1
- package/dist/src/testing/test-transactor.d.ts +9 -1
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +12 -3
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/actions-engine.d.ts +3 -0
- package/dist/src/transaction/actions-engine.d.ts.map +1 -1
- package/dist/src/transaction/actions-engine.js +3 -0
- package/dist/src/transaction/actions-engine.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +16 -5
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +203 -48
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transaction/entry-commit.d.ts +45 -10
- package/dist/src/transaction/entry-commit.d.ts.map +1 -1
- package/dist/src/transaction/entry-commit.js +20 -7
- package/dist/src/transaction/entry-commit.js.map +1 -1
- package/dist/src/transaction/errors.d.ts +69 -1
- package/dist/src/transaction/errors.d.ts.map +1 -1
- package/dist/src/transaction/errors.js +70 -1
- package/dist/src/transaction/errors.js.map +1 -1
- package/dist/src/transaction/index.d.ts +1 -1
- package/dist/src/transaction/index.d.ts.map +1 -1
- package/dist/src/transaction/index.js +1 -1
- package/dist/src/transaction/index.js.map +1 -1
- package/dist/src/transaction/session.d.ts.map +1 -1
- package/dist/src/transaction/session.js +18 -5
- package/dist/src/transaction/session.js.map +1 -1
- package/dist/src/transaction/transaction.d.ts +74 -15
- package/dist/src/transaction/transaction.d.ts.map +1 -1
- package/dist/src/transaction/transaction.js +35 -15
- package/dist/src/transaction/transaction.js.map +1 -1
- package/dist/src/transaction/validator.d.ts +25 -4
- package/dist/src/transaction/validator.d.ts.map +1 -1
- package/dist/src/transaction/validator.js +45 -5
- package/dist/src/transaction/validator.js.map +1 -1
- package/dist/src/transactor/block-floors.d.ts +12 -1
- package/dist/src/transactor/block-floors.d.ts.map +1 -1
- package/dist/src/transactor/block-floors.js +14 -1
- package/dist/src/transactor/block-floors.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +8 -0
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +9 -4
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor-source.d.ts +11 -0
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +5 -1
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/dist/src/transactor/transactor.d.ts +8 -0
- package/dist/src/transactor/transactor.d.ts.map +1 -1
- package/dist/src/transform/cache-source.d.ts +7 -0
- package/dist/src/transform/cache-source.d.ts.map +1 -1
- package/dist/src/transform/cache-source.js +14 -18
- package/dist/src/transform/cache-source.js.map +1 -1
- package/package.json +1 -1
- package/src/btree/btree.ts +92 -18
- package/src/collection/action.ts +13 -5
- package/src/collection/collection.ts +3272 -3039
- package/src/collection/struct.ts +493 -456
- package/src/collection/write-attempts.ts +213 -109
- package/src/collections/diary/diary.ts +5 -1
- package/src/collections/tree/tree.ts +33 -18
- package/src/hlc/clock-store.ts +68 -0
- package/src/hlc/clock.ts +196 -0
- package/src/hlc/hlc.ts +56 -0
- package/src/hlc/index.ts +4 -0
- package/src/hlc/skew.ts +55 -0
- package/src/index.ts +2 -0
- package/src/lane/index.ts +1 -0
- package/src/lane/lane.ts +54 -0
- package/src/log/log.ts +341 -310
- package/src/log/struct.ts +62 -1
- package/src/network/i-repo.ts +11 -1
- package/src/network/stale-failure.ts +31 -7
- package/src/network/struct.ts +50 -0
- package/src/offline/entry.ts +16 -0
- package/src/offline/journal.ts +10 -0
- package/src/offline/queue.ts +38 -4
- package/src/offline/status.ts +7 -0
- package/src/testing/test-transactor.ts +1065 -1050
- package/src/transaction/actions-engine.ts +3 -0
- package/src/transaction/coordinator.ts +235 -54
- package/src/transaction/entry-commit.ts +56 -14
- package/src/transaction/errors.ts +60 -0
- package/src/transaction/index.ts +1 -1
- package/src/transaction/session.ts +18 -5
- package/src/transaction/transaction.ts +90 -17
- package/src/transaction/validator.ts +46 -5
- package/src/transactor/block-floors.ts +15 -1
- package/src/transactor/network-transactor.ts +1908 -1896
- package/src/transactor/transactor-source.ts +15 -1
- package/src/transactor/transactor.ts +9 -0
- package/src/transform/cache-source.ts +15 -17
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { IBlock, BlockId, Action, ActionId, BlockContentDigests, BlockStore, Transforms, WriteDurability, WriteFailure } from "../index.js";
|
|
2
|
-
import type { BlockBaseRevs } from "../network/struct.js";
|
|
2
|
+
import type { BlockBaseRevs, StaleFailure } from "../network/struct.js";
|
|
3
|
+
import type { Hlc } from "../hlc/hlc.js";
|
|
4
|
+
import type { Lane } from "../lane/lane.js";
|
|
3
5
|
import { Log } from "../log/log.js";
|
|
4
6
|
import { Tracker } from "../transform/tracker.js";
|
|
5
7
|
import type { BasePins } from "../transform/base-pins.js";
|
|
@@ -50,41 +52,202 @@ const DefaultMaxBackoffMs = 5000;
|
|
|
50
52
|
* loser reads a view that moves nowhere for two consecutive rounds. The check only strikes when the
|
|
51
53
|
* refresh made NO progress at all, so a client merely catching up is already excluded; if a
|
|
52
54
|
* spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
|
|
53
|
-
const DefaultMaxStalledAttempts = 2;
|
|
55
|
+
export const DefaultMaxStalledAttempts = 2;
|
|
56
|
+
|
|
57
|
+
/** A revision a responder CONFIRMED is committed under some other action — `StaleFailure.staleAt`
|
|
58
|
+
* when it is present. A producer sets it only after reading the revision out of its own storage, so
|
|
59
|
+
* it is evidence, where an absent `staleAt` is merely "no confirmed number". */
|
|
60
|
+
export type ConfirmedRevision = NonNullable<StaleFailure['staleAt']>;
|
|
61
|
+
|
|
62
|
+
/** What {@link RevisionStallWatch.check} says about the attempt about to run: `clear` (progress, no
|
|
63
|
+
* evidence, or no news this round), a `strike` counted toward the limit, or `stalled` — the strike
|
|
64
|
+
* that reaches it, after which the write path throws its own stalled error. */
|
|
65
|
+
export type StallVerdict =
|
|
66
|
+
| { kind: 'clear' }
|
|
67
|
+
| { kind: 'strike'; strike: number; of: number; staleAt: ConfirmedRevision }
|
|
68
|
+
| { kind: 'stalled'; strike: number; of: number; staleAt: ConfirmedRevision };
|
|
54
69
|
|
|
55
70
|
/**
|
|
56
|
-
* The
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
* (`confirmedLosses`, counted over the whole write as a member counts its stale refusals of one
|
|
62
|
-
* action) the members that refused it hold the next slot for it — so sleeping only spends that hold.
|
|
63
|
-
* Both sides count the same refusals against the same constant, so the writer stops sleeping on the
|
|
64
|
-
* loss after which a member holds, with nothing said over the wire. A loss the latest refusal did not
|
|
65
|
-
* confirm (`latestConfirmed` false — a race against something still in flight) keeps the jitter.
|
|
71
|
+
* The evidence one collection's write gathers from its refusals, and the two rules that read it: the
|
|
72
|
+
* stall check ({@link check}) and the slot hold ({@link holdsSlot}). One instance per collection per
|
|
73
|
+
* write. Both retry loops keep one — {@link AttemptCycle} for `Collection.sync` and `commitHead`,
|
|
74
|
+
* `TransactionCoordinator.commitAttempts` one per participant — so the rule exists once and the two
|
|
75
|
+
* loops cannot drift (docs/internals.md, "Monotonicity is also what makes a wedged view detectable").
|
|
66
76
|
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
* `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it only
|
|
71
|
-
* delayed the same give-up.
|
|
77
|
+
* Revisions are per-collection counters, so a confirmed revision is only ever compared against the
|
|
78
|
+
* counter of the collection that owns it; a caller holding a refusal that covers several collections
|
|
79
|
+
* attributes the number before noting it (`staleAtOwner` in network/stale-failure.ts).
|
|
72
80
|
*/
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
81
|
+
export class RevisionStallWatch {
|
|
82
|
+
// Highest confirmed revision any responder has reported holding, accumulated with the
|
|
83
|
+
// codebase's single rule for picking among several candidates. Reported on the error, and the
|
|
84
|
+
// evidence the stall check reasons from.
|
|
85
|
+
private highest: ConfirmedRevision | undefined;
|
|
86
|
+
// Whether the refusal most recently noted carried its OWN staleAt. A strike needs the responder
|
|
87
|
+
// to have re-confirmed the number this round, not merely an older observation left standing in
|
|
88
|
+
// `highest`.
|
|
89
|
+
private latestRefusalConfirmed = false;
|
|
90
|
+
// Whether the failure most recently handled — the refusal, or a refresh round refused since —
|
|
91
|
+
// was a loss to a rival already durably committed. Only the attempt's own refusal can say so; a
|
|
92
|
+
// round refused for a cause that can clear is a race against something still in flight. Read by
|
|
93
|
+
// the slot hold alone, where `latestRefusalConfirmed` is the stall check's.
|
|
94
|
+
private latestLossConfirmed = false;
|
|
95
|
+
// Refused attempts of this write that carried their own staleAt. Never reset: a member counts
|
|
96
|
+
// its stale refusals per action for the action's whole life, and this is the writer's view of
|
|
97
|
+
// that count (see holdsSlot).
|
|
98
|
+
private confirmedLosses = 0;
|
|
99
|
+
// Consecutive refreshes that moved the requested revision nowhere at all while a confirmed
|
|
100
|
+
// revision stood at or above it.
|
|
101
|
+
private consecutiveStalls = 0;
|
|
102
|
+
// The revision the PREVIOUS attempt would have requested, so the stall check can tell a refresh
|
|
103
|
+
// that moved nowhere from one that is still climbing toward the confirmed number.
|
|
104
|
+
private previousRequestedRev: number | undefined;
|
|
105
|
+
|
|
106
|
+
constructor(private readonly maxStalledAttempts: number) {}
|
|
107
|
+
|
|
108
|
+
/** The highest confirmed revision noted so far, for the write path's error. */
|
|
109
|
+
get staleAt(): ConfirmedRevision | undefined {
|
|
110
|
+
return this.highest;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** A refused attempt of this collection: its own confirmed revision, or `undefined` when the
|
|
114
|
+
* refusal carried none — a bare refusal, or a refusal of a sibling's batch whose number belongs
|
|
115
|
+
* to another collection. */
|
|
116
|
+
noteRefusal(staleAt: ConfirmedRevision | undefined): void {
|
|
117
|
+
// Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
|
|
118
|
+
// responder reporting a LOWER number understates the binding constraint. Same rule the
|
|
119
|
+
// producers and the transactor's aggregation already use.
|
|
120
|
+
this.highest = highestStaleAt([this.highest, staleAt]);
|
|
121
|
+
this.latestRefusalConfirmed = staleAt !== undefined;
|
|
122
|
+
this.latestLossConfirmed = this.latestRefusalConfirmed;
|
|
123
|
+
if (this.latestRefusalConfirmed) this.confirmedLosses++;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** A refresh round refused for a cause that can clear (a `completion-refused` `TornActionError`):
|
|
127
|
+
* the latest loss no longer confirms a committed rival, so the next round sleeps. The stall
|
|
128
|
+
* evidence is unchanged — the attempt's own refusal still said what it said. */
|
|
129
|
+
noteRefusedRound(): void {
|
|
130
|
+
this.latestLossConfirmed = false;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Forward progress — an attempt committed, or a refresh finished the write's own entry: the
|
|
134
|
+
* stall evidence starts over. `confirmedLosses` is NOT reset: a member counts per action for the
|
|
135
|
+
* action's whole life. */
|
|
136
|
+
noteCommit(): void {
|
|
137
|
+
this.highest = undefined;
|
|
138
|
+
this.latestRefusalConfirmed = false;
|
|
139
|
+
this.latestLossConfirmed = false;
|
|
140
|
+
this.consecutiveStalls = 0;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* The check before an attempt that would request `requestedRev`. It asks: can the attempt about
|
|
145
|
+
* to run possibly differ from the one that just failed? Only when BOTH of these hold is the
|
|
146
|
+
* answer provably no:
|
|
147
|
+
*
|
|
148
|
+
* - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
|
|
149
|
+
* only after reading that revision as durably held by someone else out of its own storage,
|
|
150
|
+
* revisions are one per-collection counter that every commit touches, and a confirmed
|
|
151
|
+
* revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
|
|
152
|
+
* lost before it is sent.
|
|
153
|
+
* - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
|
|
154
|
+
* held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
|
|
155
|
+
* refresh that moved forward WITHOUT clearing the confirmed number is a collection still
|
|
156
|
+
* climbing (it read a replica that lags the holder — the same partial catch-up
|
|
157
|
+
* `reportShortfall` exists to report), and its next attempt is a genuinely different request
|
|
158
|
+
* that may yet win.
|
|
159
|
+
*
|
|
160
|
+
* This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
|
|
161
|
+
* sole rule for that, untouched. It only ever stops a loop that rule had already decided to
|
|
162
|
+
* continue. Called once before EVERY attempt, the first included, so the previous requested
|
|
163
|
+
* revision is always the one the attempt just refused asked for.
|
|
164
|
+
*/
|
|
165
|
+
check(requestedRev: number): StallVerdict {
|
|
166
|
+
const refreshMoved = this.previousRequestedRev !== undefined && requestedRev > this.previousRequestedRev;
|
|
167
|
+
this.previousRequestedRev = requestedRev;
|
|
168
|
+
if (this.highest === undefined) {
|
|
169
|
+
return { kind: 'clear' };
|
|
170
|
+
}
|
|
171
|
+
if (requestedRev > this.highest.rev || refreshMoved) {
|
|
172
|
+
// Either the refresh adopted a revision above the confirmed one — ordinary contention,
|
|
173
|
+
// where the rival's commit is exactly what we just read — or it made partial forward
|
|
174
|
+
// progress. Both mean the next attempt differs. Not a stall.
|
|
175
|
+
this.consecutiveStalls = 0;
|
|
176
|
+
return { kind: 'clear' };
|
|
177
|
+
}
|
|
178
|
+
if (!this.latestRefusalConfirmed) {
|
|
179
|
+
// The responder told us nothing new this round. No strike, and no reset either — the
|
|
180
|
+
// budget stays bounded by maxAttempts.
|
|
181
|
+
return { kind: 'clear' };
|
|
182
|
+
}
|
|
183
|
+
this.consecutiveStalls++;
|
|
184
|
+
// Two strikes, not one: a legitimate loser can transiently read a view that has not yet
|
|
185
|
+
// caught up with the rival's commit, which looks identical for one round.
|
|
186
|
+
const kind = this.consecutiveStalls >= this.maxStalledAttempts ? 'stalled' : 'strike';
|
|
187
|
+
return { kind, strike: this.consecutiveStalls, of: this.maxStalledAttempts, staleAt: this.highest };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Whether the members that refused this write hold the next slot for it, so its retry should
|
|
192
|
+
* follow with no sleep. The jittered backoff exists to separate a herd of writers that lost the
|
|
193
|
+
* SAME race, so they do not re-collide on the next tick. A loss to a rival that had already
|
|
194
|
+
* committed (the refusal carried a confirmed `staleAt`) is not a herd, and once this write has
|
|
195
|
+
* taken `SlotHoldAfterLosses` such losses (`confirmedLosses`, counted over the whole write as a
|
|
196
|
+
* member counts its stale refusals of one action) the members that refused it hold the next slot
|
|
197
|
+
* for it — so sleeping only spends that hold. Both sides count the same refusals against the same
|
|
198
|
+
* constant, so the writer stops sleeping on the loss after which a member holds, with nothing
|
|
199
|
+
* said over the wire. A loss the latest refusal did not confirm (a race against something still
|
|
200
|
+
* in flight) keeps the jitter.
|
|
201
|
+
*
|
|
202
|
+
* NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member
|
|
203
|
+
* the refused pends did not reach) the write retries without sleeping and so spends the same
|
|
204
|
+
* attempt budget sooner, at about twice one writer's pend rate (measured in the plan stage of
|
|
205
|
+
* `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it
|
|
206
|
+
* only delayed the same give-up.
|
|
207
|
+
*/
|
|
208
|
+
holdsSlot(): boolean {
|
|
209
|
+
return this.latestLossConfirmed && this.confirmedLosses >= SlotHoldAfterLosses;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** The highest confirmed revision, as the floor of the refresh's log-tail read, so a machine
|
|
213
|
+
* that has not caught up to it is not the last one asked (see `Collection.updateInternal`). */
|
|
214
|
+
tailFloor(): number | undefined {
|
|
215
|
+
return this.highest?.rev;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** The sleep before the refresh that follows a write's `failures`-th consecutive loss: none while a
|
|
220
|
+
* member holds the next slot for this write ({@link RevisionStallWatch.holdsSlot}), else jittered. */
|
|
221
|
+
function retryBackoffMs(failures: number, watch: RevisionStallWatch, config: JitteredBackoffConfig, rand?: RandFn): number {
|
|
222
|
+
if (watch.holdsSlot()) {
|
|
76
223
|
return 0;
|
|
77
224
|
}
|
|
78
225
|
return jitteredBackoffMs(failures - 1, config, rand);
|
|
79
226
|
}
|
|
80
227
|
|
|
81
|
-
/** What a write is identified as on the wire: the action id every attempt of it reuses,
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
228
|
+
/** What a write is identified as on the wire: the action id every attempt of it reuses, the two
|
|
229
|
+
* per-write time values its log entry carries, the HLC and the wall-clock timestamp, and its lane.
|
|
230
|
+
* All four are minted once per WRITE, never per attempt, so a retry at the same revision appends the
|
|
231
|
+
* same entry (docs/internals.md, "A write's log append is a fixed function of the write and the
|
|
232
|
+
* revision it is requesting"). */
|
|
85
233
|
export type WriteIdentity = {
|
|
86
234
|
readonly actionId: ActionId;
|
|
235
|
+
/** The wall-clock time the write was minted at (`LogEntry.timestamp`). */
|
|
87
236
|
readonly timestamp: number;
|
|
237
|
+
/** The HLC the write was minted with (`LogEntry.hlc`), from the writer's clock
|
|
238
|
+
* (`clockOf(transactor).now()`), and carried on every pend of it (`PendRequest.hlc`).
|
|
239
|
+
* NOTE: a write refused as stale is retried over a refresh that may have observed newer HLCs,
|
|
240
|
+
* and the retry keeps the HLC it was first minted with — the identity rule above — so its entry
|
|
241
|
+
* can carry an HLC below a rival it was re-staged over. Nothing is wrong today: in the strong
|
|
242
|
+
* lane the revision is the order and the HLC is observational. The replay stage (backlog
|
|
243
|
+
* `feat-replay-mode-collections-merge-divergent-histories`) commits locally without a log-tail
|
|
244
|
+
* retry, so it never meets the case. */
|
|
245
|
+
readonly hlc: Hlc;
|
|
246
|
+
/** The write's lane (`ActionEntry.lane`, `PendRequest.lane`, `CommitRequest.lane`): the caller's
|
|
247
|
+
* classification when it gave one (`SyncOptions.lane`), else the collection's own declaration
|
|
248
|
+
* over what the write stages (`Collection.laneOf`); the queue's for a head commit
|
|
249
|
+
* (`HeadCommitOptions.lane`). */
|
|
250
|
+
readonly lane: Lane;
|
|
88
251
|
};
|
|
89
252
|
|
|
90
253
|
/** What one attempt is built from: the actions its log entry records, the transforms they produced
|
|
@@ -122,8 +285,8 @@ export type StagedAttempt = {
|
|
|
122
285
|
* — but if an attempt ever grows a per-attempt footprint beyond the log blocks, reset the abandoned
|
|
123
286
|
* tracker on the stale-failure branch instead of letting the residue ride to the end of the write.
|
|
124
287
|
*
|
|
125
|
-
* The append is a fixed function of the write and `rev`: the timestamp
|
|
126
|
-
* data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
|
|
288
|
+
* The append is a fixed function of the write and `rev`: the HLC, timestamp and lane are the identity's, and
|
|
289
|
+
* any data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
|
|
127
290
|
* `Collection.logAppendBlockIds`, which replays the ids an earlier attempt at this revision minted.
|
|
128
291
|
*
|
|
129
292
|
* The declarations are computed AFTER the append so the log tail and header transforms the attempt
|
|
@@ -150,7 +313,7 @@ export async function stageAttempt<TAction>(
|
|
|
150
313
|
throw new Error(`Log not found for collection ${collectionId}`);
|
|
151
314
|
}
|
|
152
315
|
const added = await collectionLog.addActions([...target.actions], identity.actionId, rev,
|
|
153
|
-
() => tracker.transformedBlockIds(), { timestamp: identity.timestamp });
|
|
316
|
+
() => tracker.transformedBlockIds(), { timestamp: identity.timestamp, hlc: identity.hlc, lane: identity.lane });
|
|
154
317
|
const touched = await describeTouchedBlocks(tracker, tracker.transformedBlockIds());
|
|
155
318
|
const tooLarge = oversizedBlock(touched.bytes);
|
|
156
319
|
if (tooLarge) throw tooLarge;
|
|
@@ -243,7 +406,6 @@ export type Refresh = (report: RefreshReport, lastChance: boolean, tailFloor: nu
|
|
|
243
406
|
*/
|
|
244
407
|
export class AttemptCycle {
|
|
245
408
|
private readonly maxAttempts: number;
|
|
246
|
-
private readonly maxStalledAttempts: number;
|
|
247
409
|
private readonly backoff: JitteredBackoffConfig;
|
|
248
410
|
private readonly deadlineMs: number | undefined;
|
|
249
411
|
private readonly signal: AbortSignal | undefined;
|
|
@@ -255,24 +417,8 @@ export class AttemptCycle {
|
|
|
255
417
|
// sync (which iterates many times committing progress) never trips it.
|
|
256
418
|
private consecutiveFailures = 0;
|
|
257
419
|
private lastReason: string | undefined;
|
|
258
|
-
//
|
|
259
|
-
|
|
260
|
-
// the evidence the stall check reasons from.
|
|
261
|
-
private lastStaleAt: { blockId: BlockId; rev: number } | undefined;
|
|
262
|
-
// Whether the failure most recently handled carried its OWN staleAt. A strike needs the
|
|
263
|
-
// responder to have re-confirmed the number this round, not merely an older observation
|
|
264
|
-
// left standing in `lastStaleAt`.
|
|
265
|
-
private lastFailureConfirmedStaleAt = false;
|
|
266
|
-
// Refused attempts of this write that carried their own staleAt. Never reset: a member counts
|
|
267
|
-
// its stale refusals per action for the action's whole life, and this is the writer's view of
|
|
268
|
-
// that count (see retryBackoffMs).
|
|
269
|
-
private confirmedLosses = 0;
|
|
270
|
-
// Consecutive refreshes that moved the requested revision nowhere at all while a confirmed
|
|
271
|
-
// revision stood at or above it.
|
|
272
|
-
private consecutiveStalls = 0;
|
|
273
|
-
// The revision the PREVIOUS attempt would have requested, so the stall check can tell a
|
|
274
|
-
// refresh that moved nowhere from one that is still climbing toward the confirmed number.
|
|
275
|
-
private previousRequestedRev: number | undefined;
|
|
420
|
+
// What the responders confirmed, and the two rules that read it (the stall check, the slot hold).
|
|
421
|
+
private readonly watch: RevisionStallWatch;
|
|
276
422
|
|
|
277
423
|
/** Who holds what the write has committed so far. Stays `undefined` while nothing has been
|
|
278
424
|
* committed, which is also the answer when no attempt runs at all — a write with nothing staged
|
|
@@ -289,7 +435,7 @@ export class AttemptCycle {
|
|
|
289
435
|
options?: SyncOptions,
|
|
290
436
|
) {
|
|
291
437
|
this.maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
|
|
292
|
-
this.
|
|
438
|
+
this.watch = new RevisionStallWatch(options?.maxStalledAttempts ?? DefaultMaxStalledAttempts);
|
|
293
439
|
this.backoff = {
|
|
294
440
|
baseMs: options?.baseBackoffMs ?? PendingRetryDelayMs,
|
|
295
441
|
capMs: options?.maxBackoffMs ?? DefaultMaxBackoffMs,
|
|
@@ -309,27 +455,9 @@ export class AttemptCycle {
|
|
|
309
455
|
|
|
310
456
|
/**
|
|
311
457
|
* The checks that run before an attempt is built: the abort signal, the wall-clock deadline, and
|
|
312
|
-
* the stall check. The deadline comes before the stall check
|
|
313
|
-
* outer bound, so a write that is both past it and stalled
|
|
314
|
-
*
|
|
315
|
-
* The stall check asks: can the attempt about to run possibly differ from the one that just
|
|
316
|
-
* failed? Only when BOTH of these hold is the answer provably no:
|
|
317
|
-
*
|
|
318
|
-
* - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
|
|
319
|
-
* only after reading that revision as durably held by someone else out of its own storage,
|
|
320
|
-
* revisions are one per-collection counter that every commit touches, and a confirmed
|
|
321
|
-
* revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
|
|
322
|
-
* lost before it is sent.
|
|
323
|
-
* - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
|
|
324
|
-
* held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
|
|
325
|
-
* refresh that moved forward WITHOUT clearing the confirmed number is a collection still
|
|
326
|
-
* climbing (it read a replica that lags the holder — the same partial catch-up
|
|
327
|
-
* `reportShortfall` exists to report), and its next attempt is a genuinely different request
|
|
328
|
-
* that may yet win.
|
|
329
|
-
*
|
|
330
|
-
* This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
|
|
331
|
-
* sole rule for that, untouched. It only ever stops a loop that rule had already decided to
|
|
332
|
-
* continue.
|
|
458
|
+
* the stall check ({@link RevisionStallWatch.check}). The deadline comes before the stall check
|
|
459
|
+
* deliberately: it is the documented outer bound, so a write that is both past it and stalled
|
|
460
|
+
* reports the deadline.
|
|
333
461
|
*
|
|
334
462
|
* @param requestedRev the revision the attempt would request (`Collection.getNextRev`).
|
|
335
463
|
* @param heldRev the revision the collection holds, for the stall report.
|
|
@@ -339,34 +467,21 @@ export class AttemptCycle {
|
|
|
339
467
|
throw makeAbortError(this.signal);
|
|
340
468
|
}
|
|
341
469
|
if (this.pastDeadline()) {
|
|
342
|
-
throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason ?? 'deadline exceeded', this.
|
|
470
|
+
throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason ?? 'deadline exceeded', this.watch.staleAt);
|
|
343
471
|
}
|
|
344
|
-
const
|
|
345
|
-
|
|
346
|
-
if (this.lastStaleAt === undefined) {
|
|
472
|
+
const verdict = this.watch.check(requestedRev);
|
|
473
|
+
if (verdict.kind === 'clear') {
|
|
347
474
|
return;
|
|
348
475
|
}
|
|
349
|
-
if (
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
this.
|
|
356
|
-
|
|
357
|
-
log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d',
|
|
358
|
-
this.collectionId, this.instanceTag, heldRev ?? 'none', requestedRev,
|
|
359
|
-
this.lastStaleAt.blockId, this.lastStaleAt.rev, this.consecutiveStalls, this.maxStalledAttempts);
|
|
360
|
-
}
|
|
361
|
-
// Two strikes, not one: a legitimate loser can transiently read a view that has not yet
|
|
362
|
-
// caught up with the rival's commit, which looks identical for one round.
|
|
363
|
-
if (this.consecutiveStalls >= this.maxStalledAttempts) {
|
|
364
|
-
throw new SyncRevisionStalledError(this.collectionId, this.consecutiveFailures, this.lastStaleAt,
|
|
365
|
-
requestedRev, heldRev, this.lastReason);
|
|
366
|
-
}
|
|
476
|
+
if (log.enabled) {
|
|
477
|
+
log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d',
|
|
478
|
+
this.collectionId, this.instanceTag, heldRev ?? 'none', requestedRev,
|
|
479
|
+
verdict.staleAt.blockId, verdict.staleAt.rev, verdict.strike, verdict.of);
|
|
480
|
+
}
|
|
481
|
+
if (verdict.kind === 'stalled') {
|
|
482
|
+
throw new SyncRevisionStalledError(this.collectionId, this.consecutiveFailures, verdict.staleAt,
|
|
483
|
+
requestedRev, heldRev, this.lastReason);
|
|
367
484
|
}
|
|
368
|
-
// Else: the responder told us nothing new this round. No strike, and no reset either — the
|
|
369
|
-
// budget stays bounded by maxAttempts.
|
|
370
485
|
}
|
|
371
486
|
|
|
372
487
|
/**
|
|
@@ -409,14 +524,9 @@ export class AttemptCycle {
|
|
|
409
524
|
}
|
|
410
525
|
this.consecutiveFailures++;
|
|
411
526
|
this.lastReason = refusal.reason ?? this.lastReason;
|
|
412
|
-
|
|
413
|
-
// responder reporting a LOWER number understates the binding constraint. Same rule the
|
|
414
|
-
// producers and the transactor's aggregation already use.
|
|
415
|
-
this.lastStaleAt = highestStaleAt([this.lastStaleAt, refusal.staleAt]);
|
|
416
|
-
this.lastFailureConfirmedStaleAt = refusal.staleAt !== undefined;
|
|
417
|
-
if (this.lastFailureConfirmedStaleAt) this.confirmedLosses++;
|
|
527
|
+
this.watch.noteRefusal(refusal.staleAt);
|
|
418
528
|
if (this.consecutiveFailures >= this.maxAttempts) {
|
|
419
|
-
throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason, this.
|
|
529
|
+
throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason, this.watch.staleAt);
|
|
420
530
|
}
|
|
421
531
|
}
|
|
422
532
|
|
|
@@ -427,9 +537,7 @@ export class AttemptCycle {
|
|
|
427
537
|
this.durability = this.durability === undefined ? durability : mergeDurability([this.durability, durability]);
|
|
428
538
|
this.consecutiveFailures = 0;
|
|
429
539
|
this.lastReason = undefined;
|
|
430
|
-
this.
|
|
431
|
-
this.lastFailureConfirmedStaleAt = false;
|
|
432
|
-
this.consecutiveStalls = 0;
|
|
540
|
+
this.watch.noteCommit();
|
|
433
541
|
}
|
|
434
542
|
|
|
435
543
|
/**
|
|
@@ -466,18 +574,14 @@ export class AttemptCycle {
|
|
|
466
574
|
* @throws TornActionError with any reason but an unsettled `completion-refused`, and that one too
|
|
467
575
|
* once the budget or the deadline is spent; whatever else the refresh throws. */
|
|
468
576
|
async refreshUntilSettled(refresh: Refresh): Promise<boolean> {
|
|
469
|
-
// Whether the failure just handled was a loss to a rival already durably committed. Only the
|
|
470
|
-
// attempt's own refusal can say so; a refresh refused for a cause that can clear (below) is a
|
|
471
|
-
// race against something still in flight, and clears it.
|
|
472
|
-
let latestConfirmed = this.lastFailureConfirmedStaleAt;
|
|
473
577
|
for (;;) {
|
|
474
578
|
const delay = this.consecutiveFailures === 0
|
|
475
579
|
? 0
|
|
476
|
-
: retryBackoffMs(this.consecutiveFailures, this.
|
|
580
|
+
: retryBackoffMs(this.consecutiveFailures, this.watch, this.backoff, this.rand);
|
|
477
581
|
await abortableDelay(delay, this.signal);
|
|
478
582
|
try {
|
|
479
583
|
const report: RefreshReport = {};
|
|
480
|
-
await refresh(report, this.consecutiveFailures + 1 >= this.maxAttempts, this.
|
|
584
|
+
await refresh(report, this.consecutiveFailures + 1 >= this.maxAttempts, this.watch.tailFloor());
|
|
481
585
|
const completed = report.ownEntryFinished?.durability;
|
|
482
586
|
if (completed !== undefined) {
|
|
483
587
|
this.recordCommit(completed);
|
|
@@ -492,7 +596,7 @@ export class AttemptCycle {
|
|
|
492
596
|
// the log holds an entry for a write that was not saved.
|
|
493
597
|
this.consecutiveFailures++;
|
|
494
598
|
this.lastReason = err.detail;
|
|
495
|
-
|
|
599
|
+
this.watch.noteRefusedRound();
|
|
496
600
|
if (this.consecutiveFailures >= this.maxAttempts || this.pastDeadline()) {
|
|
497
601
|
throw err;
|
|
498
602
|
}
|
|
@@ -15,7 +15,11 @@ function diaryInit<TEntry>(): CollectionInitOptions<TEntry> {
|
|
|
15
15
|
},
|
|
16
16
|
createHeaderBlock: (id: BlockId, store: BlockStore<IBlock>) => ({
|
|
17
17
|
header: store.createBlockHeader(DiaryHeaderBlockType, id)
|
|
18
|
-
})
|
|
18
|
+
}),
|
|
19
|
+
// Append-only is the first natural convergent case: two machines' concurrent appends merge as
|
|
20
|
+
// their union (docs/partition-healing.md §I-Confluence Classification). The handler touches no
|
|
21
|
+
// block, so there is nothing to rebase and no `filterConflict`.
|
|
22
|
+
merge: { laneOf: () => 'convergent' },
|
|
19
23
|
};
|
|
20
24
|
}
|
|
21
25
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Collection, type CollectionInitOptions, type CollectionId, type CollectionSnapshot, type ReadViewOptions } from "../../collection/index.js";
|
|
1
|
+
import { Collection, type CollectionInitOptions, type CollectionId, type CollectionSnapshot, type ReadViewOptions, type SyncOptions } from "../../collection/index.js";
|
|
2
2
|
import type { ITransactor, BlockId, BlockStore, IBlock, ActionId, WriteDurability } from "../../index.js";
|
|
3
3
|
import { BTree, type Path, type KeyRange } from "../../btree/index.js";
|
|
4
4
|
import { CollectionTrunk } from "./collection-trunk.js";
|
|
@@ -173,21 +173,27 @@ export interface TreeReadView<TKey, TEntry> {
|
|
|
173
173
|
get(key: TKey): Promise<TEntry | undefined>;
|
|
174
174
|
at(path: Path<TKey, TEntry>): TEntry | undefined;
|
|
175
175
|
range(range: KeyRange<TKey>): AsyncIterableIterator<Path<TKey, TEntry>>;
|
|
176
|
+
entriesIn(range: KeyRange<TKey>): Promise<TEntry[]>;
|
|
176
177
|
ascending(path: Path<TKey, TEntry>): AsyncIterableIterator<Path<TKey, TEntry>>;
|
|
177
178
|
isValid(path: Path<TKey, TEntry>): boolean;
|
|
178
179
|
}
|
|
179
180
|
|
|
180
181
|
/** Carries the read {@link BTree} from wherever it gets built (the `createHeaderBlock`
|
|
181
|
-
* callback on the create path, {@link Tree.attach} on the open path) to the
|
|
182
|
-
*
|
|
182
|
+
* callback on the create path, {@link Tree.attach} on the open path) to the collection's
|
|
183
|
+
* `onViewChanged` callback, which needs the live instance to invalidate outstanding paths. */
|
|
183
184
|
interface BTreeHolder<TKey, TEntry> {
|
|
184
185
|
btree?: BTree<TKey, TEntry>;
|
|
185
186
|
}
|
|
186
187
|
|
|
187
188
|
/** The collection options a tree's opener may set — the ones that are not the tree's own to
|
|
188
|
-
* decide
|
|
189
|
-
*
|
|
190
|
-
|
|
189
|
+
* decide: the sealed-segment refusal callback the offline-sync queue installs (see
|
|
190
|
+
* {@link CollectionInitOptions.onSealedRefused}), and the convergent-lane merge declaration
|
|
191
|
+
* ({@link CollectionInitOptions.merge}), which a raw tree user may make. `Tree` itself declares none:
|
|
192
|
+
* a tree is strong unless its opener says otherwise, and the SQL table over a tree never declares
|
|
193
|
+
* one here, because it classifies per statement and passes the lane on each write
|
|
194
|
+
* (`SyncOptions.lane`). */
|
|
195
|
+
export type TreeCollectionOptions<TKey = unknown, TEntry = unknown> =
|
|
196
|
+
Pick<CollectionInitOptions<TreeReplaceAction<TKey, TEntry>>, 'onSealedRefused' | 'merge'>;
|
|
191
197
|
|
|
192
198
|
export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
|
|
193
199
|
|
|
@@ -212,7 +218,7 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
|
|
|
212
218
|
/** See {@link Tree.createOrOpen}'s `nodeCapacity`. */
|
|
213
219
|
nodeCapacity?: number,
|
|
214
220
|
/** See {@link Tree.createOrOpen}'s `options`. */
|
|
215
|
-
options?: TreeCollectionOptions,
|
|
221
|
+
options?: TreeCollectionOptions<TKey, TEntry>,
|
|
216
222
|
): Promise<Tree<TKey, TEntry> | undefined> {
|
|
217
223
|
const held: BTreeHolder<TKey, TEntry> = {};
|
|
218
224
|
const init = Tree.buildInit(id, keyFromEntry, compare, nodeCapacity, held, options);
|
|
@@ -238,8 +244,9 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
|
|
|
238
244
|
* the header and read it back on reopen rather than trusting the caller to re-supply it. */
|
|
239
245
|
nodeCapacity?: number,
|
|
240
246
|
/** Collection options the opener sets rather than the tree: the sealed-segment refusal
|
|
241
|
-
* callback (see {@link TreeCollectionOptions}). Omit for a
|
|
242
|
-
|
|
247
|
+
* callback and the merge declaration (see {@link TreeCollectionOptions}). Omit for a strong
|
|
248
|
+
* tree nothing seals. */
|
|
249
|
+
options?: TreeCollectionOptions<TKey, TEntry>,
|
|
243
250
|
): Promise<Tree<TKey, TEntry>> {
|
|
244
251
|
const held: BTreeHolder<TKey, TEntry> = {};
|
|
245
252
|
const init = Tree.buildInit(id, keyFromEntry, compare, nodeCapacity, held, options);
|
|
@@ -255,7 +262,7 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
|
|
|
255
262
|
compare: (a: TKey, b: TKey) => number,
|
|
256
263
|
nodeCapacity: number | undefined,
|
|
257
264
|
held: BTreeHolder<TKey, TEntry>,
|
|
258
|
-
options: TreeCollectionOptions | undefined,
|
|
265
|
+
options: TreeCollectionOptions<TKey, TEntry> | undefined,
|
|
259
266
|
): CollectionInitOptions<TreeReplaceAction<TKey, TEntry>> {
|
|
260
267
|
return {
|
|
261
268
|
...options,
|
|
@@ -318,13 +325,16 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
|
|
|
318
325
|
await actionTree.deleteAt(path);
|
|
319
326
|
}
|
|
320
327
|
}
|
|
321
|
-
// Mutations landed in `trx`, not the read btree, so its version counter never moved.
|
|
322
|
-
// Bump it to invalidate any Path a caller still holds — preserving the path-invalidation
|
|
323
|
-
// the previous in-place handler gave for free.
|
|
324
|
-
held.btree?.invalidatePaths();
|
|
325
328
|
return tally.result();
|
|
326
329
|
}
|
|
327
330
|
},
|
|
331
|
+
// Mutations land in `trx` and in the collection's cache, never through the read btree, so
|
|
332
|
+
// its version counter would never move on its own. The collection says when what its reads
|
|
333
|
+
// see changes — a staged action landing, a replay, a restore, a refresh, a commit folded in
|
|
334
|
+
// — and the bump invalidates every Path handed out before, and tells a point read or a
|
|
335
|
+
// range scan's setup part-way through a descent to read again (docs/btree.md §Reads under
|
|
336
|
+
// a concurrent change).
|
|
337
|
+
onViewChanged: () => held.btree?.invalidatePaths(),
|
|
328
338
|
createHeaderBlock: (hid: BlockId, store: BlockStore<IBlock>) => { // Only called if the collection does not exist
|
|
329
339
|
// Tricky bootstrapping here:
|
|
330
340
|
// We need the root id to initialize the collection header, so we create the btree here.
|
|
@@ -344,7 +354,7 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
|
|
|
344
354
|
/** Bind an opened collection to its read btree. On the create path `createHeaderBlock` already
|
|
345
355
|
* built one (it needed the root id for the header); on the open path it never ran, so build it
|
|
346
356
|
* over the collection's existing tracker. Either way the result is written back into `held` so
|
|
347
|
-
* the
|
|
357
|
+
* the collection's path-invalidation (`onViewChanged`) targets the very btree reads go through. */
|
|
348
358
|
private static attach<TKey, TEntry>(
|
|
349
359
|
collection: Collection<TreeReplaceAction<TKey, TEntry>>,
|
|
350
360
|
held: BTreeHolder<TKey, TEntry>,
|
|
@@ -382,9 +392,10 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
|
|
|
382
392
|
}
|
|
383
393
|
|
|
384
394
|
/** Flush all staged (and any other pending) changes to the transactor.
|
|
385
|
-
* Equivalent to the flush half of {@link replace}.
|
|
386
|
-
|
|
387
|
-
|
|
395
|
+
* Equivalent to the flush half of {@link replace}. `options` is the write's, {@link SyncOptions.lane}
|
|
396
|
+
* included: the SQL bridge classifies per statement and passes the fold here. */
|
|
397
|
+
async sync(options?: SyncOptions): Promise<WriteDurability | undefined> {
|
|
398
|
+
return await this.collection.updateAndSync(options);
|
|
388
399
|
}
|
|
389
400
|
|
|
390
401
|
/** Whether {@link sync} has anything to push — see
|
|
@@ -526,6 +537,10 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
|
|
|
526
537
|
return this.btree.range(range);
|
|
527
538
|
}
|
|
528
539
|
|
|
540
|
+
async entriesIn(range: KeyRange<TKey>): Promise<TEntry[]> {
|
|
541
|
+
return await this.btree.entriesIn(range);
|
|
542
|
+
}
|
|
543
|
+
|
|
529
544
|
ascending(path: Path<TKey, TEntry>): AsyncIterableIterator<Path<TKey, TEntry>> {
|
|
530
545
|
return this.btree.ascending(path);
|
|
531
546
|
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { Hlc } from './hlc.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Where an `HlcClock` keeps its high-water mark across restarts, so a node whose wall clock is
|
|
5
|
+
* behind at restart does not issue an HLC below ones it issued before.
|
|
6
|
+
*
|
|
7
|
+
* A binding only stores and returns the value. When to save, and what to make of the loaded value,
|
|
8
|
+
* is the clock's policy, so every binding behaves the same way: the clock saves on a jump and at
|
|
9
|
+
* most once per {@link DEFAULT_HLC_PERSIST_INTERVAL_MS} otherwise, never awaits a save on its write
|
|
10
|
+
* path, and resumes from {@link restartFloor}.
|
|
11
|
+
*/
|
|
12
|
+
export interface HlcClockStore {
|
|
13
|
+
/** The mark last saved, or `undefined` when none was ever saved. */
|
|
14
|
+
load(): Promise<Hlc | undefined>;
|
|
15
|
+
/** Replace the mark. The clock has at most one call in flight; a rejection is logged by the clock and retried on its next qualifying advance. */
|
|
16
|
+
save(hlc: Hlc): Promise<void>;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* How far `wallMs` must have moved past the last saved mark before the clock saves again: a node
|
|
21
|
+
* issuing on its own wall clock writes at most once a second under load and nothing while idle,
|
|
22
|
+
* while an observed HLC far ahead of local time moves it past this at once and is saved at once.
|
|
23
|
+
* It is also the larger part of what {@link DEFAULT_HLC_RESTART_SLACK_MS} has to cover.
|
|
24
|
+
*/
|
|
25
|
+
export const DEFAULT_HLC_PERSIST_INTERVAL_MS = 1000;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* How far above the stored mark a restarted clock resumes when its local clock has not already
|
|
29
|
+
* passed that point. It covers the HLCs issued after the stored mark: at most
|
|
30
|
+
* {@link DEFAULT_HLC_PERSIST_INTERVAL_MS} of wall time before the next save, plus whatever the
|
|
31
|
+
* binding's store had not yet made durable when the process stopped.
|
|
32
|
+
*
|
|
33
|
+
* The second part, for a db-p2p node: the clock never flushes, so its mark is durable once some
|
|
34
|
+
* flush of the store covers it. The longest any of a node's stores may defer that is the relaxed
|
|
35
|
+
* flush window's cap, `maxWindowMs`, 5000 ms by default (`resolveDurabilityWindowPolicy` in
|
|
36
|
+
* `packages/db-p2p/src/storage/durability-window.ts`, packages/db-p2p/docs/storage.md §6); a write
|
|
37
|
+
* no flush covers is synced by the write group's idle timer, one second after it. So at the defaults
|
|
38
|
+
* the durable mark trails what the clock issued by at most 1000 + 5000 = 6000 ms, and 10 s leaves
|
|
39
|
+
* 4 s of margin. A deployment that raises `maxWindowMs` past 9 s should raise this with it
|
|
40
|
+
* (`HlcClockStartOptions.restartSlackMs`).
|
|
41
|
+
*
|
|
42
|
+
* In the ordinary case a real restart takes longer than this and the local clock wins, so the floor
|
|
43
|
+
* costs nothing; it matters for a fast restart or a clock that went backwards, where the node's HLCs
|
|
44
|
+
* then sit up to this far ahead of its wall clock until wall time catches up.
|
|
45
|
+
*
|
|
46
|
+
* NOTE: the slack covers an unsaved stretch only while the mark tracked wall time. A mark that
|
|
47
|
+
* jumped — to an observed HLC far ahead, or to the restart floor itself — is saved at once but not
|
|
48
|
+
* flushed, so a crash inside the store's durability window loses the jump, and the next start can
|
|
49
|
+
* issue HLCs at or below ones the lost run issued (a crash loop faster than that window repeats one
|
|
50
|
+
* floor). Harmless while HLCs are observational (Stage 1 of docs/crdt-sync.md); once anything orders
|
|
51
|
+
* by them, the save after a jump must be durable before the clock issues past it.
|
|
52
|
+
*
|
|
53
|
+
* NOTE: the floor is saved, so each restart that comes faster than the slack resumes a further
|
|
54
|
+
* slack above the last one: a crash loop restarting every few seconds moves the node's HLCs ahead
|
|
55
|
+
* of its wall clock by about the slack less the restart interval, per restart, without bound. Fine
|
|
56
|
+
* while HLCs are observational; if HLCs start deciding order (Stage 2), a remedy is to wait out the
|
|
57
|
+
* floor at start rather than resume above it.
|
|
58
|
+
*/
|
|
59
|
+
export const DEFAULT_HLC_RESTART_SLACK_MS = 10_000;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* The HLC a clock restarted over `stored` treats as already observed: `slackMs` above the stored
|
|
63
|
+
* `wallMs`, logical 0, in this node's name. The clock only ever raises its mark, so a local clock
|
|
64
|
+
* already past this point simply wins.
|
|
65
|
+
*/
|
|
66
|
+
export function restartFloor(stored: Hlc, slackMs: number, nodeId: string): Hlc {
|
|
67
|
+
return { wallMs: stored.wallMs + slackMs, logical: 0, nodeId };
|
|
68
|
+
}
|