@optimystic/db-core 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,2061 +1,2118 @@
1
- import type { IBlock, Action, ActionType, ActionHandler, BlockId, ITransactor, BlockStore, Transforms, ActionId, ActionLineage, BlockContentDigests, GetBlockResult } from "../index.js";
2
- import { Log } from "../log/log.js";
3
- import type { LogBlock } from "../log/log.js";
4
- import type { ActionEntry, GetFromResult, LogEntry } from "../log/struct.js";
5
- import { Atomic } from "../transform/atomic.js";
6
- import { Tracker } from "../transform/tracker.js";
7
- import type { BasePins } from "../transform/base-pins.js";
8
- import { CacheSource } from "../transform/cache-source.js";
9
- import { computeBlockContentDigests, baseRevsField } from "../transform/digest.js";
10
- import type { BlockBaseRevs } from "../network/struct.js";
11
- import { copyTransforms, isTransformsEmpty } from "../transform/helpers.js";
12
- import { TransactorSource, answeredBlock, servedRevision } from "../transactor/transactor-source.js";
13
- import { BlockFloors } from "../transactor/block-floors.js";
14
- import type { WriteDurability } from "../network/struct.js";
15
- import { mergeDurability } from "../network/durability.js";
16
- import { highestStaleAt } from "../network/stale-failure.js";
17
- import type { CollectionHeaderBlock, CollectionId, ICollection, SyncOptions, TornActionReason } from "./index.js";
18
- import { CollectionHeaderVanishedError, SyncRetryExhaustedError, SyncRevisionStalledError, TornActionError } from "./struct.js";
19
- import type { ActionContext } from "./action.js";
20
- import { actionIdAt } from "./action.js";
21
- import type { ReadDependency } from "../transaction/transaction.js";
22
- import { clampPriority } from "../transaction/transaction.js";
23
- import { ReadDependencyCollector } from "../transaction/read-dependency-collector.js";
24
- import { randomBytes } from '@noble/hashes/utils.js';
25
- import { toString as uint8ArrayToString } from 'uint8arrays/to-string';
26
- import { Latches } from "../utility/latches.js";
27
- import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
28
- import { createLogger } from "../logger.js";
29
-
30
- const log = createLogger('collection');
31
-
32
- /** Which of {@link Collection.advanceContext}'s two callers is reporting — printed as `site=` on
33
- * every line it emits, because the two compare DIFFERENT pairs of things and a divergence means
34
- * something different in each:
35
- *
36
- * - `refresh` ({@link Collection.updateInternal}) — this instance's own copy against the stored
37
- * log. A divergence here indicts a forked REPLICA: two copies of one collection id built
38
- * separately, each internally consistent.
39
- * - `attach` ({@link Collection.attachToLog}, during open) — the log tail block's claim about
40
- * which action produced the latest revision, against a walk of that same tail's own chain. Both
41
- * sides come from storage, so a divergence here indicts STORAGE being self-inconsistent about
42
- * one revision, not a replica.
43
- *
44
- * Without this field the two are indistinguishable in a log, and they lead an operator to
45
- * completely different places. */
46
- type DivergenceSite = 'refresh' | 'attach';
47
-
48
- /** The lowest revision two {@link ActionContext}s provably disagree at, and the action each names
49
- * there — what {@link Collection.earliestFork} reports and `collection:lineage-divergence` prints
50
- * as `forkRev=` / `heldAction=` / `readAction=`. */
51
- type LineageFork = { rev: number, heldAction: ActionId, readAction: ActionId };
52
-
53
- /** Exactly what one failed write attempt sent to the transactor — enough to send the same thing
54
- * again. Retained by {@link Collection.retainInFlightAttempt} so that a refresh which then finds
55
- * this attempt's own log entry can land the blocks the attempt left behind, at the same action id
56
- * and revision (see {@link Collection.completeOwnEntry}). */
57
- export type InFlightAttempt = {
58
- /** The revision the attempt's log entry was stamped with, and pended and committed at. */
59
- rev: number;
60
- /** Every transform the attempt pended — the log blocks included. The caller hands over a copy
61
- * it will not mutate; the collection keeps it as given. */
62
- transforms: Transforms;
63
- /** The attempt's log tail block, committed first. */
64
- tailId: BlockId;
65
- /** The per-block content declarations the attempt's commit carried, if any. */
66
- blockDigests?: BlockContentDigests;
67
- /** The per-block bases the attempt's pend carried, if any (see `PendRequest.baseRevs`). Kept
68
- * with the transforms because they describe them: a re-send of these operations must name the
69
- * same bases they were computed against. */
70
- baseRevs?: BlockBaseRevs;
71
- };
72
-
73
- /** What one refresh ({@link Collection.refreshInFlight}, and the refresh inside a sync) found out
74
- * about the write in flight on the instance's behalf. The caller hands in an empty report and the
75
- * refresh fills it in AS IT GOES, rather than returning it at the end, so a refresh that throws
76
- * after it saved the write still says so: the write is saved in storage whatever happens to this
77
- * instance's local bookkeeping afterwards.
78
- *
79
- * NOTE: filled in, not returned, because a return value is lost on a throw — and a refresh CAN
80
- * throw after finishing its own entry (the invalidation read and the replay both run later). */
81
- export type RefreshReport = {
82
- /** Set once the refresh found the in-flight write's own log entry and FINISHED it
83
- * ({@link Collection.completeOwnEntry}): every block the entry names holds the write, so it is
84
- * saved. Left unset on every other refresh, including every reader's.
85
- *
86
- * `durability` is who holds the finished write, when the refresh learned it. It is absent when
87
- * finishing found every block already holding the write without re-sending anything (the
88
- * status-read fallback), which is still a saved write — so test the field, never `durability`,
89
- * for "saved". */
90
- ownEntryFinished?: { durability?: WriteDurability };
91
- };
92
-
93
- /** The two blocks every refresh starts from, as {@link Collection.readLogEnds} read them. */
94
- type LogEnds = {
95
- header: CollectionHeaderBlock;
96
- /** The repo's answer for the log tail block the header names; absent when the header names no
97
- * tail or the repo returned no entry for it. */
98
- tail?: GetBlockResult;
99
- /** Every block read, with the revision it was served at — the seed for the refresh's block cache. */
100
- served: Array<[BlockId, IBlock, number]>;
101
- };
102
-
103
- /** Default base backoff (and historical fixed delay) between sync retries, in ms. */
104
- const PendingRetryDelayMs = 100;
105
- /** Default max consecutive no-progress stale-failure retries before {@link Collection.sync} gives up. */
106
- const DefaultMaxAttempts = 10;
107
- /** Default ceiling on a single exponential-backoff sleep, in ms. */
108
- const DefaultMaxBackoffMs = 5000;
109
- /** Default consecutive stalled refreshes — ones that moved this collection's revision NOWHERE
110
- * while a responder CONFIRMED a revision at or above the one being requested — before
111
- * {@link Collection.sync} gives up. Two, not one, so a single transiently-lagging read is absorbed.
112
- *
113
- * NOTE: two is a judgement call, not a measurement — nothing here counts how often a legitimate
114
- * loser reads a view that moves nowhere for two consecutive rounds. The check only strikes when the
115
- * refresh made NO progress at all, so a client merely catching up is already excluded; if a
116
- * spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
117
- const DefaultMaxStalledAttempts = 2;
118
-
119
- export type CollectionInitOptions<TAction> = {
120
- modules: Record<ActionType, ActionHandler<TAction>>;
121
- createHeaderBlock: (id: BlockId, store: BlockStore<IBlock>) => IBlock;
122
- /** Called for each local action that is potentially in conflict with a remote action.
123
- * @param action - The local action to check
124
- * @param potential - The remote actions that are potentially in conflict
125
- * @returns The original action (return the same instance to keep it as-is), a replacement
126
- * action (return a new instance to apply instead of the original — it is re-staged via
127
- * the conflict replay), or undefined to discard this action
128
- */
129
- filterConflict?: (action: Action<TAction>, potential: Action<TAction>[]) => Action<TAction> | undefined
130
- }
131
-
132
- /** Options for building a committed read view (see {@link Collection.createReadTracker}
133
- * and `Tree.readView`). */
134
- export interface ReadViewOptions {
135
- /** Record read dependencies into the collection's shared collector.
136
- * Default false — a pinned committed view is not part of any transaction's
137
- * conflict set, so its reads must not be able to fail the writer's commit
138
- * validation. Deferred-constraint safety does not depend on these reads:
139
- * validator peers re-execute the transaction's recorded statements against
140
- * their own committed state, so a constraint that no longer holds is caught
141
- * at validation regardless. */
142
- recordReads?: boolean;
143
- /** Pin the view to this committed boundary instead of the collection's CURRENT
144
- * action context. Pass a {@link CollectionSnapshot.context} so the view describes
145
- * the same committed moment the snapshot's transforms sat on — even when the
146
- * collection has committed further since (e.g. a multi-tree commit sweep that has
147
- * already flushed THIS tree but not its siblings). Blocks cached at revisions
148
- * newer than the pin are excluded from the view's warm seed and refetched from
149
- * the transactor at the pinned revision. Default: the current context. */
150
- pinContext?: ActionContext;
151
- }
152
-
153
- /** A point-in-time copy of a collection's staged (un-synced) state, produced by
154
- * {@link Collection.snapshotPending} and consumed by {@link Collection.restorePending}. */
155
- export interface CollectionSnapshot<TAction> {
156
- /** Deep-cloned tracker transforms at snapshot time. */
157
- transforms: Transforms;
158
- /** The bases the transforms' update operations were computed against (see {@link BasePins}),
159
- * copied at snapshot time. Restored with the transforms, so a restore never pairs operations
160
- * with the bases of some later re-stage: the pend would then declare a base the operations
161
- * were not built on. Absent on a snapshot built by hand; such a restore keeps whatever pins
162
- * the tracker holds for the restored ids. */
163
- pins?: BasePins;
164
- /** Pending actions queued at snapshot time. */
165
- pending: Action<TAction>[];
166
- /** The committed boundary (action context) the staged state sat on when captured.
167
- * `undefined` for a collection with no committed revision yet (an invented
168
- * collection whose header/root still live in the tracker). A read view built
169
- * from this snapshot pins to this boundary (see {@link ReadViewOptions.pinContext}),
170
- * so the view stays coherent with the snapshot's transforms even if the
171
- * collection commits further before the view is built. */
172
- context?: ActionContext;
173
- }
174
-
175
- export class Collection<TAction> implements ICollection<TAction> {
176
- private pending: Action<TAction>[] = [];
177
- private readonly latchId: string;
178
-
179
- /** The action id of a write currently in flight ON THIS INSTANCE'S BEHALF, or `undefined`
180
- * outside a write. Read by {@link updateInternal}: if the committed log now carries an entry
181
- * under this id, that action's LOG TAIL landed despite the failure answer that sent us back here
182
- * — every member applies the log tail before the action's other blocks, and
183
- * `NetworkTransactor.commit` reports failure when the commit carrying the tail was refused after
184
- * landing on a minority (`commit-not-durable`; in one round the other blocks may have landed on
185
- * that minority too, in two the sweep never ran) and when a later sweep block confirmed a
186
- * conflict. Such an entry is never REPLAYED, because replaying re-appends content
187
- * the committed tail already carries, producing a duplicate entry under one action id at two
188
- * revisions. But it is not proof the write is saved either: the entry proves only that the tail
189
- * landed, and the writer's own cancel has since dropped the pending records of every block that
190
- * did not. So the refresh first FINISHES the action ({@link completeOwnEntry}) — landing the
191
- * remaining blocks at the same action id and revision, from {@link inFlightAttempt} — and only
192
- * then consumes the entry ({@link consumeOwnEntry}).
193
- *
194
- * The collection owns this fact rather than taking it as a `updateInternal` argument so that no
195
- * refresh path can forget to supply it — {@link update} and {@link updateAndSync} are refreshes
196
- * on behalf of a READER, the field is unset for them, and the consume branch cannot fire. Before
197
- * this was a field, `TransactionCoordinator.commit`'s inter-attempt refresh went through
198
- * `update()` and was therefore indistinguishable from a reader refresh even though the
199
- * coordinator held the very id it was retrying. (It now goes through {@link refreshInFlight},
200
- * which differs from `update()` only in reporting what the refresh saved — the field, not the
201
- * method, is still what makes the refresh recognise the entry.)
202
- *
203
- * LIFETIME is the whole attempt CYCLE, not the latched span: it must survive the refresh
204
- * BETWEEN a failed attempt and its retry, which is the only moment it is ever read. In
205
- * {@link syncInternal} that cycle is contained inside the collection latch `sync()` holds; in
206
- * `TransactionCoordinator.commit` the inter-attempt refresh runs OUTSIDE the commit latch
207
- * span by design (`Latches` is non-reentrant), so the coordinator's clear necessarily runs
208
- * latch-free. That is safe: this is a single field write, {@link beginInFlightAction}'s
209
- * disposer only clears an id it still owns, and the only reader runs under the latch — so the
210
- * worst a foreign concurrent refresh can see is a cleared field (it stops consuming), never a
211
- * field it should not have consumed. */
212
- private inFlightActionId?: ActionId;
213
-
214
- /** The most recent FAILED attempt made under {@link inFlightActionId} — exactly what it sent,
215
- * kept so a refresh that finds that attempt's own log entry can finish the action instead of
216
- * assuming it is finished (see {@link completeOwnEntry}). Set by {@link retainInFlightAttempt},
217
- * cleared with the mark by {@link beginInFlightAction}'s disposer, and meaningless without it.
218
- *
219
- * The transforms are retained VERBATIM, not rebuilt: a rebuilt attempt re-appends the log entry
220
- * with a fresh timestamp, so its log tail would differ byte-for-byte from the one already
221
- * stored under the same action id and revision — and any replica that had not yet stored the
222
- * tail would then store the second version, leaving two contents under one `(action, revision)`.
223
- *
224
- * Only the latest attempt is kept. An own entry can only be visible at the revision of an
225
- * attempt whose tail landed, and a later attempt at a DIFFERENT revision is only made after a
226
- * refresh adopted somebody else's entry at the earlier one — which is proof the earlier tail did
227
- * not land. {@link completeOwnEntry} still checks the revision and refuses on a mismatch. */
228
- private inFlightAttempt?: InFlightAttempt;
229
-
230
- /** The log tail block id the most recent header read named. A refresh asks for this block in the
231
- * same request as the header ({@link readLogEnds}): the tail id only changes when the tail block
232
- * fills, so an idle refresh is one request rather than two. A stale value costs one extra request
233
- * and nothing else. */
234
- private logTailId?: BlockId;
235
-
236
- protected constructor(
237
- public readonly id: CollectionId,
238
- public readonly transactor: ITransactor,
239
- private readonly handlers: Record<ActionType, ActionHandler<TAction>>,
240
- private readonly source: TransactorSource<IBlock>,
241
- /** Cache of unmodified blocks from the source */
242
- private readonly sourceCache: CacheSource<IBlock>,
243
- /** Tracked Changes */
244
- public readonly tracker: Tracker<IBlock>,
245
- /** What each block named by a walked log entry must be at least as new as — raised by
246
- * {@link updateInternal}, and shared with every read source this handle builds (see
247
- * {@link BlockFloors}). */
248
- private readonly floors: BlockFloors,
249
- private readonly filterConflict?: (action: Action<TAction>, potential: Action<TAction>[]) => Action<TAction> | undefined,
250
- /** Short random tag naming THIS instance (see {@link newInstanceTag}). Open paths generate
251
- * it BEFORE construction (so pre-construction diagnostics such as attachToLog can carry it);
252
- * the default covers direct construction in tests. */
253
- public readonly instanceTag: string = Collection.newInstanceTag(),
254
- ) {
255
- // Instance-scoped, deliberately NOT shared across instances of one collection id. The
256
- // latch protects per-instance state only — the tracker, the pending queue, and
257
- // source.actionContext, none of which two instances over the same id share — while
258
- // cross-instance races are resolved by the transactor's optimistic concurrency (that is
259
- // the design; the old process-global `Collection:${id}` key serialized instances by
260
- // accident). Instance scope is also what lets TransactionCoordinator hold this latch
261
- // across its whole commit span: `Latches` is a non-reentrant FIFO mutex, and a rival
262
- // writer driving a SECOND instance of the same id from inside transactor.pend (see
263
- // CompetingWriterTransactor) would otherwise wait on the very latch the parked commit
264
- // holds — a deadlock, not contention.
265
- this.latchId = `Collection:${this.id}#${this.instanceTag}`;
266
- }
267
-
268
- /** A fresh instance tag: four random bytes rendered base64url — six characters, enough that
269
- * two instances over one collection id do not collide by accident, short enough to ride on
270
- * every trace line (same shape as the node tag in quereus-plugin-optimystic's
271
- * collection-factory). Scopes {@link latchId} per instance and labels diagnostics. */
272
- private static newInstanceTag(): string {
273
- return uint8ArrayToString(randomBytes(4), 'base64url');
274
- }
275
-
276
- /** Open an EXISTING collection.
277
- *
278
- * Resolves to `undefined` when the header block probe comes back empty — an
279
- * authoritatively absent header, meaning nothing has ever been committed under this id.
280
- * A header the storage layer could not RETRIEVE (a revision this node cannot
281
- * reconstruct, an unreachable cohort) is not absent: the probe throws
282
- * {@link BlockUnavailableError} instead of resolving `undefined`, so an unreachable
283
- * collection can never be mistaken for a nonexistent one.
284
- *
285
- * Use this wherever reading — not creating — is what was meant. {@link createOrOpen}
286
- * would instead stage a fresh empty collection, and reads through it would report an
287
- * absent dataset as a legitimately empty one. */
288
- static async open<TAction>(transactor: ITransactor, id: CollectionId, init: CollectionInitOptions<TAction>): Promise<Collection<TAction> | undefined> {
289
- // Generated BEFORE anything reads, so every diagnostic of this handle — the floors the probe
290
- // wires up and the log-attach-time lines included — names the instance the same way
291
- // post-construction ones do.
292
- const instanceTag = Collection.newInstanceTag();
293
- const { source, sourceCache, tracker, floors, header } = await Collection.probeHeader(transactor, id, instanceTag);
294
- if (!header) {
295
- // Return before anything is staged: the tracker's transforms stay empty, so a caller
296
- // that ignores the undefined cannot later sync a phantom collection into existence.
297
- return undefined;
298
- }
299
- await Collection.attachToLog<TAction>(source, transactor, tracker, id, instanceTag, header);
300
- const collection = new Collection(id, transactor, init.modules, source, sourceCache, tracker, floors, init.filterConflict, instanceTag);
301
- collection.logTailId = header.tailId;
302
- return collection;
303
- }
304
-
305
- /** Open an existing collection, or stage a fresh empty one in the local tracker when the
306
- * header is authoritatively absent. Nothing is written to storage until {@link sync}.
307
- *
308
- * Correct only where inventing a collection is genuinely intended — a first write, a
309
- * bootstrap path. The create branch logs `collection:invented`; prefer {@link open} on
310
- * any pure read path. */
311
- static async createOrOpen<TAction>(transactor: ITransactor, id: CollectionId, init: CollectionInitOptions<TAction>): Promise<Collection<TAction>> {
312
- // Pre-construction for the same reason as in open(): see the comment there.
313
- const instanceTag = Collection.newInstanceTag();
314
- const { source, sourceCache, tracker, floors, header } = await Collection.probeHeader(transactor, id, instanceTag);
315
-
316
- if (header) { // Collection already exists
317
- await Collection.attachToLog<TAction>(source, transactor, tracker, id, instanceTag, header);
318
- } else { // Collection does not exist
319
- log('collection:invented id=%s — no committed header found; staging a fresh empty collection', id);
320
- const headerBlock = init.createHeaderBlock(id, tracker);
321
- tracker.insert(headerBlock);
322
- source.actionContext = undefined;
323
- await Log.open<Action<TAction>>(tracker, id);
324
- }
325
-
326
- const collection = new Collection(id, transactor, init.modules, source, sourceCache, tracker, floors, init.filterConflict, instanceTag);
327
- collection.logTailId = header?.tailId;
328
- return collection;
329
- }
330
-
331
- /** The per-instance read wiring every open path needs, plus the header probe result.
332
- * Shared by {@link open} and {@link createOrOpen} so the two cannot drift. */
333
- private static async probeHeader(transactor: ITransactor, id: CollectionId, instanceTag: string): Promise<{
334
- source: TransactorSource<IBlock>,
335
- sourceCache: CacheSource<IBlock>,
336
- tracker: Tracker<IBlock>,
337
- floors: BlockFloors,
338
- header: CollectionHeaderBlock | undefined,
339
- }> {
340
- // Start with a context that has an infinite revision number to ensure that we always fetch the latest log information.
341
- // One shared read-dependency collector feeds both the source (direct structural reads) and the cache (every
342
- // cache hit/miss), so a block read from either layer records a dependency — cache hits included.
343
- const collector = new ReadDependencyCollector();
344
- const floors = Collection.newFloors(id, instanceTag);
345
- const source = new TransactorSource(id, transactor, undefined, collector, floors);
346
- const sourceCache = new CacheSource(source, undefined, collector);
347
- const tracker = new Tracker(sourceCache);
348
- const header = await source.tryGet(id) as CollectionHeaderBlock | undefined;
349
- return { source, sourceCache, tracker, floors, header };
350
- }
351
-
352
- /** A new handle's floors: none yet (opening walks no entries), wired to report every below-floor
353
- * answer any of the handle's read sources receives. The line is the only trace such an answer
354
- * leaves — the read itself succeeds (see the accepted-tradeoff NOTE at
355
- * `TransactorSource.mayRetain`) — so repeated lines for one block with `servedRev` short of
356
- * `floorRev` are how an operator sees a machine that has not caught up, and lines that never
357
- * stop are how they see a log entry whose blocks never landed. */
358
- private static newFloors(id: CollectionId, instanceTag: string): BlockFloors {
359
- return new BlockFloors(({ blockId, floor, servedRev }) => {
360
- log('collection:block-below-floor id=%s tag=%s block=%s floorRev=%d floorAction=%s servedRev=%d',
361
- id, instanceTag, blockId, floor.rev, floor.actionId, servedRev);
362
- });
363
- }
364
-
365
- /** Walk an existing collection's log and point the source at its latest action context.
366
- * A header we just probed successfully but whose log will not open is a fault, not an
367
- * absence — throw rather than let the collection read as empty. (The re-read goes through
368
- * the tracker/cache, so it can disagree with the probe when storage is flaky mid-open.) */
369
- private static async attachToLog<TAction>(
370
- source: TransactorSource<IBlock>,
371
- transactor: ITransactor,
372
- tracker: Tracker<IBlock>,
373
- id: CollectionId,
374
- /** The tag the calling open path minted for the Collection it is ABOUT to construct, so a
375
- * diagnostic emitted here carries the same instance name as every post-construction one. */
376
- instanceTag: string,
377
- header: CollectionHeaderBlock,
378
- ): Promise<void> {
379
- // Bootstrap ActionContext from the committed tail before walking the chain.
380
- // This allows the transactor to serve pending non-tail blocks during Log.open.
381
- Collection.bootstrapContext(source, header.tailId === undefined ? undefined : await Collection.readLogTail(transactor, header.tailId));
382
-
383
- const collectionLog = await Log.open<Action<TAction>>(tracker, id);
384
- if (!collectionLog) {
385
- throw new Error(`Log not found for collection ${id}`);
386
- }
387
- // Monotonic, not an overwrite: getActionContext resolves undefined when the chain has no
388
- // tail or the tail block carries zero entries, and that must not erase the revision
389
- // bootstrapContext just read off the committed tail.
390
- Collection.advanceContext(source, id, instanceTag, 'attach', await collectionLog.getActionContext());
391
- }
392
-
393
- /** Adopt a freshly-read action context WITHOUT ever lowering the revision already held.
394
- *
395
- * The revision a collection last committed at is knowledge it earned; a read that found
396
- * nothing — or found an older view of the log — cannot un-earn it. Silently accepting the
397
- * lower value makes the next sync ask for a revision that is long gone, and every retry
398
- * repeats the same doomed request because each retry re-runs the same losing read.
399
- *
400
- * Equal revisions still adopt `next`: the rev is unchanged but its `committed` list may be
401
- * more complete than what we hold.
402
- *
403
- * This is also the one seam where lineage divergence is observable: at every revision BOTH
404
- * sides name an action for, the two ids must agree. Revision
405
- * numbers are per-collection counters, so two separately-built copies under one id can each
406
- * occupy the same revision with DIFFERENT actions while each stays internally self-consistent
407
- * — {@link reportShortfall} structurally cannot see that (its two numbers come from one
408
- * chain), and this is the only place two `committed` lists meet. Naming different actions at
409
- * one revision proves the two sides are different lineages (`collection:lineage-divergence`;
410
- * see docs/debugging.md § "Did the refresh itself fail to close the gap?"). WHICH two sides
411
- * depends on the caller, and the line says so in `site=` — see {@link DivergenceSite}, which
412
- * defines the two values and what each one indicts.
413
- *
414
- * Every line from here also carries `tag=`, the {@link Collection.instanceTag} of the handle
415
- * reporting. One process routinely holds several handles on one collection id; without the
416
- * tag, two handles' lines interleave into what reads like one handle contradicting itself.
417
- *
418
- * Logs, does not throw — same reasoning as {@link reportShortfall}: `update()` runs
419
- * blanket-style over every registered collection between commit retries, and aborting here
420
- * would promote a diagnosis to production behaviour before the line has ever been seen to
421
- * fire in the wild. Adoption then proceeds unchanged, which means the line is a PER-DISCOVERY
422
- * report, not a per-refresh one: adopting `next` overwrites the held lineage marker with the
423
- * log's, so the next refresh of this instance compares log-to-log and stays silent — even
424
- * though block content materialized under the old lineage may still be in caches. The line
425
- * marks the refresh that first observed the disagreement.
426
- *
427
- * NOTE: adoption resolves the CONTEXT disagreement, not the content one — the read caches on
428
- * this instance still hold blocks materialized under the old lineage, and since the revision
429
- * did not change nothing re-reads them. Conditional today: no fork has been reproduced (see
430
- * the still-open upstream reproducer), so this instrument exists to find out whether one
431
- * happens at all. If the line is ever seen firing in the field, decide then whether a
432
- * divergence should also drop the read cache (and whether to keep re-reporting per refresh)
433
- * — that is a behaviour change, and this seam deliberately makes none.
434
- *
435
- * The comparison is {@link earliestFork}, not a single lookup at the current revision: the
436
- * two `committed` lists overlap across several revisions, and the LOWEST one they disagree at
437
- * is where the lineages actually parted — a fork below the current revision was previously
438
- * silent. `forkRev=` names it, `heldAction=`/`readAction=` are the two ids AT it, and
439
- * `heldRev=`/`readRev=` are the two contexts' own revisions, so the line says both where the
440
- * split began and how far each side has since travelled.
441
- *
442
- * The refusal line reports its two action ids at `readRev=` — the read's revision — on BOTH
443
- * sides, because that is the only revision the two can be compared at: `next` never names an
444
- * action above its own revision, so looking each side up at its own revision would compare
445
- * different revisions and print two different ids for one honest lineage. Equal ids there mean
446
- * the read is an older view of THIS lineage (ordinary lag, correctly refused); different ids
447
- * mean a fork; `none` on the held side means this handle's own list does not reach back to the
448
- * read's revision — the signature of a context bootstrapped from an over-claiming tail (see
449
- * the NOTE in {@link bootstrapContext}), which is exactly the case {@link earliestFork} has no
450
- * shared revision to report on.
451
- *
452
- * Gated on `log.enabled`, like every {@link actionIdAt} caller: the comparison buys nothing
453
- * when the line has no sink, and the lists — one entry per commit between context reads,
454
- * truncated at each checkpoint — are only walked on a run that has the namespace turned on.
455
- * Silence proves nothing either way, because a revision is only comparable when BOTH sides
456
- * name an action for it: an invented collection has no context at all, a revision slot the log
457
- * gave to a checkpoint or invalidation entry names none, and a revision older than the read
458
- * log's most recent checkpoint has already fallen off the read side's list. */
459
- private static advanceContext(
460
- source: TransactorSource<IBlock>,
461
- id: CollectionId,
462
- instanceTag: string,
463
- site: DivergenceSite,
464
- next: ActionContext | undefined,
465
- ): void {
466
- const current = source.actionContext;
467
- if (next === undefined) {
468
- return; // The read learned nothing — keep what we already know.
469
- }
470
- if (current !== undefined && log.enabled) {
471
- const fork = Collection.earliestFork(current, next);
472
- if (fork !== undefined) {
473
- log('collection:lineage-divergence id=%s tag=%s site=%s forkRev=%d heldAction=%s readAction=%s heldRev=%d readRev=%d',
474
- id, instanceTag, site, fork.rev, fork.heldAction, fork.readAction, current.rev, next.rev);
475
- }
476
- }
477
- if (current !== undefined && next.rev < current.rev) {
478
- // The refusal itself is unconditional; only the id lookups that explain it are gated.
479
- if (log.enabled) {
480
- log('collection:context-not-lowered id=%s tag=%s site=%s heldRev=%d readRev=%d heldAction=%s readAction=%s',
481
- id, instanceTag, site, current.rev, next.rev,
482
- actionIdAt(current, next.rev) ?? 'none', actionIdAt(next, next.rev) ?? 'none');
483
- }
484
- return;
485
- }
486
- source.actionContext = next;
487
- }
488
-
489
- /** The EARLIEST revision the two contexts provably disagree about: the lowest revision both
490
- * `committed` lists name an action for, where the two ids differ.
491
- *
492
- * Comparing only at the holder's current revision — what this used to do — misses a fork that
493
- * began earlier and has since been overtaken by same-numbered commits on both sides, which is
494
- * the shape a replica that forked and kept writing actually has. Taking the lowest disagreeing
495
- * revision instead names the split point rather than an arbitrary later symptom of it.
496
- *
497
- * Revisions only one side names are skipped, not treated as disagreement: {@link actionIdAt}'s
498
- * `undefined` is legitimate (checkpoint/invalidation slots, and revisions that predate the
499
- * other side's most recent checkpoint), so a one-sided entry is missing evidence, not evidence
500
- * of a fork.
501
- *
502
- * NOTE: linear in the two lists, which hold one entry per commit between context reads and
503
- * truncate at each checkpoint. Every caller is `log.enabled`-gated, so this does not run at
504
- * all on a normal run; if a non-diagnostic caller ever appears, index by revision instead. */
505
- private static earliestFork(held: ActionContext, read: ActionContext): LineageFork | undefined {
506
- // NOTE: a `committed` list carrying TWO entries at one revision would be a defect in its own
507
- // right, and this keeps the last of them arbitrarily. Harmless while every caller is a
508
- // diagnostic; if such a list is ever seen, report the duplicate rather than silently
509
- // picking one.
510
- const readIds = new Map(read.committed.map(entry => [entry.rev, entry.actionId]));
511
- let earliest: LineageFork | undefined;
512
- for (const entry of held.committed) {
513
- const readAction = readIds.get(entry.rev);
514
- if (readAction === undefined || readAction === entry.actionId) {
515
- continue;
516
- }
517
- if (earliest === undefined || entry.rev < earliest.rev) {
518
- earliest = { rev: entry.rev, heldAction: entry.actionId, readAction };
519
- }
520
- }
521
- return earliest;
522
- }
523
-
524
- /** Report a refresh that failed to move FORWARDS past a revision it had already read for
525
- * itself — the sibling of {@link advanceContext}'s `collection:context-not-lowered`, which
526
- * reports a collection declining to move BACKWARDS.
527
- *
528
- * `tailRev` is what the committed log tail claimed is committed under this id; `after` is
529
- * where a SEPARATE read path (the chain walk) actually landed. Landing below the claim means
530
- * this refresh closed nothing, which from outside the class is otherwise indistinguishable
531
- * from "there was nothing newer to adopt".
532
- *
533
- * This detects LAG, and only lag. It CANNOT see lineage divergence: both of its numbers
534
- * come from the same chain — `tailRev` off the tail block this collection's own header
535
- * names, `after` from a walk of that same chain — and a forked replica is internally
536
- * self-consistent, its tail claiming exactly what its own walk reaches. Two copies of one
537
- * collection id holding the same revision under different actions therefore keep this line
538
- * silent forever. That case is `collection:lineage-divergence`, reported from
539
- * {@link advanceContext}, which compares action ids — the one value comparable across
540
- * copies — rather than revision counters.
541
- *
542
- * Carries the same `tag=` as {@link advanceContext}'s lines, and for the same reason: several
543
- * handles on one collection id otherwise read as one self-contradicting handle.
544
- *
545
- * Logs, does not throw: `update()` is called blanket-style over every registered collection
546
- * between commit retries, and a shortfall is not yet known to be illegitimate — an abort here
547
- * would promote an unproven diagnosis to production behaviour. Deliberately does NOT adopt
548
- * `tailRev` either: the two numbers come from different read paths, and papering over the
549
- * disagreement destroys the evidence this line exists to produce. */
550
- private static reportShortfall(id: CollectionId, instanceTag: string, tailRev: number | undefined, before: number | undefined, after: number | undefined): void {
551
- if (tailRev === undefined || (after !== undefined && after >= tailRev)) {
552
- return;
553
- }
554
- log('collection:context-short-of-tail id=%s tag=%s before=%s after=%s tail=%d',
555
- id, instanceTag, before ?? 'none', after ?? 'none', tailRev);
556
- }
557
-
558
- async act(...actions: Action<TAction>[]) {
559
- const release = await Latches.acquire(this.latchId);
560
- try {
561
- await this.actInternal(...actions);
562
- } finally {
563
- release();
564
- }
565
- }
566
-
567
- private async actInternal(...actions: Action<TAction>[]) {
568
- await this.internalTransact(...actions);
569
- this.pending.push(...actions);
570
- }
571
-
572
- private async internalTransact(...actions: Action<TAction>[]) {
573
- const atomic = new Atomic(this.tracker);
574
-
575
- for (const action of actions) {
576
- const handler = this.handlers[action.type];
577
- if (!handler) {
578
- throw new Error(`No handler for action type ${action.type}`);
579
- }
580
- await handler(action, atomic);
581
- }
582
-
583
- atomic.commit();
584
- }
585
-
586
- /** Load external changes and update our context to the latest log revision - resolve any conflicts with our pending actions. */
587
- async update() {
588
- const release = await Latches.acquire(this.latchId);
589
- try {
590
- await this.updateInternal({});
591
- } finally {
592
- release();
593
- }
594
- }
595
-
596
- /** The refresh `TransactionCoordinator.commit` runs between attempts: exactly {@link update},
597
- * plus a report of whether it finished the write in flight on this instance's behalf (see
598
- * {@link RefreshReport}). The coordinator needs that fact to tell its caller which participants
599
- * are already saved when the commit later fails; a reader's `update()` has no write in flight
600
- * and nothing to report.
601
- *
602
- * `report` is REQUIRED so a write path cannot refresh without learning what the refresh saved.
603
- * It is filled in as the refresh goes, so it is accurate when this throws too.
604
- *
605
- * `lastChance` is REQUIRED for the same reason: only the caller knows whether its retry budget
606
- * ends with this round, and a round that is the last must settle a half-landed write rather than
607
- * ask for another (see {@link completeOwnEntry}). */
608
- async refreshInFlight(report: RefreshReport, lastChance: boolean): Promise<void> {
609
- const release = await Latches.acquire(this.latchId);
610
- try {
611
- await this.updateInternal(report, lastChance);
612
- } finally {
613
- release();
614
- }
615
- }
616
-
617
- /** Finishes a half-landed write BEFORE its own log entry is consumed, so that consuming never
618
- * reports a write as saved on the strength of its log entry alone.
619
- *
620
- * THE RULE: a write may be reported saved only if EVERY block its log entry names holds the
621
- * write — at the write's own revision, or at a later revision that was BUILT FROM it. Finding the
622
- * entry proves only that the log tail landed (see {@link inFlightActionId} for the two ways
623
- * `NetworkTransactor.commit` answers failure over a stored tail). The writer's cancel has since
624
- * removed the pending records of every block that did not land, so nothing else will ever land
625
- * them — if this does not, the entry stands in the log, the blocks stay at their previous
626
- * revision on every node, and readers materialize blocks, not log entries: the write is
627
- * silently gone.
628
- *
629
- * Finishing is a plain re-send of the retained failed attempt ({@link inFlightAttempt}) — the
630
- * SAME transforms, action id, revision and tail. Every storage tier treats a block that already
631
- * holds exactly this action at exactly this revision as satisfied rather than as a rival
632
- * (`isOwnRevision`: `StorageRepo.pend`/`.commit`, `ClusterMember`, `CoordinatorRepo`), so the
633
- * re-send rolls forward precisely the blocks that are missing and is a no-op for the rest,
634
- * including when nothing is missing at all. It is never sent at a new revision: the refresh has
635
- * already seen the entry at this one, and a second revision would record the entry twice.
636
- *
637
- * A refused re-send is NOT yet an answer. Storage refuses it whenever any block has moved past
638
- * the write's revision — and every later commit to the collection moves the log tail past it —
639
- * which says a rival was there, not whether the rival built on this write or over it. That is
640
- * asked separately, of the blocks' own history ({@link settleUnfinished}).
641
- *
642
- * Runs at the top of {@link updateInternal}, before that method has changed anything on this
643
- * instance, so every throw from here leaves the collection exactly as the failed attempt left
644
- * it — staged actions and transforms intact, revision not advanced.
645
- *
646
- * @param lastChance - the caller will not refresh again for this write (its retry budget ends
647
- * with this round), so a refusal that could clear is settled now instead of asking for another
648
- * round: the error that escapes then says whether the write can still land.
649
- * @returns who holds the finished write, for the sync to report.
650
- * @throws TornActionError — see {@link TornActionReason} for the three causes. Only an unsettled
651
- * `completion-refused` (always `final: false`) is retried by the write paths.
652
- *
653
- * NOTE: the re-send costs a full pend and commit round even when every block had in fact landed
654
- * (a lost or masked success). That is deliberate — it is the source of the durability the sync
655
- * reports when nothing has superseded the write — and the case is rare: after a returned failure
656
- * the network transactor has, by construction, NOT swept every block. If own-entry refreshes ever
657
- * show up as a cost, ask `getLineage` first and skip the re-send when every block contains the
658
- * write. */
659
- private async completeOwnEntry(entry: ActionEntry<Action<TAction>>, entryRev: number | undefined, lastChance: boolean): Promise<WriteDurability | undefined> {
660
- const attempt = this.inFlightAttempt;
661
- const rev = entryRev ?? attempt?.rev;
662
- if (attempt === undefined || rev === undefined || attempt.rev !== rev) {
663
- // Nothing to finish the action WITH. That is only acceptable if there is nothing to
664
- // finish. `getStatus` is the cheap first question; it judges a block by who holds its
665
- // LATEST revision, so a block this action landed and a later action has since built on
666
- // reads there as not committed — those are asked again, properly, by settleUnfinished.
667
- // Both write paths retain an attempt before any refresh can run, so production only
668
- // reaches this branch when the entry sits at a revision the retained attempt was not
669
- // made at — a forked lineage.
670
- const [status] = await this.transactor.getStatus([{ actionId: entry.actionId, blockIds: entry.blockIds }]);
671
- if (entry.blockIds.every((_, i) => status?.statuses[i] === 'committed')) {
672
- // Whole, and saved: the refresh reports it as finished (see RefreshReport), so the
673
- // coordinator counts this participant as committed. Only WHO holds it is unknown.
674
- // NOTE: a sync whose only commit was recognised here therefore answers `undefined` —
675
- // the "nothing was written" answer — for a write that is saved, because the sync reads
676
- // the report's durability, not the finished flag. Reachable only on the forked-lineage
677
- // path above; if that path ever becomes ordinary, take the durability from
678
- // `getLineage` (as settleUnfinished does) instead of stopping at the status read.
679
- return undefined;
680
- }
681
- return await this.settleUnfinished(entry, rev, undefined, 'transforms-not-held',
682
- attempt === undefined
683
- ? 'no failed attempt is retained for this action'
684
- : `the retained attempt was made at rev ${attempt.rev}`);
685
- }
686
-
687
- // NOTE: priority 0. The attempt's aged retry priority is a fairness hint for a race over a
688
- // free revision; this revision is already this action's own, so there is no race to rank in.
689
- // NOTE: this is a plain pend. When the failed attempt came from `TransactionCoordinator`, its
690
- // pend carried `validation` (the transaction and its operations hash) and
691
- // `superclusterNominees`; neither is retained, so the re-send carries neither. Harmless while
692
- // no deployment hands members a transaction validator (and nothing reads the nominees on
693
- // the receiving side at all). Once a validator is wired, members approve these
694
- // blocks unchecked under `unvalidatablePendPolicy: 'accept'` and refuse them under `'reject'`
695
- // (surfacing as a `completion-refused` TornActionError). Simply retaining and re-sending the
696
- // pair is not obviously right either: a member re-executing the transaction after a sibling
697
- // participant has landed no longer sees the state it was staged against. Tracked as an arm
698
- // of tickets/backlog/feat-no-deployment-validates-transactions-at-pend.
699
- const result = await this.source.transact(attempt.transforms, entry.actionId, rev, this.id, attempt.tailId, 0, attempt.blockDigests, attempt.baseRevs);
700
- if (result.success) {
701
- return result.durability;
702
- }
703
- // A refusal that CONFIRMS a committed revision under another action — `staleAt`, which every
704
- // producer sets only after reading it out of its own storage and never for this action's own
705
- // revision, or a non-empty `missing` list of committed rival transforms — will be repeated by
706
- // every later re-send. Anything else (a rival merely PENDING on a block, a revision not yet
707
- // held by a majority, a bare reason) can clear, and is worth the caller's next round.
708
- // NOTE: `staleAt` is read here as "a rival committed", which is a second consumer of a field
709
- // documented as never a retryability signal (docs/internals.md). It is the same kind of use
710
- // `syncAttempts`' stall check makes: it can only END a retry, never start one.
711
- const rivalConfirmed = result.staleAt !== undefined || (result.missing?.length ?? 0) > 0;
712
- // NOTE: a confirmed rival settles at once, even with retry budget left: re-sending is
713
- // pointless, but the lineage question is asked only this once, so a member that is silent
714
- // just now makes the answer `final: false`. If unsettled outcomes ever show up often under
715
- // contention, spend the remaining rounds re-asking `getLineage` before giving up.
716
- if (!rivalConfirmed && !lastChance) {
717
- // The refusal does not say which blocks lack the revision, so name every block the entry
718
- // lists other than the tail — the entry being visible is what proves the tail holds it.
719
- return this.throwTorn(entry, rev, entry.blockIds.filter(blockId => blockId !== attempt.tailId),
720
- 'completion-refused', false, result.reason ?? 'the re-send was refused');
721
- }
722
- return await this.settleUnfinished(entry, rev, attempt.tailId,
723
- rivalConfirmed ? 'rival-holds-revision' : 'completion-refused',
724
- result.reason ?? (rivalConfirmed ? 'a different action holds a later revision' : 'the re-send was refused'),
725
- result.staleAt);
726
- }
727
-
728
- /** Decides what a write that can no longer be finished BY RE-SENDING amounts to, and answers
729
- * one of exactly three things — never a guess between them:
730
- *
731
- * - SAVED (returns): every block the entry names holds content built from the write, on a
732
- * strict majority of its cohort (`ITransactor.getLineage`). The rival that refused the re-send
733
- * had read this write and added to it. Returns who holds it.
734
- * - TORN AND FINAL (throws, `final: true`): every block still missing the write answered, for its
735
- * whole cohort, that it does not hold it — and the write's pending records were confirmed gone
736
- * BEFORE the blocks were asked. The order is the point: a pending record left standing can be
737
- * promoted by any later read that knows this write's log entry is committed
738
- * (`StorageRepo.get`), so a block that answered "not reached" could still take the write
739
- * afterwards. With the records gone first, nothing is left that could land it.
740
- * - TORN, OUTCOME NOT ESTABLISHED (throws, `final: false`): anything else — the transactor
741
- * cannot answer for lineage, a cohort did not all answer or contradicted itself, fewer than a
742
- * majority hold the write, or the cancel could not be confirmed.
743
- *
744
- * `tailId` is the log block this write's entry was appended to, when known. It is never asked
745
- * about: the refresh has just READ the entry out of it, which is direct evidence that the
746
- * block's current content was built from the write, whatever any member's records can prove.
747
- *
748
- * NOTE: a write recognised as saved here reports the durability `getLineage` assembled. When
749
- * the log block's own cohort could not vouch for it (members that took it as a replica), that is
750
- * absent and the sync answers `undefined` for a saved write — the same wart the status-read
751
- * branch of {@link completeOwnEntry} documents. If it is ever seen, fold the data blocks'
752
- * reports alone rather than inventing one for the log block. */
753
- private async settleUnfinished(
754
- entry: ActionEntry<Action<TAction>>,
755
- rev: number | undefined,
756
- tailId: BlockId | undefined,
757
- reason: TornActionReason,
758
- detail: string,
759
- staleAt?: { blockId: BlockId; rev: number },
760
- ): Promise<WriteDurability | undefined> {
761
- const discharged = await this.dischargeOwnPendings(entry);
762
- const lineage = rev === undefined ? undefined : await this.lineageOfOwnEntry(entry, rev);
763
- const unsaved = entry.blockIds.filter((blockId, i) => blockId !== tailId && lineage?.blocks[i] !== 'contains');
764
- if (lineage !== undefined && unsaved.length === 0) {
765
- log('collection:own-entry-superseded-but-saved id=%s tag=%s action=%s rev=%d blocks=%d',
766
- this.id, this.instanceTag, entry.actionId, rev, entry.blockIds.length);
767
- return lineage.durability;
768
- }
769
- const cannotLand = lineage !== undefined && unsaved.every(blockId => {
770
- const answer = lineage.blocks[entry.blockIds.indexOf(blockId)];
771
- return answer === 'excludes' || answer === 'behind';
772
- });
773
- return this.throwTorn(entry, rev ?? -1, unsaved, reason, discharged && cannotLand, detail, staleAt);
774
- }
775
-
776
- /** Cancels every pending record this write may have left, and says whether that is CONFIRMED.
777
- * `ITransactor.cancel` returns only once the records are gone and throws otherwise; a throw here
778
- * is reported as "not confirmed" rather than raised, because the caller is already reporting a
779
- * torn write and that must not be displaced. Cancelling a block that holds no record is a no-op,
780
- * so naming every block the entry lists is safe. */
781
- private async dischargeOwnPendings(entry: ActionEntry<Action<TAction>>): Promise<boolean> {
782
- try {
783
- await this.transactor.cancel({ actionId: entry.actionId, blockIds: entry.blockIds });
784
- return true;
785
- } catch (err) {
786
- log('collection:torn-cancel-unconfirmed id=%s tag=%s action=%s error=%s',
787
- this.id, this.instanceTag, entry.actionId, err instanceof Error ? err.message : String(err));
788
- return false;
789
- }
790
- }
791
-
792
- /** What the blocks' own history says about this write, or `undefined` when nothing can say: the
793
- * transactor (or a wrapper around it) does not offer `getLineage`, or the question failed. */
794
- private async lineageOfOwnEntry(entry: ActionEntry<Action<TAction>>, rev: number): Promise<ActionLineage | undefined> {
795
- if (this.transactor.getLineage === undefined) {
796
- return undefined;
797
- }
798
- try {
799
- return await this.transactor.getLineage({ actionId: entry.actionId, blockIds: entry.blockIds, rev });
800
- } catch (err) {
801
- log('collection:lineage-unanswered id=%s tag=%s action=%s rev=%d error=%s',
802
- this.id, this.instanceTag, entry.actionId, rev, err instanceof Error ? err.message : String(err));
803
- return undefined;
804
- }
805
- }
806
-
807
- private throwTorn(
808
- entry: ActionEntry<Action<TAction>>,
809
- rev: number,
810
- blockIds: BlockId[],
811
- reason: TornActionReason,
812
- final: boolean,
813
- detail: string,
814
- staleAt?: { blockId: BlockId; rev: number },
815
- ): never {
816
- log('collection:torn id=%s tag=%s action=%s rev=%d reason=%s final=%s blocks=%o',
817
- this.id, this.instanceTag, entry.actionId, rev, reason, final, blockIds);
818
- throw new TornActionError(this.id, entry.actionId, rev, blockIds, reason, final, detail, staleAt);
819
- }
820
-
821
- /** Drops the pending actions this sync's OWN committed entry made durable, instead of replaying
822
- * them into a duplicate entry (see {@link inFlightActionId}). Only ever called once
823
- * {@link completeOwnEntry} has returned for this entry — the entry alone is not proof the write
824
- * is saved.
825
- *
826
- * `addActions` wrote exactly the snapshot pending list under this action id, and the entry's
827
- * actions are therefore the LEADING `entry.actions.length` items of `this.pending` — anything
828
- * staged since is behind them, because {@link actInternal} appends. Under
829
- * {@link syncInternal} nothing can even be staged mid-cycle (`act()` shares the collection
830
- * latch); under `TransactionCoordinator.commit` the mark spans a latch-free inter-attempt
831
- * window, so an `act()` there CAN grow `pending` — still only at the tail, so the slice stays
832
- * right.
833
- *
834
- * The guard below is the load-bearing part: `slice` fails SILENTLY if that correspondence ever
835
- * breaks, dropping actions that were never committed, so an entry longer than `pending` throws
836
- * instead of losing work. (See the sibling note on `syncInternal`'s post-commit replay, which
837
- * rests on the same invariant.) */
838
- private consumeOwnEntry(entry: ActionEntry<Action<TAction>>) {
839
- if (entry.actions.length > this.pending.length) {
840
- throw new Error(
841
- `Collection ${this.id}: own committed entry for action ${entry.actionId} holds `
842
- + `${entry.actions.length} actions but only ${this.pending.length} are pending; `
843
- + `consuming it would drop actions that were never committed`);
844
- }
845
- // `mutated` is unconditional, even for a zero-action entry: the tracker still holds this
846
- // action's staged transforms, and only the replay at the end of `updateInternal` — which
847
- // resets the tracker and re-stages just what remains — drops them. That reset is what turns
848
- // `hasUnsyncedChanges()` false so the sync loop exits reporting the success the writer is
849
- // owed (the action IS durable).
850
- return { after: this.pending.slice(entry.actions.length), mutated: true };
851
- }
852
-
853
- /** Maps each pending action to its effective form against a remote entry: the original, a
854
- * replacement, or dropped. A replacement or a discard changes the pending set; the tracker still
855
- * holds the pre-filter transforms, so report it as mutated to force a replay that re-stages
856
- * against the effective actions. Identity comparison per the contract: keep => same instance,
857
- * replace => new instance.
858
- * NOTE: a filterConflict hook that always allocates a fresh (but equal) instance instead of
859
- * returning the same one forces a replay on every update — if that ever shows up as a hot path,
860
- * compare by value/id here instead of by reference. */
861
- private filterAgainstEntry(entry: ActionEntry<Action<TAction>>) {
862
- const before = this.pending;
863
- const after = before
864
- .map(p => this.doFilterConflict(p, entry.actions))
865
- .filter((a): a is Action<TAction> => a !== undefined);
866
- return { after, mutated: after.length !== before.length || after.some((a, i) => a !== before[i]) };
867
- }
868
-
869
- /** Refresh this instance against the stored log: adopt the latest committed revision, resolve
870
- * pending actions against everything that landed since, and replay them if anything conflicts.
871
- *
872
- * Takes no in-flight action id — it reads {@link inFlightActionId} off `this`, which is set for
873
- * exactly the write attempt cycles that own one (see that field). Callers cannot get this wrong
874
- * by omission.
875
- *
876
- * @param report - Filled in as the refresh goes (see {@link RefreshReport}): its
877
- * `ownEntryFinished` is set the moment the refresh has found this write's own log entry and
878
- * finished it ({@link completeOwnEntry}) — before the entry is consumed, so a later throw from
879
- * here still leaves it set. Never set on a reader's refresh. A caller with no use for it passes `{}`.
880
- * @param lastChance - Whether the write in flight will get no further refresh (see
881
- * {@link completeOwnEntry}). Meaningless, and left false, on a reader's refresh.
882
- * @throws TornActionError when it found that entry and could not finish the action — thrown
883
- * before anything on this instance changed, and with `report` untouched. */
884
- private async updateInternal(report: RefreshReport, lastChance = false): Promise<void> {
885
- // Start with a context that can see to the end of the log
886
- const source = new TransactorSource(this.id, this.transactor, undefined);
887
-
888
- // A header the storage layer could not retrieve throws BlockUnavailableError out of
889
- // this read (it is not a StaleFailure, so sync's retry loop does not absorb it).
890
- const ends = await Collection.readLogEnds(this.transactor, this.id, this.logTailId);
891
- if (!ends) {
892
- if (this.source.actionContext) {
893
- // An absent header is only believable for a collection that has never committed.
894
- // We hold a committed revision, so the two answers contradict each other — surface it
895
- // as a fault instead of no-opping into a forgotten revision and a rev-1 retry spin.
896
- // NOTE: this aborts every caller of update(), including TransactionCoordinator's
897
- // blanket refresh of ALL registered collections between commit retries — a
898
- // non-participant with a momentarily-absent header now fails the whole retry rather
899
- // than being skipped (the coordinator still refreshes the remaining collections first,
900
- // so a participant that can be finished is). That is the intended loud failure; if it ever shows up as
901
- // otherwise-healthy transactions aborting, narrow that refresh to the transaction's
902
- // participants (see the note at coordinator.ts's update loop) rather than softening
903
- // this throw.
904
- throw new CollectionHeaderVanishedError(this.id, this.source.actionContext.rev);
905
- }
906
- // The header is genuinely absent AND we hold no revision: nothing was ever committed under
907
- // this id, so there is no log to walk and nothing to adopt — correct here, rather than a
908
- // masked failure.
909
- return;
910
- }
911
- this.logTailId = ends.header.tailId;
912
- // Bootstrap context from committed tail so pending blocks are accessible.
913
- Collection.bootstrapContext(source, ends.tail);
914
-
915
- // The revision the committed tail just claimed, captured before anything else can touch
916
- // the local source. This is the authoritative "latest committed under this id" number,
917
- // read straight off the tail block's state; the chain walk below arrives at its own
918
- // number by a different path, and the two disagreeing is worth saying out loud (see the
919
- // {@link reportShortfall} call after advanceContext). Stays undefined when there is no
920
- // tail, or a tail with no `latest` — both legitimate "nothing committed yet" states.
921
- const tailRev = source.actionContext?.rev;
922
-
923
- const actionContext = this.source.actionContext;
924
- // A write's retry refresh always walks. Not needed for soundness — the write's own entry would
925
- // sit above the held revision, which the tail test already refuses — but losing that entry
926
- // loses the write, and a retry refresh is rare enough that the walk costs nothing that matters.
927
- if (this.inFlightActionId === undefined && Collection.tailShowsNothingNewer(actionContext, ends.tail)) {
928
- return;
929
- }
930
-
931
- // One block cache for the whole refresh, seeded with the header and tail just read: Log.open,
932
- // the entry walk and the invalidation walk each start again from the header and the tail, and
933
- // would otherwise fetch both every time. The seed came from UNPINNED reads and is served to
934
- // reads pinned at the tail's claim, which is sound: the tail's content at its own claimed
935
- // revision is that pinned view, and the header changes only when the tail block fills, which
936
- // Chain.getTail already tolerates by following `nextId` from whichever tail the header names.
937
- // NOTE: the cache keeps its default size (128 blocks, about 4,000 log entries). The entry walk
938
- // reads every log block back to the head (no checkpoints), newest first, so past that size
939
- // the newest blocks are evicted first and the invalidation walk fetches them again. If logs
940
- // get that long before checkpoints land, walk entries and invalidations in one pass.
941
- const tracker = new Tracker(new CacheSource(source, undefined, undefined, ends.served));
942
-
943
- // Get the latest entries from the log, starting from where we left off
944
- const collectionLog = await Log.open<Action<TAction>>(tracker, this.id);
945
- const latest = collectionLog ? await collectionLog.getFrom(actionContext?.rev ?? 0) : undefined;
946
-
947
- // This write's own entry, if its log tail landed despite the failure that sent us here.
948
- // Decided ONCE, here: the mark can be cleared latch-free while the completion below is
949
- // awaiting (see {@link inFlightActionId}), and an entry that was finished as our own must
950
- // not then be run through the conflict filter as a stranger's and replayed.
951
- const inFlightActionId = this.inFlightActionId;
952
- const ownEntry = inFlightActionId === undefined
953
- ? undefined
954
- : latest?.entries.find(entry => entry.actionId === inFlightActionId);
955
- // Finish it BEFORE anything below changes this instance — see {@link completeOwnEntry}. A
956
- // throw from here (the action cannot be finished, or not yet) therefore abandons the refresh
957
- // with the staged actions, the tracker and the held revision exactly as they were.
958
- // The entry's revision comes from the context the same walk built; an entry older than a
959
- // checkpoint is not restated there, and completeOwnEntry falls back to the attempt's own.
960
- const entryRevs = Collection.revisionsByAction(latest?.context);
961
- if (ownEntry !== undefined) {
962
- const durability = await this.completeOwnEntry(ownEntry, entryRevs.get(ownEntry.actionId), lastChance);
963
- // Saved from here on, whatever below throws — record it before anything else can.
964
- report.ownEntryFinished = { durability };
965
- }
966
-
967
- // Process the entries and track the blocks they affect
968
- let anyConflicts = false;
969
- for (const entry of latest?.entries ?? []) {
970
- const isOwnEntry = entry === ownEntry;
971
- const { after, mutated } = isOwnEntry
972
- ? this.consumeOwnEntry(entry)
973
- : this.filterAgainstEntry(entry);
974
- this.pending = after;
975
- anyConflicts = anyConflicts || mutated || this.tracker.conflicts(new Set(entry.blockIds)).length > 0;
976
- }
977
-
978
- // React to durable invalidations that landed since we last synced. getFrom intentionally skips
979
- // invalidation entries (they are not pending/committed actions), so surface them separately: an
980
- // invalidation reverted committed content this client may have read, so treat it like a stale
981
- // read — drop the reverted blocks from the read cache and replay pending work against the reverted
982
- // base (docs/right-is-right.md §Client notification). De-duped across cascade children by reverted
983
- // block; over-inclusive by design (over-invalidation just resubmits — it never wrongly retains).
984
- const invalidations = collectionLog ? await collectionLog.getInvalidationsFrom(actionContext?.rev ?? 0) : [];
985
- const revertedBlockIds = [...new Set(invalidations.flatMap(inv => inv.reverted.map(r => r.blockId)))];
986
- if (invalidations.length > 0 && this.pending.length > 0) {
987
- anyConflicts = true;
988
- }
989
-
990
- this.forgetAndAdopt(latest, entryRevs, revertedBlockIds);
991
-
992
- Collection.reportShortfall(this.id, this.instanceTag, tailRev, actionContext?.rev, this.source.actionContext?.rev);
993
-
994
- // Re-stage the pending actions against the adopted revision. The affected
995
- // blocks were already dropped from sourceCache above (per log entry / per invalidation),
996
- // so the replay's reads re-materialize from the transactor.
997
- // NOTE: a throw out of replayActions leaves the tracker holding only the transforms
998
- // replayed so far while `pending` still lists them all; the caller's error handling is
999
- // expected to abort/reset the collection rather than keep staging. The coordinator's partial
1000
- // report after a refresh-saved sibling does not (backlog: debt-a-failed-refresh-can-leave-a-
1001
- // collection-half-restaged). If replay ever gains a
1002
- // routinely-throwing read path, rebuild into a scratch tracker and swap on success.
1003
- if (this.mustReplay(anyConflicts, actionContext)) {
1004
- await this.replayActions();
1005
- }
1006
- }
1007
-
1008
- /** Forget every block the refresh saw change, floor the ones a log entry names, and adopt the
1009
- * revision the log is at — ONE synchronous step, which must stay free of any `await`.
1010
- *
1011
- * Reads are not latched, so one can run while a refresh is under way. Forgetting a block while
1012
- * this handle still reads at the revision it is LEAVING invites exactly the wrong re-read: the
1013
- * block comes back as it was at that revision (correctly — that is what was asked for, and no
1014
- * floor applies to a read below it), the cache keeps it, and the advance that follows turns it
1015
- * into old content nothing will ever clear, the entry that named it having been consumed. That
1016
- * needs no lagging machine; storage can be perfectly current. With no gap between the forgetting
1017
- * and the adopting, a concurrent read either lands before both and is forgotten with the rest,
1018
- * or lands after both and is judged at the adopted revision — against the floor
1019
- * ({@link TransactorSource.tryGet} reads its context when the answer arrives, not when it was
1020
- * asked for) and against the generation the clear moved (`CacheSource.stillWanted`).
1021
- *
1022
- * The advance is monotonic (see {@link advanceContext}): an empty or unopenable log yields no
1023
- * context at all, and a log read that lags what this handle already committed yields an older
1024
- * one; neither is grounds for forgetting the revision held. It also has to precede
1025
- * {@link replayActions}, which re-reads blocks through `this.source` at whatever revision the
1026
- * context names — replaying first would refill the cache at the revision being left. */
1027
- private forgetAndAdopt(
1028
- latest: GetFromResult<Action<TAction>> | undefined,
1029
- entryRevs: ReadonlyMap<ActionId, number>,
1030
- revertedBlockIds: BlockId[],
1031
- ): void {
1032
- for (const entry of latest?.entries ?? []) {
1033
- this.sourceCache.clear(entry.blockIds);
1034
- this.raiseFloors(entry, entryRevs.get(entry.actionId));
1035
- }
1036
- this.sourceCache.clear(revertedBlockIds);
1037
- Collection.advanceContext(this.source, this.id, this.instanceTag, 'refresh', latest?.context);
1038
- }
1039
-
1040
- /** The revision each action in `context` committed at, keyed by action id. `Log.getFrom` returns
1041
- * entries without their revisions; the context the same walk built is where they are restated. */
1042
- private static revisionsByAction(context: ActionContext | undefined): Map<ActionId, number> {
1043
- return new Map((context?.committed ?? []).map(({ actionId, rev }) => [actionId, rev]));
1044
- }
1045
-
1046
- /** Records that the walked `entry` changed the blocks it names, so a later answer for one of
1047
- * them that is older than the entry is recognised and never remembered (see {@link BlockFloors}).
1048
- * Beside the cache clear on purpose (see {@link forgetAndAdopt}): the clear is what sends the next
1049
- * read of these blocks back to storage, and that read is the one a lagging machine can answer too
1050
- * old — after which nothing would clear the block again, this entry having been consumed.
1051
- *
1052
- * NOTE: an entry whose revision the walk did not restate sets no floor. That is an entry older
1053
- * than the log's most recent checkpoint, and no checkpoint is written today
1054
- * (tickets/backlog/debt-the-collection-log-never-writes-a-checkpoint). Once they are, such an
1055
- * entry's blocks go unguarded unless `Log.getFrom` starts returning each entry's revision. */
1056
- private raiseFloors(entry: ActionEntry<Action<TAction>>, rev: number | undefined): void {
1057
- if (rev !== undefined) {
1058
- this.floors.raise(entry.blockIds, { rev, actionId: entry.actionId });
1059
- }
1060
- }
1061
-
1062
- /** Whether the log tail a refresh just read proves the log holds nothing newer than `held` —
1063
- * the test that lets a refresh with nothing to find stop after one request instead of walking
1064
- * the log.
1065
- *
1066
- * Sound because every commit and every invalidation appends a log entry, and entries only ever
1067
- * go on the tail block. A tail block that is still the end of the chain (no `nextId`), whose
1068
- * newest entry is at the held revision and names the action `held` names there — and whose own
1069
- * claim (`state.latest`) says the same — has nothing above `held`. The walk would then find no
1070
- * entries, no invalidations, and a context {@link advanceContext} adopts at an unchanged
1071
- * revision, so it would change nothing.
1072
- *
1073
- * Says "no" — sending the refresh down the full walk — whenever the tail and the held context
1074
- * disagree in any way, so everything that walk reports still fires:
1075
- * - a claim above `held` (ordinary catch-up) or below it (a lagging read);
1076
- * - a newest entry that is not the claimed action, or not an action at all: the entries lag the
1077
- * claim (`collection:context-short-of-tail`, or `collection:context-not-lowered` for a handle
1078
- * pinned by an over-claiming tail), or an invalidation or checkpoint took the newest slot;
1079
- * - any action entry in the tail block naming a different action than `held` names at the same
1080
- * revision (`collection:lineage-divergence`, and the walk's adoption of the log's list).
1081
- *
1082
- * NOTE: the lineage comparison sees only the entries the tail block holds. A fork below them, on
1083
- * a log that has not moved, is not looked at again until a refresh finds something new — that
1084
- * walk compares the whole list. Walking every time to look is exactly the cost this avoids.
1085
- *
1086
- * NOTE: an invalidation entry in the newest slot never matches (it names no action), so every
1087
- * refresh after one walks the log until the next commit lands. Fine while disputes are rare; if
1088
- * they are not, match an invalidation slot against the held revision too. */
1089
- private static tailShowsNothingNewer(held: ActionContext | undefined, tail: GetBlockResult | undefined): boolean {
1090
- const claim = tail?.state.latest;
1091
- const block = tail?.block as LogBlock<unknown> | undefined;
1092
- if (held === undefined || claim === undefined || !block || block.nextId !== undefined) {
1093
- return false;
1094
- }
1095
- const newest = block.entries[block.entries.length - 1];
1096
- return claim.rev === held.rev
1097
- && newest?.rev === claim.rev
1098
- && newest.action?.actionId === claim.actionId
1099
- && actionIdAt(held, held.rev) === claim.actionId
1100
- && !Collection.disagreesWithHeld(held, block.entries);
1101
- }
1102
-
1103
- /** Whether any action entry in `entries` names a different action than `held` names at the same
1104
- * revision. A revision only one side names is missing evidence, not disagreement — the rule
1105
- * {@link earliestFork} applies too. */
1106
- private static disagreesWithHeld(held: ActionContext, entries: readonly LogEntry<unknown>[]): boolean {
1107
- const logged = new Map(entries.flatMap(entry => entry.action ? [[entry.rev, entry.action.actionId] as const] : []));
1108
- return held.committed.some(entry => {
1109
- const loggedAction = logged.get(entry.rev);
1110
- return loggedAction !== undefined && loggedAction !== entry.actionId;
1111
- });
1112
- }
1113
-
1114
- /** Whether {@link updateInternal} must re-stage `pending` after adopting `latest`, given the
1115
- * context it held BEFORE the refresh.
1116
- *
1117
- * The answer is deliberately NOT just "a conflict was found". A conflict is detected by an
1118
- * incoming log entry naming a block this tracker already holds a transform for — so an action
1119
- * that changed NO block can never register one. A staged delete of a key this instance cannot
1120
- * see is exactly that action: the tree's `replace` handler misses on `find` and `deleteAt`
1121
- * returns false without writing. Gated on conflicts alone, such an action stayed in `pending`
1122
- * unapplied until the commit wrote a log entry listing it whose transforms did nothing — and
1123
- * readers materialize blocks, not log entries, so the action was lost on every node, silently
1124
- * and permanently. The invariant that has to hold is that **a pending action was applied
1125
- * against the revision it commits over**, so a mere revision advance is reason enough.
1126
- *
1127
- * The advance test rests on {@link advanceContext} being monotonic (it refuses to lower), which
1128
- * is what makes "the adopted rev differs from the held one" mean "it went up".
1129
- *
1130
- * The second conjunct is `pending.length` and deliberately NOT {@link hasUnsyncedChanges},
1131
- * which also counts tracker transforms. A collection {@link createOrOpen} just INVENTED holds
1132
- * its staged header/root in the tracker with NO pending action naming them, and
1133
- * {@link replayActions} resets the tracker before re-staging — so counting transforms here
1134
- * would drop those blocks and leave a brand-new collection unreadable (the same hazard
1135
- * {@link snapshotPending} documents). Nothing to re-stage means nothing to replay.
1136
- *
1137
- * NOTE: this makes a refresh that adopts a newer revision O(pending) rather than free, so a
1138
- * read taken mid-transaction while a rival keeps committing re-stages every action staged so
1139
- * far, on each such read. Measured as no change to the storage-op budgets
1140
- * (`index-backfill-cost.spec.ts`, `cold-apply-cost.spec.ts`), because the replay's reads are
1141
- * served from `sourceCache`. If a long transaction's reads ever show up as slow, narrow the
1142
- * replay to the actions whose reads the adopted entries actually invalidated — NOT back to
1143
- * conflicts alone, which is the defect above. */
1144
- private mustReplay(anyConflicts: boolean, priorContext: ActionContext | undefined): boolean {
1145
- if (anyConflicts) {
1146
- return true;
1147
- }
1148
- const adoptedRev = this.source.actionContext?.rev;
1149
- if (adoptedRev !== undefined && adoptedRev !== priorContext?.rev && this.pending.length > 0) {
1150
- return true;
1151
- }
1152
- // Third reason: a staged block's base has MOVED — the read cache no longer describes it at
1153
- // the revision its operations were computed against (see `Tracker.movedBases`). A refresh
1154
- // that adopted nothing and found no conflict can still be standing over such a block: a
1155
- // concurrent read served content the cache then kept at a newer revision, say. Re-staging
1156
- // is the only repair; declaring the pinned revision on the pend would merely get it refused.
1157
- return this.pending.length > 0 && this.logMovedBases(this.tracker.movedBases()) > 0;
1158
- }
1159
-
1160
- /** Re-judge every staged update's pinned base against the read cache and re-stage the pending
1161
- * queue if any has moved. Runs immediately before each pend attempt — {@link syncAttempts} and
1162
- * the coordinator's commit span both call it — because that is the last moment before the
1163
- * base the pend declares is put on the wire, and nothing between a refused attempt and the
1164
- * re-pend reads the block again (the refresh may have moved nothing and so replayed nothing).
1165
- *
1166
- * Two steps. First, one read for each pinned block the cache does NOT retain — content handed
1167
- * through unkept (a below-floor answer, which the cache re-asks for on every read anyway): the
1168
- * pin names the revision last served, storage may have caught up since without the log moving,
1169
- * and only a read can tell. An evicted-but-kept base needs no read: its content met every floor
1170
- * the handle knows, so storage cannot have moved it without a log entry the next refresh will
1171
- * walk (and the pend would be refused as stale on revision alone). Then, one cache probe per
1172
- * pinned block ({@link Tracker.movedBases}) and a replay if any moved.
1173
- *
1174
- * Latch-free by contract, like {@link snapshotPending}: the caller holds this instance's latch
1175
- * ({@link replayActions} is always run under it).
1176
- *
1177
- * NOTE: reads are not latched, so a concurrent read can still move a base between this check
1178
- * and the pend. The pend then declares the pin's (old) revision, storage refuses it, and the
1179
- * next attempt's call here re-stages — one wasted round trip, never a wrong base. If that
1180
- * refusal ever shows up in practice, the closure is to hold the pend under the read latch.
1181
- *
1182
- * @returns whether the pending queue was re-staged. */
1183
- async restageIfBasesMoved(): Promise<boolean> {
1184
- for (const id of this.tracker.unretainedBases()) {
1185
- // `navigation` never upgrades the purpose the original read recorded (value-wins), and
1186
- // the revision recorded is the one returned, which the replay below re-records anyway.
1187
- await this.sourceCache.tryGet(id, 'navigation');
1188
- }
1189
- if (this.pending.length === 0 || this.logMovedBases(this.tracker.movedBases()) === 0) {
1190
- return false;
1191
- }
1192
- await this.replayActions();
1193
- return true;
1194
- }
1195
-
1196
- /** Report each moved base, naming the revision the operations were computed against and the
1197
- * one the cache describes now, and return how many there were. */
1198
- private logMovedBases(moved: readonly BlockId[]): number {
1199
- if (log.enabled) {
1200
- for (const blockId of moved) {
1201
- log('collection:restage-moved-base id=%s tag=%s block=%s pinnedRev=%s currentRev=%s',
1202
- this.id, this.instanceTag, blockId,
1203
- this.tracker.stagedBaseRevs([blockId])[blockId] ?? 'none',
1204
- this.sourceCache.getCachedRevision(blockId) ?? 'none');
1205
- }
1206
- }
1207
- return moved.length;
1208
- }
1209
-
1210
- /** Capture the current staged state — tracker transforms plus the pending
1211
- * action queue — so it can be restored later via {@link restorePending}.
1212
- *
1213
- * Use to bracket a unit of staged DML that may need to be rolled back. Unlike
1214
- * a blanket "reset to empty", restoring this snapshot preserves any structural
1215
- * baseline that predates the staged DML — most importantly a brand-new
1216
- * collection's header/root blocks, which live in the tracker (uncommitted)
1217
- * until the first sync. Resetting such a collection to empty would leave it
1218
- * unreadable; restoring the snapshot returns it to its prior (readable) state.
1219
- *
1220
- * The returned snapshot is deep-cloned and independent of subsequent mutations.
1221
- * Synchronous and latch-free: intended to bracket transaction-scoped staging,
1222
- * when no concurrent act/sync is in flight. */
1223
- snapshotPending(): CollectionSnapshot<TAction> {
1224
- return {
1225
- transforms: copyTransforms(this.tracker.transforms),
1226
- pins: this.tracker.pins.copy(),
1227
- pending: [...this.pending],
1228
- context: structuredClone(this.source.actionContext),
1229
- };
1230
- }
1231
-
1232
- /** Restore the staged state captured by {@link snapshotPending}, discarding any
1233
- * mutations staged since. Reads through the collection then observe exactly the
1234
- * snapshot state again; storage is untouched because nothing was ever synced.
1235
- *
1236
- * A snapshot is only restorable VERBATIM onto the committed boundary it was captured
1237
- * on. If this collection has since ADOPTED a newer committed revision — a rival's
1238
- * commit folded in by a refresh while the snapshot's transaction was in flight, e.g.
1239
- * the conflict replay that refused a guarded insert (TreeKeyTakenError) — the
1240
- * snapshot's transforms describe block state at the OLD boundary, and reinstalling
1241
- * them would shadow committed blocks with stale structure. The observed case: an
1242
- * INVENTED collection's pre-commit header/root transforms restored over the rival's
1243
- * now-committed collection make every later read descend an empty tree, silently
1244
- * hiding the committed rows. When the snapshot's pending queue is empty (the
1245
- * transaction-rollback shape: the capture predates the transaction's first stage),
1246
- * the correct restore target IS the committed state — reset the tracker empty and
1247
- * let reads flow through to the adopted revision.
1248
- *
1249
- * NOTE: a snapshot that carries PENDING actions across a moved boundary (a
1250
- * mid-transaction savepoint captured before a mid-transaction refresh adopted a
1251
- * rival's commit) still restores verbatim below — rebasing it would require an async
1252
- * replay this synchronous method cannot run. That shape predates this guard and
1253
- * keeps its old behaviour; if it is ever observed producing stale reads, the rebase
1254
- * belongs in an async caller that can replay the pending queue (see replayActions).
1255
- * The WRITE side of that shape is closed here: the snapshot's base pins are restored with
1256
- * its transforms, so the restored operations name the bases they were computed on, and the
1257
- * pre-pend re-validation (restageIfBasesMoved) finds those bases moved and re-stages. */
1258
- restorePending(snapshot: CollectionSnapshot<TAction>): void {
1259
- const capturedRev = snapshot.context?.rev;
1260
- const currentRev = this.source.actionContext?.rev;
1261
- const boundaryMoved = currentRev !== undefined && (capturedRev === undefined || currentRev > capturedRev);
1262
- if (boundaryMoved && snapshot.pending.length === 0) {
1263
- this.tracker.reset();
1264
- this.pending = [];
1265
- return;
1266
- }
1267
- this.tracker.reset(copyTransforms(snapshot.transforms));
1268
- if (snapshot.pins) this.tracker.pins.replaceWith(snapshot.pins);
1269
- this.pending = [...snapshot.pending];
1270
- }
1271
-
1272
- /** A read-only {@link Tracker} pinned to this collection's committed state AS OF the
1273
- * moment of this call, seeded with a (deep-copied) set of pre-transaction transforms.
1274
- * Reads through it observe exactly that revision plus exactly those transforms — NOT
1275
- * the mutations staged into this collection's live tracker afterward, and NOT commits
1276
- * that fold into (or clear) the live read cache while the view is being walked. Used
1277
- * to build a committed read view (see {@link Tree.readView}) that a scan can trust
1278
- * from first row to last.
1279
- *
1280
- * The pinning has three legs, built in ONE synchronous block so they all describe the
1281
- * same instant (do not introduce an await between them):
1282
- * - a private {@link TransactorSource} whose action context is a deep copy FROZEN at
1283
- * view-creation time, so a block first read after a later commit still materializes
1284
- * at the pinned revision (the transactor honours `context.rev` on get);
1285
- * - a private {@link CacheSource} nothing else references, so the live collection's
1286
- * `transformCache`/`clear` cannot reach it;
1287
- * - that private cache is seeded from the shared cache's current entries, so the
1288
- * common committed read (deferred CHECK over a warm cache) stays warm instead of
1289
- * refetching every block over the network.
1290
- *
1291
- * By default the view records NO read dependencies — it is not part of any
1292
- * transaction's conflict set (see {@link ReadViewOptions.recordReads}).
1293
- *
1294
- * NOTE: each view holds up to the cache LRU budget (128) of cloned blocks plus
1295
- * whatever it faults in. Views are per-scan and dropped when the scan ends; a very
1296
- * long-lived committed scan pins that much memory.
1297
- *
1298
- * NOTE: an INVENTED collection (createOrOpen found no header, so `actionContext` is
1299
- * undefined) pins to no revision — its private source asks the transactor for the
1300
- * latest. Harmless today because such a collection's blocks all live in the tracker
1301
- * transforms, so a view never reaches storage; if invented collections ever gain
1302
- * committed blocks not covered by their transforms, this view would follow storage
1303
- * forward instead of staying pinned.
1304
- *
1305
- * When {@link ReadViewOptions.pinContext} is supplied, the view pins to THAT
1306
- * boundary instead of the current context, and cache entries committed at a newer
1307
- * revision are dropped from the seed (they would otherwise be served blindly — the
1308
- * cache never checks a hit against the requested context). Dropped entries are
1309
- * refetched from the transactor, which resolves the highest committed revision at
1310
- * or below the pin. This is what lets a snapshot captured BEFORE a commit still
1311
- * yield a coherent pre-commit view AFTER that commit folded into the shared cache
1312
- * (a mid-sweep multi-tree commit being the motivating case). */
1313
- createReadTracker(transforms: Transforms, options?: ReadViewOptions): Tracker<IBlock> {
1314
- const collector = options?.recordReads ? this.source.getCollector() : undefined;
1315
- const pinContext = options?.pinContext ?? this.source.actionContext;
1316
- const pinRev = options?.pinContext?.rev;
1317
- let seed = this.sourceCache.snapshotEntries();
1318
- if (pinRev !== undefined) {
1319
- // NOTE: entries whose revision the cache never learned read as 0 and pass this
1320
- // filter. Committed blocks always carry a real revision (the transactor reports
1321
- // it on load; transformCache stamps it on fold), so a rev-0 entry newer than the
1322
- // pin does not occur on the paths that reach here.
1323
- seed = seed.filter(([, , revision]) => revision <= pinRev);
1324
- }
1325
- // The view shares this handle's floors, so a block the last refresh saw change is not kept
1326
- // too old by the view either. A view pinned below a floor is untouched by it.
1327
- const pinnedSource = new TransactorSource<IBlock>(
1328
- this.id, this.transactor, structuredClone(pinContext), collector, this.floors);
1329
- const pinnedCache = new CacheSource<IBlock>(
1330
- pinnedSource, undefined, collector, seed);
1331
- return new Tracker(pinnedCache, copyTransforms(transforms));
1332
- }
1333
-
1334
- /** The staged (not-yet-synced) actions queued by {@link act}.
1335
- *
1336
- * Exposed so a {@link TransactionCoordinator} can append them to the log at
1337
- * commit time — mirroring what {@link sync} does internally — when the actions
1338
- * were staged directly into this collection (e.g. through a Tree's stage())
1339
- * rather than applied via the coordinator's own action path. */
1340
- getPendingActions(): Action<TAction>[] {
1341
- return this.pending;
1342
- }
1343
-
1344
- /** Drop the staged actions after they have been committed through a
1345
- * coordinator. Counterpart to {@link getPendingActions}; {@link sync} clears
1346
- * its own pending inline, so this is only needed when commit was orchestrated
1347
- * externally. */
1348
- clearPendingActions(): void {
1349
- this.pending = [];
1350
- }
1351
-
1352
- /** Whether {@link sync} has anything to push: staged actions, or tracker transforms
1353
- * that were never committed (an INVENTED collection's header/root blocks live there
1354
- * until its first sync, with no pending action to name them).
1355
- *
1356
- * This is the exact predicate {@link syncInternal} loops on, so `false` means a sync
1357
- * would commit nothing. Exposed so a caller that would otherwise flush
1358
- * unconditionally can skip the round trip — note that {@link Tree.sync} routes
1359
- * through {@link updateAndSync}, so an unnecessary flush still pays a full,
1360
- * cache-bypassing {@link update} before discovering it has nothing to do. */
1361
- hasUnsyncedChanges(): boolean {
1362
- return this.pending.length > 0 || !isTransformsEmpty(this.tracker.transforms);
1363
- }
1364
-
1365
- /** The committed revision this collection currently READS at, or `undefined` for an
1366
- * INVENTED collection that has never adopted a committed revision
1367
- * ({@link createOrOpen} found no header and staged a fresh empty one).
1368
- *
1369
- * Not the revision a pending write will land at: {@link getNextRev} is this plus one
1370
- * (`undefined` counting as 0), so a diagnostic that prints this value BEFORE a commit
1371
- * is naming the revision the commit will supersede, not the one it produces.
1372
- *
1373
- * DIAGNOSTIC ONLY — do not branch on this. Every block this collection reads is
1374
- * materialized at this revision ({@link TransactorSource.tryGet} passes it as the
1375
- * read context), and the revision advances ONLY through an explicit call on THIS
1376
- * instance — {@link update} or {@link sync} on the single-node path, or
1377
- * {@link recordCommitted} when a {@link TransactionCoordinator} commits this
1378
- * collection (the session/consensus path, where no `update()` is involved at all).
1379
- * Nothing moves it passively — not time, not another collection's
1380
- * commit, not a peer's notification. So a collection sitting here at a lagging
1381
- * revision silently serves an old root with no error, and two collections in one
1382
- * process can be at different revisions at the same instant. That gap is invisible
1383
- * from outside the class without this accessor, which is the whole reason it
1384
- * exists: `docs/debugging.md` (§ "Which revision did a read descend?") explains
1385
- * how an operator reads the difference.
1386
- *
1387
- * The one exception is `undefined` itself, which is a STATE rather than a revision:
1388
- * "this instance invented the collection and has never adopted a committed revision".
1389
- * For the same reason (only this instance moves it), it stays true until this instance
1390
- * updates, syncs or records a commit, so a caller holding a freshly opened instance may
1391
- * branch on it — the Quereus adapter does, to leave an invented, never-written index tree
1392
- * unflushed exactly as an unwritten table tree is left. Never branch on the NUMBER. */
1393
- committedRevision(): number | undefined {
1394
- return this.source.actionContext?.rev;
1395
- }
1396
-
1397
- /** The id of the action that PRODUCED the revision {@link committedRevision} reports —
1398
- * this collection's lineage marker at that revision — or `undefined` when the action
1399
- * context holds no entry at the current revision.
1400
- *
1401
- * `undefined` is legitimate, not an error: an INVENTED collection has no context at
1402
- * all, and otherwise {@link actionIdAt} resolves nothing at the current revision for
1403
- * the reasons listed there. A caller printing this must therefore carry a placeholder
1404
- * rather than invent an id. The contexts this class writes itself
1405
- * ({@link recordCommitted}, the inline bump in `syncInternal`,
1406
- * {@link bootstrapContext}) always do hold one.
1407
- *
1408
- * DIAGNOSTIC ONLY — do not branch on this. Its value is the one thing about a revision
1409
- * that IS comparable across collections and across nodes: a revision number is
1410
- * per-collection and says nothing on its own, so two nodes reporting the same
1411
- * collection id at the same revision are indistinguishable between "one collection,
1412
- * one node lagging" and "two separately-built collections each counting from 1". Equal
1413
- * action ids mean one lineage; different action ids at the same revision mean two.
1414
- * `docs/debugging.md` (§ "Which revision did a read descend?") spells out how an
1415
- * operator reads the pair. */
1416
- committedActionId(): ActionId | undefined {
1417
- const context = this.source.actionContext;
1418
- return context === undefined ? undefined : actionIdAt(context, context.rev);
1419
- }
1420
-
1421
- /** Fold a just-committed set of transforms into this collection's read cache
1422
- * so subsequent reads (and stages) through THIS instance observe the committed
1423
- * state, mirroring what {@link sync} does inline after a successful transact.
1424
- *
1425
- * Needed when commit was orchestrated externally (a coordinator): the tracker
1426
- * is reset to empty, but the cache still holds the pre-commit blocks. Without
1427
- * this, a collection that already had committed state (e.g. a pre-synced index
1428
- * tree, or any collection on its second commit) keeps serving the stale prior
1429
- * revision because {@link update} sees its rev is already current and refetches
1430
- * nothing. Call BEFORE resetting the tracker (the transforms are read live).
1431
- *
1432
- * @param revision - the committed revision these transforms land at (from
1433
- * {@link recordCommitted}), so cached read-dependency revisions advance to it. */
1434
- applyCommittedToCache(transforms: Transforms, revision: number): void {
1435
- this.sourceCache.transformCache(transforms, revision);
1436
- }
1437
-
1438
- /** Next revision this collection would commit at (current committed rev + 1). */
1439
- getNextRev(): number {
1440
- return (this.source.actionContext?.rev ?? 0) + 1;
1441
- }
1442
-
1443
- /** Record a just-committed action: append its ActionRev to the committed list
1444
- * and advance the revision. Returns the new revision. Mirrors the inline bump
1445
- * in {@link syncInternal} — which needs no such rev check because it computes and
1446
- * uses its `newRev` inside one latched span.
1447
- *
1448
- * @param rev - the revision this action was PENDED at, captured once (at the log
1449
- * append in `TransactionCoordinator.applyActionsToCollection`) and threaded through
1450
- * the pend/commit round trips. Storage assigned the action THAT number; recording it
1451
- * at any other would fork this instance's revision counter from storage permanently
1452
- * (context adoption is one-way — see {@link advanceContext}). With the coordinator
1453
- * holding this instance's latch across the whole commit span the mismatch cannot
1454
- * happen; the throw is the tripwire for any path that still bypasses the latch. */
1455
- recordCommitted(actionId: ActionId, rev: number): number {
1456
- const expected = this.getNextRev();
1457
- if (rev !== expected) {
1458
- throw new Error(`Collection ${this.id}: action ${actionId} was pended at rev ${rev} ` +
1459
- `but the collection now expects rev ${expected} — the collection was refreshed mid-commit`);
1460
- }
1461
- this.source.actionContext = {
1462
- committed: [...(this.source.actionContext?.committed ?? []), { actionId, rev }],
1463
- rev,
1464
- };
1465
- return rev;
1466
- }
1467
-
1468
- /** Acquire this instance's latch — the same mutex {@link act}, {@link update},
1469
- * {@link sync}, and {@link updateAndSync} serialize behind — returning its release.
1470
- * Exists so a TransactionCoordinator can hold the latch across its WHOLE commit span
1471
- * (log append → pend → commit → local fold), keeping any refresh of this instance from
1472
- * interleaving with a mid-flight commit. `Latches` is non-reentrant: while holding this,
1473
- * the holder must not call any of those latched methods on this instance. The caller
1474
- * MUST call the release exactly once, in a `finally`.
1475
- *
1476
- * NOTE: the commit paths are the ONLY callers — {@link act}, {@link update} and {@link sync}
1477
- * take the same mutex through `Latches.acquire` directly, never through here. The acquisition
1478
- * -order cases in `test/coordinator-latch-span.spec.ts` rely on that: they shadow this method
1479
- * to record the span's order, so routing a refresh path through it would silently fold
1480
- * refreshes into those recordings and weaken the assertions rather than fail them. Route a new
1481
- * refresh caller here only alongside a filter in that spy. */
1482
- acquireLatch(): Promise<() => void> {
1483
- return Latches.acquire(this.latchId);
1484
- }
1485
-
1486
- /** Bracket a write attempt cycle on this instance under `actionId`, so a refresh taken between
1487
- * a failed attempt and its retry recognises that action's own already-durable log entry (see
1488
- * {@link inFlightActionId} for why, and for the lifetime this must span).
1489
- *
1490
- * The returned disposer clears the mark and MUST be called in a `finally` covering every exit
1491
- * from the retry cycle — return, retry exhaustion, partial commit, hard error, abort. A mark
1492
- * left behind would let a LATER, unrelated refresh consume a foreign entry that happens to
1493
- * carry the same id. The clear is id-guarded, so a disposer whose mark has since been replaced
1494
- * by another attempt is a no-op rather than wiping the newer one; disposers may therefore be
1495
- * called out of order and more than once. Shaped like {@link acquireLatch} deliberately: a
1496
- * disposer is harder to forget than a paired `end…` call.
1497
- *
1498
- * A concurrent READER's refresh landing in that latch-free window is fine, and is reachable
1499
- * today (`OptimysticModule` declares `concurrencyMode = 'reentrant-reads'`, so scans inside a
1500
- * transaction share one instance and only serialize on the latch). Consuming is a property of
1501
- * the (instance, action id) pair, not of who calls: whoever refreshes first drops exactly the
1502
- * durable entry's actions and the write's own later refresh then finds nothing new. The leading
1503
- * slice stays right because {@link actInternal} APPENDS, so anything staged after the entry was
1504
- * written survives it.
1505
- *
1506
- * NOTE: a concurrent WRITER is the hazard. The mark deliberately outlives the latch (see
1507
- * {@link inFlightActionId}), so two writes overlapping on ONE instance can trample each other's:
1508
- * a second write that acquires the latch between the first's failed attempt and its
1509
- * retry-refresh replaces the id, and the first's refresh then reads the SECOND write's id —
1510
- * consuming that write's durable entry and dropping pending actions of its own that never
1511
- * landed. Not reachable today: a coordinator commit and a `sync()` both run on one session call
1512
- * path, which is the same assumption the participant selection in
1513
- * `TransactionCoordinator.commitOnce` and its `rollback` already rest on. If a second writer is
1514
- * ever allowed to drive the SAME instance concurrently, this must become a per-attempt token (a
1515
- * mark object compared by identity, refusing to replace a live one) rather than a bare id. */
1516
- beginInFlightAction(actionId: ActionId): () => void {
1517
- if (this.inFlightActionId !== actionId) {
1518
- // A different action's failed attempt cannot finish this one. Re-marking the SAME id (a
1519
- // coordinator re-marks on every attempt) keeps what the previous attempt retained — that
1520
- // is the attempt the next refresh may need.
1521
- this.inFlightAttempt = undefined;
1522
- }
1523
- this.inFlightActionId = actionId;
1524
- return () => {
1525
- if (this.inFlightActionId === actionId) {
1526
- this.inFlightActionId = undefined;
1527
- this.inFlightAttempt = undefined;
1528
- }
1529
- };
1530
- }
1531
-
1532
- /** Keep what a FAILED attempt under `actionId` sent, so that if the refresh before its retry
1533
- * finds that attempt's own log entry it can finish the action rather than assume it finished
1534
- * (see {@link inFlightAttempt} and {@link completeOwnEntry}). Every write path that marks an
1535
- * action in flight must call this for each attempt that fails, BEFORE the refresh that follows —
1536
- * a path that does not still never reports a half-landed write as saved (the refresh then
1537
- * refuses with a `transforms-not-held` {@link TornActionError}), but it cannot recover from one.
1538
- *
1539
- * A no-op unless `actionId` is the action currently marked, so a late call from an attempt whose
1540
- * cycle has already ended cannot plant transforms under somebody else's mark. Latch-free and
1541
- * synchronous, like the mark itself: `TransactionCoordinator` calls it from inside its commit
1542
- * span, where it already holds this instance's latch. */
1543
- retainInFlightAttempt(actionId: ActionId, attempt: InFlightAttempt): void {
1544
- if (this.inFlightActionId === actionId) {
1545
- this.inFlightAttempt = attempt;
1546
- }
1547
- }
1548
-
1549
- /** Push our pending actions to the transactor.
1550
- *
1551
- * @returns who holds what this sync committed, or `undefined` when NOTHING WAS WRITTEN — a sync
1552
- * with no staged changes does no pend and no commit, so there is no durability to report and none
1553
- * is fabricated. That is the one case a caller must handle; every other outcome either returns a
1554
- * class or throws ({@link SyncRetryExhaustedError} for a write that never landed). Present a change
1555
- * as saved only via `isFullyDurable`, never by comparing `quorum`. */
1556
- async sync(options?: SyncOptions): Promise<WriteDurability | undefined> {
1557
- const release = await Latches.acquire(this.latchId);
1558
- try {
1559
- return await this.syncInternal(options);
1560
- } finally {
1561
- release();
1562
- }
1563
- }
1564
-
1565
- /** Mints the one action id this sync reuses across all of its retry attempts, and owns it for
1566
- * the WHOLE cycle — including the inter-attempt refresh, which is the only thing that reads it
1567
- * (see {@link inFlightActionId}). `sync()`/`updateAndSync()` hold the collection latch across
1568
- * all of this, so the mark's lifetime is contained inside the latched span here; the disposer
1569
- * runs on every exit, including a throw out of retry exhaustion or an abort. */
1570
- private async syncInternal(options?: SyncOptions): Promise<WriteDurability | undefined> {
1571
- const bytes = randomBytes(16);
1572
- const actionId = uint8ArrayToString(bytes, 'base64url');
1573
-
1574
- const endInFlight = this.beginInFlightAction(actionId);
1575
- try {
1576
- return await this.syncAttempts(actionId, options);
1577
- } finally {
1578
- endInFlight();
1579
- }
1580
- }
1581
-
1582
- /** The retry loop behind {@link syncInternal}, run with `actionId` already marked in flight.
1583
- *
1584
- * @returns the durability of what this sync committed, or `undefined` when the loop never ran a
1585
- * commit (nothing staged). See {@link sync}. */
1586
- private async syncAttempts(actionId: ActionId, options?: SyncOptions): Promise<WriteDurability | undefined> {
1587
- const maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
1588
- const baseBackoffMs = options?.baseBackoffMs ?? PendingRetryDelayMs;
1589
- const maxBackoffMs = options?.maxBackoffMs ?? DefaultMaxBackoffMs;
1590
- const maxStalledAttempts = options?.maxStalledAttempts ?? DefaultMaxStalledAttempts;
1591
- const deadlineMs = options?.deadlineMs;
1592
- const signal = options?.signal;
1593
- const startedAt = Date.now();
1594
-
1595
- // Count of consecutive stale failures that made no forward progress. Reset to 0 on every
1596
- // successful transact, so the cap bounds only a persistently-failing sync — a legitimate
1597
- // large multi-batch sync (which iterates many times committing progress) never trips it.
1598
- let consecutiveFailures = 0;
1599
- let lastReason: string | undefined;
1600
- // Highest confirmed revision any responder has reported holding, accumulated with the
1601
- // codebase's single rule for picking among several candidates. Reported on the error, and
1602
- // the evidence the stall check below reasons from.
1603
- let lastStaleAt: { blockId: BlockId; rev: number } | undefined;
1604
- // Whether the failure most recently handled carried its OWN staleAt. A strike needs the
1605
- // responder to have re-confirmed the number this round, not merely an older observation
1606
- // left standing in `lastStaleAt`.
1607
- let lastFailureConfirmedStaleAt = false;
1608
- // Consecutive refreshes that moved `getNextRev()` nowhere at all while a confirmed revision
1609
- // stood at or above it.
1610
- let consecutiveStalls = 0;
1611
- // The revision the PREVIOUS iteration would have requested, so the stall check can tell a
1612
- // refresh that moved nowhere from one that is still climbing toward the confirmed number.
1613
- let previousRequestedRev: number | undefined;
1614
- // Who holds what this sync has committed so far. Stays `undefined` while nothing has been
1615
- // committed, which is also the answer when the loop never runs at all — a sync with nothing
1616
- // staged writes nothing, and there is no durability to fabricate for it. Failed attempts never
1617
- // contribute: a refused attempt left nothing behind, so the answer is the committing attempt's.
1618
- let durability: WriteDurability | undefined;
1619
-
1620
- while (this.hasUnsyncedChanges()) {
1621
- if (signal?.aborted) {
1622
- throw makeAbortError(signal);
1623
- }
1624
- // Progress-agnostic ceiling: give up if the wall-clock deadline passed. Deliberately
1625
- // ahead of the stall check: the deadline is the documented outer bound, so a sync that
1626
- // is both past it and stalled reports the deadline.
1627
- if (deadlineMs !== undefined && Date.now() - startedAt >= deadlineMs) {
1628
- throw new SyncRetryExhaustedError(this.id, consecutiveFailures, lastReason ?? 'deadline exceeded', lastStaleAt);
1629
- }
1630
-
1631
- // Can the attempt about to run possibly differ from the one that just failed? Only when
1632
- // BOTH of these hold is the answer provably no:
1633
- //
1634
- // - The revision it would request is at or below one a responder CONFIRMED is taken. A
1635
- // producer sets `staleAt` only after reading that revision as durably held by someone
1636
- // else out of its own storage, revisions are one per-collection counter that every
1637
- // commit touches, and a confirmed revision never becomes un-taken (invalidation takes
1638
- // a NEW slot). So the request is already lost before it is sent.
1639
- // - The refresh in between moved the collection nowhere. `advanceContext` never lowers
1640
- // the held revision, so "nowhere" is exactly `requestedRev` unchanged since the last
1641
- // iteration. A refresh that moved forward WITHOUT clearing the confirmed number is a
1642
- // collection still climbing (it read a replica that lags the holder — the same
1643
- // partial catch-up `reportShortfall` exists to report), and its next attempt is a
1644
- // genuinely different request that may yet win.
1645
- //
1646
- // This is NOT a second answer to "is this failure retryable?" — `isConflictFailure`
1647
- // remains the sole rule for that, untouched. It only ever stops a loop that rule had
1648
- // already decided to continue.
1649
- const requestedRev = this.getNextRev();
1650
- const refreshMoved = previousRequestedRev !== undefined && requestedRev > previousRequestedRev;
1651
- previousRequestedRev = requestedRev;
1652
- if (lastStaleAt !== undefined) {
1653
- if (requestedRev > lastStaleAt.rev || refreshMoved) {
1654
- // Either the refresh adopted a revision above the confirmed one — ordinary
1655
- // contention, where the rival's commit is exactly what we just read — or it made
1656
- // partial forward progress. Both mean the next attempt differs. Not a stall.
1657
- consecutiveStalls = 0;
1658
- } else if (lastFailureConfirmedStaleAt) {
1659
- consecutiveStalls++;
1660
- if (log.enabled) {
1661
- log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d',
1662
- this.id, this.instanceTag, this.source.actionContext?.rev ?? 'none', requestedRev,
1663
- lastStaleAt.blockId, lastStaleAt.rev, consecutiveStalls, maxStalledAttempts);
1664
- }
1665
- // Two strikes, not one: a legitimate loser can transiently read a view that has
1666
- // not yet caught up with the rival's commit, which looks identical for one round.
1667
- if (consecutiveStalls >= maxStalledAttempts) {
1668
- throw new SyncRevisionStalledError(this.id, consecutiveFailures, lastStaleAt,
1669
- requestedRev, this.source.actionContext?.rev, lastReason);
1670
- }
1671
- }
1672
- // Else: the responder told us nothing new this round. No strike, and no reset either
1673
- // — the budget stays bounded by maxAttempts.
1674
- }
1675
-
1676
- // A pending action is never pended over a base that has moved under it — before EVERY
1677
- // attempt, first and retries alike (see restageIfBasesMoved for why the retry needs it).
1678
- await this.restageIfBasesMoved();
1679
-
1680
- // Snapshot the pending actions so that any new actions aren't assumed to be part of this action
1681
- const pending = [...this.pending];
1682
-
1683
- // Create a snapshot tracker for the action, so that we can ditch the log changes if we have to retry the action.
1684
- // It SHARES the live tracker's pin store: the snapshot inherits transforms by copy and never
1685
- // stages the data-block updates itself, so without the shared pins the digest pass below
1686
- // could only describe bases still resident in the read cache.
1687
- // NOTE: an attempt that ends in a stale failure is abandoned WITHOUT reset(), so the log
1688
- // tail/header bases its append pinned stay in the shared store until the live tracker's
1689
- // next reset. Bounded and harmless — a few log blocks per attempt, capped by maxAttempts,
1690
- // each re-validated against the source generation before any use — but if this loop ever
1691
- // grows a per-attempt footprint beyond the log blocks, reset the abandoned tracker on the
1692
- // stale-failure branch instead of letting the residue ride to the end of the sync.
1693
- const snapshot = copyTransforms(this.tracker.transforms);
1694
- const tracker = new Tracker(this.sourceCache, snapshot, this.tracker.pins);
1695
-
1696
- // Add the action to the log (in local tracking space)
1697
- const collectionLog = await Log.open<Action<TAction>>(tracker, this.id);
1698
- if (!collectionLog) {
1699
- throw new Error(`Log not found for collection ${this.id}`);
1700
- }
1701
- const newRev = this.getNextRev();
1702
- const addResult = await collectionLog.addActions(pending, actionId, newRev, () => tracker.transformedBlockIds());
1703
-
1704
- // Declare what each touched block will contain once committed, computed from this snapshot
1705
- // tracker (which layers over `this.sourceCache`, so the peek/getCachedRevision probes are
1706
- // live). Purely local — an id whose base is not already cached is simply omitted and falls
1707
- // back to corroboration on the member side. Computed AFTER the log append so the log tail
1708
- // and header transforms this attempt just staged are digested too.
1709
- // NOTE: recomputed from scratch on every retry attempt (the snapshot tracker is rebuilt each
1710
- // iteration), so a sync that loses N races pays N full hashing passes over its touched
1711
- // blocks. Unmeasured and cheap relative to the round trips it is retrying; if a
1712
- // high-contention sync ever shows digest hashing in a profile, memoize per (id, staged ops).
1713
- const blockDigests = await computeBlockContentDigests(tracker, tracker.transformedBlockIds());
1714
- // The base each update-only block's operations were computed against, for the pend (see
1715
- // `PendRequest.baseRevs`): the tracker's pinned revisions, which `restageIfBasesMoved` at the
1716
- // top of this iteration has just re-judged, so a moved base was re-staged before it is named.
1717
- const baseRevs = tracker.stagedBaseRevs(tracker.transformedBlockIds());
1718
-
1719
- // Commit the action to the transactor. Carry the aged retry priority derived from the
1720
- // consecutive-failure count so a sync that keeps losing concurrent races out-ranks fresh
1721
- // (priority-0) rivals in the cluster's resolveRace (fairness-only; capped at MaxPriority).
1722
- // First attempt has consecutiveFailures == 0, so priority 0 — the common pend is unchanged.
1723
- const attempt = await this.source.transact(tracker.transforms, actionId, newRev, this.id, addResult.tailPath.block.header.id, clampPriority(consecutiveFailures), blockDigests, baseRevs);
1724
- if (!attempt.success) {
1725
- consecutiveFailures++;
1726
- lastReason = attempt.reason ?? lastReason;
1727
- // Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
1728
- // responder reporting a LOWER number understates the binding constraint. Same rule the
1729
- // producers and the transactor's aggregation already use.
1730
- lastStaleAt = highestStaleAt([lastStaleAt, attempt.staleAt]);
1731
- lastFailureConfirmedStaleAt = attempt.staleAt !== undefined;
1732
- // Give up once the consecutive no-progress budget is exhausted, so a transactor that
1733
- // persistently rejects the sync can no longer hold the collection latch forever.
1734
- // NOTE: this also bounds the legitimate `pending`-wait case (retrying the same action
1735
- // while another commit is in flight), which used to retry indefinitely. Default 10
1736
- // attempts ≈ 21s of exponential backoff. If a high-contention workload legitimately
1737
- // needs to wait longer for a pending commit to clear, raise maxAttempts for that caller.
1738
- // NOTE: no refresh follows the LAST budgeted attempt, so if that attempt's log tail
1739
- // landed nobody finds out: the caller gets plain exhaustion over a log that holds an
1740
- // entry for this write. The write is still never reported saved, which is the rule; what
1741
- // is lost is the more specific name (TornActionError). The leftover entry itself is
1742
- // tracked in tickets/backlog/bug-a-refused-write-can-leave-its-log-entry-behind.
1743
- if (consecutiveFailures >= maxAttempts) {
1744
- throw new SyncRetryExhaustedError(this.id, consecutiveFailures, lastReason, lastStaleAt);
1745
- }
1746
- // Keep exactly what this attempt sent. A refused commit is not proof nothing landed:
1747
- // the log tail is committed first and can be stored while the answer is still a
1748
- // failure, and `transact` has just cancelled every block that did not land. If the
1749
- // refresh below finds this attempt's own entry, these are the transforms that finish
1750
- // the action (see completeOwnEntry). The snapshot tracker is abandoned after this
1751
- // iteration, so its transforms are handed over as-is, uncopied.
1752
- this.retainInFlightAttempt(actionId, {
1753
- rev: newRev,
1754
- transforms: tracker.transforms,
1755
- tailId: addResult.tailPath.block.header.id,
1756
- ...(blockDigests === undefined ? {} : { blockDigests }),
1757
- ...baseRevsField(baseRevs),
1758
- });
1759
- // Refresh, and keep refreshing while it reports that this write's own half-landed
1760
- // action could not be finished YET. It must not fall through to a new attempt in that
1761
- // state: a new attempt rebuilds the log entry (fresh timestamp) and would send a second
1762
- // version of a log tail that is already stored under this action and revision. Each
1763
- // round is a failure against the same no-progress budget as a refused attempt.
1764
- for (;;) {
1765
- // Back off before every retry (any stale failure — reason/missing/pending), growing
1766
- // exponentially from the base delay up to the cap, with proportional random jitter so a
1767
- // herd of clients that lost the same race does not re-collide on the next tick (see
1768
- // utility/backoff.ts). The abortable sleep lets an aborted sync reject promptly instead
1769
- // of finishing the sleep.
1770
- // NOTE: the `missing`/`reason` conflict paths now pay this backoff too (they previously
1771
- // retried with zero delay); that is what stops the persistent-`reason` hot spin. If a
1772
- // high-contention workload ever shows this base delay as recovery latency, lower
1773
- // baseBackoffMs for that caller rather than reintroducing the zero-delay retry.
1774
- const delay = jitteredBackoffMs(consecutiveFailures - 1, { baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
1775
- await abortableDelay(delay, signal);
1776
- try {
1777
- // Fetch latest state - updateInternal() will call replayActions() if there are conflicts.
1778
- // This sync's actionId is marked in flight for the whole cycle (see syncInternal), so
1779
- // the refresh recognizes a log entry written by THIS action (its log tail landed but
1780
- // the commit answered failure), FINISHES that action from the attempt retained
1781
- // above, and only then consumes the entry rather than replaying it into a duplicate.
1782
- // The round that would spend the last of the budget settles a half-landed write
1783
- // instead of asking for another round, so the error that escapes below says
1784
- // whether the write can still land. (A deadline cannot be foreseen the same
1785
- // way; a write given up on it escapes unsettled, as `final: false`.)
1786
- const report: RefreshReport = {};
1787
- await this.updateInternal(report, consecutiveFailures + 1 >= maxAttempts);
1788
- const completed = report.ownEntryFinished?.durability;
1789
- if (completed !== undefined) {
1790
- // The refresh made this sync's write durable: that is a commit, and it is
1791
- // reported and counted exactly like one made by an attempt (see the success
1792
- // branch below for why batches fold to the weakest).
1793
- durability = durability === undefined ? completed : mergeDurability([durability, completed]);
1794
- consecutiveFailures = 0;
1795
- lastReason = undefined;
1796
- lastStaleAt = undefined;
1797
- lastFailureConfirmedStaleAt = false;
1798
- consecutiveStalls = 0;
1799
- }
1800
- break;
1801
- } catch (err) {
1802
- if (!(err instanceof TornActionError) || err.reason !== 'completion-refused') {
1803
- throw err;
1804
- }
1805
- // Refused for a cause that can clear. The error already names the write as torn,
1806
- // which is the truth if the budget ends here — so it, not a plain exhaustion,
1807
- // is what escapes: the log holds an entry for a write that was not saved.
1808
- consecutiveFailures++;
1809
- lastReason = err.detail;
1810
- if (consecutiveFailures >= maxAttempts
1811
- || (deadlineMs !== undefined && Date.now() - startedAt >= deadlineMs)) {
1812
- throw err;
1813
- }
1814
- }
1815
- }
1816
- } else {
1817
- // This attempt's commit landed, so its durability is the one to report. A sync that
1818
- // commits ONCE — every sync that has a caller today — reports exactly that answer,
1819
- // untouched. A sync whose loop commits more than one batch (the `hasUnsyncedChanges`
1820
- // condition re-entering after a successful commit) folds the batches with
1821
- // `mergeDurability`, whose scalar answer is the WEAKEST of them: one batch that only
1822
- // reached the writer makes the whole sync only-on-the-writer, and reporting the last
1823
- // batch's class instead would show such a write as saved.
1824
- durability = durability === undefined ? attempt.durability : mergeDurability([durability, attempt.durability]);
1825
- // Forward progress: reset the no-progress budget.
1826
- consecutiveFailures = 0;
1827
- lastReason = undefined;
1828
- lastStaleAt = undefined;
1829
- lastFailureConfirmedStaleAt = false;
1830
- consecutiveStalls = 0;
1831
- // Clear the pending actions that were part of this action
1832
- this.pending = this.pending.slice(pending.length);
1833
- // Reset cache and replay any actions that were added during the action
1834
- const transforms = tracker.reset();
1835
- // NOTE: this replay runs BEFORE the cache fold and the context bump below — the
1836
- // inverse of the order `updateInternal` and `TransactionCoordinator.commitOnce`
1837
- // both document as required, where the newly committed state must be visible
1838
- // before anything re-reads. Dormant today: `act()` and `syncInternal` take the
1839
- // same collection latch, so `this.pending` cannot grow during the transact above
1840
- // and the slice always leaves it empty, making this replay a no-op reset. If a
1841
- // path ever stages outside that latch, move the transformCache + actionContext
1842
- // lines above this replay — otherwise the replay re-reads at the superseded
1843
- // revision over a cache that has not yet seen the commit, and re-stages onto a
1844
- // root the commit already replaced.
1845
- await this.replayActions();
1846
- this.sourceCache.transformCache(transforms, newRev);
1847
- this.source.actionContext = this.source.actionContext
1848
- ? { committed: [...this.source.actionContext.committed, { actionId, rev: newRev }], rev: newRev }
1849
- : { committed: [{ actionId, rev: newRev }], rev: newRev };
1850
- }
1851
- }
1852
- return durability;
1853
- }
1854
-
1855
- /** Refresh from the transactor, then push. Same return contract as {@link sync}: the durability of
1856
- * what was committed, or `undefined` when nothing was staged and so nothing was written. */
1857
- async updateAndSync(options?: SyncOptions): Promise<WriteDurability | undefined> {
1858
- const release = await Latches.acquire(this.latchId);
1859
- try {
1860
- await this.updateInternal({});
1861
- return await this.syncInternal(options);
1862
- } finally {
1863
- release();
1864
- }
1865
- }
1866
-
1867
- /** Stage `actions` and flush them as one call that either takes effect or leaves NOTHING behind:
1868
- * when the flush throws, the actions this call staged are unstaged again before the error escapes.
1869
- *
1870
- * For callers that own both halves of a write (`Tree.replace`, `Diary.append`). Staging with
1871
- * {@link act} and flushing separately leaves a failed write's actions staged — deliberate for a
1872
- * caller that means to call {@link sync} again, and a trap for one that does not: the actions
1873
- * ride along, unasked, with its NEXT write, so a change the caller was told had failed shows up
1874
- * later, and a caller that reacted to the failure by submitting it again stores it twice. That
1875
- * is how a write reported torn was seen to "appear one write later" (the pending-record route
1876
- * first suspected was ruled out: with the failed writer's collection discarded, 0 of 17 torn
1877
- * rows ever appeared).
1878
- *
1879
- * One latch hold spans staging and flushing, so no other {@link act} on this instance can land
1880
- * between them and the actions to take back are exactly the ones this call put there.
1881
- *
1882
- * The error is rethrown untouched. Unstaging says nothing about storage: a
1883
- * {@link TornActionError} with `final: false` still means the write may be saved or may yet
1884
- * land, and only the staged copy is gone. */
1885
- async actAndSync(actions: Action<TAction>[], options?: SyncOptions): Promise<WriteDurability | undefined> {
1886
- const release = await Latches.acquire(this.latchId);
1887
- try {
1888
- const stagedBefore = copyTransforms(this.tracker.transforms);
1889
- const revBefore = this.source.actionContext?.rev;
1890
- await this.actInternal(...actions);
1891
- try {
1892
- await this.updateInternal({});
1893
- return await this.syncInternal(options);
1894
- } catch (err) {
1895
- await this.unstage(actions, stagedBefore, revBefore);
1896
- throw err;
1897
- }
1898
- } finally {
1899
- release();
1900
- }
1901
- }
1902
-
1903
- /** Takes `actions` back out of the staged queue and rebuilds the tracker without them (always
1904
- * called under latch). `stagedBefore` / `revBefore` are the tracker's transforms and the held
1905
- * revision from before the actions were staged.
1906
- *
1907
- * While the held revision has not moved, the earlier transforms are reinstated verbatim rather
1908
- * than rebuilt by replay: an INVENTED collection keeps its header and root in the tracker with
1909
- * no staged action naming them, and a replay (which resets the tracker first) would drop them —
1910
- * the hazard {@link mustReplay} and {@link snapshotPending} document. Once the revision has
1911
- * moved those transforms describe blocks at a revision this handle has left, so what remains is
1912
- * re-staged against the adopted one, exactly as a refresh would have done. */
1913
- private async unstage(actions: Action<TAction>[], stagedBefore: Transforms, revBefore: number | undefined): Promise<void> {
1914
- // NOTE: matched by identity, which is what `filterAgainstEntry` promises for a kept action.
1915
- // A `filterConflict` hook that answers a REPLACEMENT instance for one of these actions would
1916
- // leave the replacement staged here. No collection installs such a hook today (`Tree` and
1917
- // `Diary` install none); if one ever does, carry a per-call token on the staged actions and
1918
- // match on that instead.
1919
- this.pending = this.pending.filter(staged => !actions.includes(staged));
1920
- if (this.source.actionContext?.rev === revBefore) {
1921
- this.tracker.reset(stagedBefore);
1922
- } else {
1923
- await this.replayActions();
1924
- }
1925
- }
1926
-
1927
- async *selectLog(forward = true): AsyncIterableIterator<Action<TAction>> {
1928
- const collectionLog = await Log.open<Action<TAction>>(this.tracker, this.id);
1929
- if (!collectionLog) {
1930
- throw new Error(`Log not found for collection ${this.id}`);
1931
- }
1932
- for await (const entry of collectionLog.select(undefined, forward)) {
1933
- if (entry.action) {
1934
- // NOTE: copy-then-reverse to avoid mutating the stored log entry array.
1935
- // Once tsconfig targets ES2023, `entry.action.actions.toReversed()` is cleaner.
1936
- yield* forward ? entry.action.actions : [...entry.action.actions].reverse();
1937
- }
1938
- }
1939
- }
1940
-
1941
- private async replayActions() {
1942
- this.tracker.reset();
1943
- // Replay pending actions against the fresh tracker state (always called under latch)
1944
- for (const action of this.pending) {
1945
- await this.internalTransact(action);
1946
- }
1947
- }
1948
-
1949
- getReadDependencies(): ReadDependency[] {
1950
- return this.source.getReadDependencies();
1951
- }
1952
-
1953
- clearReadDependencies(): void {
1954
- this.source.clearReadDependencies();
1955
- }
1956
-
1957
- /** Called for each local action that may be in conflict with a remote action (always called under latch).
1958
- * @param action - The local action to check
1959
- * @param potential - The remote actions that are potentially in conflict
1960
- * @returns The effective action to keep: the original (unchanged), a replacement
1961
- * instance (applied instead of the original), or undefined to discard it.
1962
- */
1963
- protected doFilterConflict(action: Action<TAction>, potential: Action<TAction>[]): Action<TAction> | undefined {
1964
- return this.filterConflict ? this.filterConflict(action, potential) : action;
1965
- }
1966
-
1967
- /** The two blocks every refresh starts from — the collection header and the log tail block it
1968
- * names — read unpinned ("latest"), in ONE request when `knownTailId` is the tail the header
1969
- * names. Only when the header names a different tail (the known one filled, or none was known)
1970
- * is that tail fetched in a second request; the out-of-date block's answer is dropped rather
1971
- * than kept, because nothing proves it is current at the revision the refresh will pin to.
1972
- *
1973
- * Both answers pass {@link answeredBlock}'s checks as unpinned reads, header first, so a doubted
1974
- * header or tail throws exactly as it would through {@link TransactorSource.tryGet}.
1975
- *
1976
- * NOTE: a batched get fails as a whole when any block in it gets no answer
1977
- * (`NetworkTransactor.get` throws on a missing id), so an unreachable out-of-date tail fails a
1978
- * refresh that would not have needed it. Harmless today: the log has no checkpoints, so a
1979
- * refresh that finds a new tail walks back through the old one anyway. If checkpoints start
1980
- * letting that walk stop short, read the known tail in its own request instead.
1981
- *
1982
- * @returns undefined when the header is authoritatively absent. */
1983
- private static async readLogEnds(transactor: ITransactor, id: CollectionId, knownTailId: BlockId | undefined): Promise<LogEnds | undefined> {
1984
- const results = await transactor.get({ blockIds: knownTailId === undefined ? [id] : [id, knownTailId] });
1985
- const headerEntry = results?.[id];
1986
- if (headerEntry === undefined) {
1987
- return undefined;
1988
- }
1989
- const header = answeredBlock(id, headerEntry, undefined) as CollectionHeaderBlock | undefined;
1990
- if (!header) {
1991
- return undefined;
1992
- }
1993
- const served: LogEnds['served'] = [[id, header, servedRevision(headerEntry)]];
1994
- const tailId = header.tailId;
1995
- if (tailId === undefined) {
1996
- return { header, served };
1997
- }
1998
- const tail = tailId === knownTailId
1999
- ? Collection.checkedLogTail(tailId, results[tailId])
2000
- : await Collection.readLogTail(transactor, tailId);
2001
- if (tail?.block) {
2002
- served.push([tailId, tail.block, servedRevision(tail)]);
2003
- }
2004
- return { header, tail, served };
2005
- }
2006
-
2007
- /** An unpinned read of the log tail block, checked as {@link checkedLogTail} describes. */
2008
- private static async readLogTail(transactor: ITransactor, tailId: BlockId): Promise<GetBlockResult | undefined> {
2009
- return Collection.checkedLogTail(tailId, (await transactor.get({ blockIds: [tailId] }))?.[tailId]);
2010
- }
2011
-
2012
- /** The repo's answer for the log tail, once it has passed {@link answeredBlock}'s unpinned-read
2013
- * checks. The raw entry, not just the block, is what a refresh needs — {@link bootstrapContext}
2014
- * reads `state.latest` off it — which is why the tail is read around {@link TransactorSource}
2015
- * and has to be checked here.
2016
- *
2017
- * Both checks matter at this seam in particular. A tail the repo could not retrieve must not
2018
- * degrade into "no context", which would leave the chain walk unable to see pending non-tail
2019
- * blocks and the collection reading as if they did not exist. And this unpinned tail read is
2020
- * the ONE seam where a lagging collection can learn a newer revision exists — every later data
2021
- * read is pinned to the context seeded from it — so seeding from a tail the repo could not
2022
- * confirm is current would freeze the collection at the stale revision with nothing ever
2023
- * reporting a problem. A tail with no `state.latest` and no flag is a real answer (nothing
2024
- * committed yet). */
2025
- private static checkedLogTail(tailId: BlockId, entry: GetBlockResult | undefined): GetBlockResult | undefined {
2026
- if (entry) {
2027
- answeredBlock(tailId, entry, undefined);
2028
- }
2029
- return entry;
2030
- }
2031
-
2032
- /** Bootstrap ActionContext from the committed tail block's state.
2033
- * Every member applies an action's tail before its other blocks (commit protocol guarantee:
2034
- * `StorageRepo.commit` orders the tail first and stops at the first failure), so wherever
2035
- * committed data of an action exists its tail does too, readable with context=undefined.
2036
- * Its state.latest contains the ActionRev of the most recent
2037
- * committed action — exactly the proof needed for the transactor to serve pending
2038
- * non-tail blocks during chain walks. A tail with no `latest` (or no tail) no-ops.
2039
- *
2040
- * NOTE: this number is adopted on trust, and adoption is one-way (advanceContext never
2041
- * lowers it). A tail that over-claims therefore pins the collection at a revision its
2042
- * own log can never reach, permanently: every later refresh walks the log (the claim's newest
2043
- * entry never matches, so {@link tailShowsNothingNewer} never lets it skip), reads the
2044
- * real (lower) revision, and is refused — so the instance emits
2045
- * `collection:context-not-lowered` forever while `collection:context-short-of-tail`
2046
- * stays silent (the held revision is at or above what the tail claims). No condition
2047
- * that makes a real tail over-claim has been demonstrated; this was seen only through a
2048
- * test double built to lie (see collection.spec.ts, 'a refresh that lands short of the
2049
- * tail it just read'). If an over-claiming tail is ever observed in the field, the fix
2050
- * belongs here — validate the claim against the log before pinning — not in the refresh.
2051
- */
2052
- private static bootstrapContext(source: TransactorSource<IBlock>, tail: GetBlockResult | undefined): void {
2053
- const latest = tail?.state.latest;
2054
- if (latest) {
2055
- source.actionContext = {
2056
- committed: [{ actionId: latest.actionId, rev: latest.rev }],
2057
- rev: latest.rev,
2058
- };
2059
- }
2060
- }
2061
- }
1
+ import type { IBlock, Action, ActionType, ActionHandler, BlockId, ITransactor, BlockStore, Transforms, ActionId, ActionLineage, BlockContentDigests, GetBlockResult } from "../index.js";
2
+ import { Log } from "../log/log.js";
3
+ import type { LogBlock } from "../log/log.js";
4
+ import type { ActionEntry, GetFromResult, LogEntry } from "../log/struct.js";
5
+ import { Atomic } from "../transform/atomic.js";
6
+ import { Tracker } from "../transform/tracker.js";
7
+ import type { BasePins } from "../transform/base-pins.js";
8
+ import { CacheSource } from "../transform/cache-source.js";
9
+ import { computeBlockContentDigests, baseRevsField } from "../transform/digest.js";
10
+ import type { BlockBaseRevs } from "../network/struct.js";
11
+ import { copyTransforms, isTransformsEmpty } from "../transform/helpers.js";
12
+ import { TransactorSource, answeredBlock, servedRevision } from "../transactor/transactor-source.js";
13
+ import { BlockFloors } from "../transactor/block-floors.js";
14
+ import type { WriteDurability } from "../network/struct.js";
15
+ import { mergeDurability } from "../network/durability.js";
16
+ import { highestStaleAt } from "../network/stale-failure.js";
17
+ import type { CollectionHeaderBlock, CollectionId, ICollection, SyncOptions, TornActionReason } from "./index.js";
18
+ import { CollectionHeaderVanishedError, SyncRetryExhaustedError, SyncRevisionStalledError, TornActionError } from "./struct.js";
19
+ import type { ActionContext } from "./action.js";
20
+ import { actionIdAt } from "./action.js";
21
+ import type { ReadDependency } from "../transaction/transaction.js";
22
+ import { clampPriority } from "../transaction/transaction.js";
23
+ import { ReadDependencyCollector } from "../transaction/read-dependency-collector.js";
24
+ import { randomBytes } from '@noble/hashes/utils.js';
25
+ import { toString as uint8ArrayToString } from 'uint8arrays/to-string';
26
+ import { Latches } from "../utility/latches.js";
27
+ import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
28
+ import { createLogger } from "../logger.js";
29
+
30
+ const log = createLogger('collection');
31
+
32
+ /** Which of {@link Collection.advanceContext}'s two callers is reporting — printed as `site=` on
33
+ * every line it emits, because the two compare DIFFERENT pairs of things and a divergence means
34
+ * something different in each:
35
+ *
36
+ * - `refresh` ({@link Collection.updateInternal}) — this instance's own copy against the stored
37
+ * log. A divergence here indicts a forked REPLICA: two copies of one collection id built
38
+ * separately, each internally consistent.
39
+ * - `attach` ({@link Collection.attachToLog}, during open) — the log tail block's claim about
40
+ * which action produced the latest revision, against a walk of that same tail's own chain. Both
41
+ * sides come from storage, so a divergence here indicts STORAGE being self-inconsistent about
42
+ * one revision, not a replica.
43
+ *
44
+ * Without this field the two are indistinguishable in a log, and they lead an operator to
45
+ * completely different places. */
46
+ type DivergenceSite = 'refresh' | 'attach';
47
+
48
+ /** The lowest revision two {@link ActionContext}s provably disagree at, and the action each names
49
+ * there — what {@link Collection.earliestFork} reports and `collection:lineage-divergence` prints
50
+ * as `forkRev=` / `heldAction=` / `readAction=`. */
51
+ type LineageFork = { rev: number, heldAction: ActionId, readAction: ActionId };
52
+
53
+ /** Exactly what one failed write attempt sent to the transactor — enough to send the same thing
54
+ * again. Retained by {@link Collection.retainInFlightAttempt} so that a refresh which then finds
55
+ * this attempt's own log entry can land the blocks the attempt left behind, at the same action id
56
+ * and revision (see {@link Collection.completeOwnEntry}). */
57
+ export type InFlightAttempt = {
58
+ /** The revision the attempt's log entry was stamped with, and pended and committed at. */
59
+ rev: number;
60
+ /** Every transform the attempt pended — the log blocks included. The caller hands over a copy
61
+ * it will not mutate; the collection keeps it as given. */
62
+ transforms: Transforms;
63
+ /** The attempt's log tail block, committed first. */
64
+ tailId: BlockId;
65
+ /** The per-block content declarations the attempt's commit carried, if any. */
66
+ blockDigests?: BlockContentDigests;
67
+ /** The per-block bases the attempt's pend carried, if any (see `PendRequest.baseRevs`). Kept
68
+ * with the transforms because they describe them: a re-send of these operations must name the
69
+ * same bases they were computed against. */
70
+ baseRevs?: BlockBaseRevs;
71
+ };
72
+
73
+ /** What one refresh ({@link Collection.refreshInFlight}, and the refresh inside a sync) found out
74
+ * about the write in flight on the instance's behalf. The caller hands in an empty report and the
75
+ * refresh fills it in AS IT GOES, rather than returning it at the end, so a refresh that throws
76
+ * after it saved the write still says so: the write is saved in storage whatever happens to this
77
+ * instance's local bookkeeping afterwards.
78
+ *
79
+ * NOTE: filled in, not returned, because a return value is lost on a throw — and a refresh CAN
80
+ * throw after finishing its own entry (the invalidation read and the replay both run later). */
81
+ export type RefreshReport = {
82
+ /** Set once the refresh found the in-flight write's own log entry and FINISHED it
83
+ * ({@link Collection.completeOwnEntry}): every block the entry names holds the write, so it is
84
+ * saved. Left unset on every other refresh, including every reader's.
85
+ *
86
+ * `durability` is who holds the finished write, when the refresh learned it. It is absent when
87
+ * finishing found every block already holding the write without re-sending anything (the
88
+ * status-read fallback), which is still a saved write — so test the field, never `durability`,
89
+ * for "saved". */
90
+ ownEntryFinished?: { durability?: WriteDurability };
91
+ };
92
+
93
+ /** The two blocks every refresh starts from, as {@link Collection.readLogEnds} read them. */
94
+ type LogEnds = {
95
+ header: CollectionHeaderBlock;
96
+ /** The repo's answer for the log tail block the header names; absent when the header names no
97
+ * tail or the repo returned no entry for it. */
98
+ tail?: GetBlockResult;
99
+ /** Every block read, with the revision it was served at — the seed for the refresh's block cache. */
100
+ served: Array<[BlockId, IBlock, number]>;
101
+ };
102
+
103
+ /** Default base backoff (and historical fixed delay) between sync retries, in ms. */
104
+ const PendingRetryDelayMs = 100;
105
+ /** Default max consecutive no-progress stale-failure retries before {@link Collection.sync} gives up. */
106
+ const DefaultMaxAttempts = 10;
107
+ /** Default ceiling on a single exponential-backoff sleep, in ms. */
108
+ const DefaultMaxBackoffMs = 5000;
109
+ /** Default consecutive stalled refreshes — ones that moved this collection's revision NOWHERE
110
+ * while a responder CONFIRMED a revision at or above the one being requested — before
111
+ * {@link Collection.sync} gives up. Two, not one, so a single transiently-lagging read is absorbed.
112
+ *
113
+ * NOTE: two is a judgement call, not a measurement — nothing here counts how often a legitimate
114
+ * loser reads a view that moves nowhere for two consecutive rounds. The check only strikes when the
115
+ * refresh made NO progress at all, so a client merely catching up is already excluded; if a
116
+ * spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
117
+ const DefaultMaxStalledAttempts = 2;
118
+
119
+ export type CollectionInitOptions<TAction> = {
120
+ modules: Record<ActionType, ActionHandler<TAction>>;
121
+ createHeaderBlock: (id: BlockId, store: BlockStore<IBlock>) => IBlock;
122
+ /** Called for each local action that is potentially in conflict with a remote action.
123
+ * @param action - The local action to check
124
+ * @param potential - The remote actions that are potentially in conflict
125
+ * @returns The original action (return the same instance to keep it as-is), a replacement
126
+ * action (return a new instance to apply instead of the original — it is re-staged via
127
+ * the conflict replay), or undefined to discard this action
128
+ */
129
+ filterConflict?: (action: Action<TAction>, potential: Action<TAction>[]) => Action<TAction> | undefined
130
+ }
131
+
132
+ /** Options for building a committed read view (see {@link Collection.createReadTracker}
133
+ * and `Tree.readView`). */
134
+ export interface ReadViewOptions {
135
+ /** Record read dependencies into the collection's shared collector.
136
+ * Default false — a pinned committed view is not part of any transaction's
137
+ * conflict set, so its reads must not be able to fail the writer's commit
138
+ * validation. Deferred-constraint safety does not depend on these reads:
139
+ * validator peers re-execute the transaction's recorded statements against
140
+ * their own committed state, so a constraint that no longer holds is caught
141
+ * at validation regardless. */
142
+ recordReads?: boolean;
143
+ /** Pin the view to this committed boundary instead of the collection's CURRENT
144
+ * action context. Pass a {@link CollectionSnapshot.context} so the view describes
145
+ * the same committed moment the snapshot's transforms sat on — even when the
146
+ * collection has committed further since (e.g. a multi-tree commit sweep that has
147
+ * already flushed THIS tree but not its siblings). Blocks cached at revisions
148
+ * newer than the pin are excluded from the view's warm seed and refetched from
149
+ * the transactor at the pinned revision. Default: the current context. */
150
+ pinContext?: ActionContext;
151
+ }
152
+
153
+ /** A point-in-time copy of a collection's staged (un-synced) state, produced by
154
+ * {@link Collection.snapshotPending} and consumed by {@link Collection.restorePending}. */
155
+ export interface CollectionSnapshot<TAction> {
156
+ /** Deep-cloned tracker transforms at snapshot time. */
157
+ transforms: Transforms;
158
+ /** The bases the transforms' update operations were computed against (see {@link BasePins}),
159
+ * copied at snapshot time. Restored with the transforms, so a restore never pairs operations
160
+ * with the bases of some later re-stage: the pend would then declare a base the operations
161
+ * were not built on. Absent on a snapshot built by hand; such a restore keeps whatever pins
162
+ * the tracker holds for the restored ids. */
163
+ pins?: BasePins;
164
+ /** Pending actions queued at snapshot time. */
165
+ pending: Action<TAction>[];
166
+ /** The committed boundary (action context) the staged state sat on when captured.
167
+ * `undefined` for a collection with no committed revision yet (an invented
168
+ * collection whose header/root still live in the tracker). A read view built
169
+ * from this snapshot pins to this boundary (see {@link ReadViewOptions.pinContext}),
170
+ * so the view stays coherent with the snapshot's transforms even if the
171
+ * collection commits further before the view is built. */
172
+ context?: ActionContext;
173
+ }
174
+
175
+ export class Collection<TAction> implements ICollection<TAction> {
176
+ private pending: Action<TAction>[] = [];
177
+ private readonly latchId: string;
178
+
179
+ /** The action id of a write currently in flight ON THIS INSTANCE'S BEHALF, or `undefined`
180
+ * outside a write. Read by {@link updateInternal}: if the committed log now carries an entry
181
+ * under this id, that action's LOG TAIL landed despite the failure answer that sent us back here
182
+ * — every member applies the log tail before the action's other blocks, and
183
+ * `NetworkTransactor.commit` reports failure when the commit carrying the tail was refused after
184
+ * landing on a minority (`commit-not-durable`; in one round the other blocks may have landed on
185
+ * that minority too, in two the sweep never ran) and when a later sweep block confirmed a
186
+ * conflict. Such an entry is never REPLAYED, because replaying re-appends content
187
+ * the committed tail already carries, producing a duplicate entry under one action id at two
188
+ * revisions. But it is not proof the write is saved either: the entry proves only that the tail
189
+ * landed, and the writer's own cancel has since dropped the pending records of every block that
190
+ * did not. So the refresh first FINISHES the action ({@link completeOwnEntry}) — landing the
191
+ * remaining blocks at the same action id and revision, from {@link inFlightAttempt} — and only
192
+ * then consumes the entry ({@link consumeOwnEntry}).
193
+ *
194
+ * The collection owns this fact rather than taking it as a `updateInternal` argument so that no
195
+ * refresh path can forget to supply it — {@link update} and {@link updateAndSync} are refreshes
196
+ * on behalf of a READER, the field is unset for them, and the consume branch cannot fire. Before
197
+ * this was a field, `TransactionCoordinator.commit`'s inter-attempt refresh went through
198
+ * `update()` and was therefore indistinguishable from a reader refresh even though the
199
+ * coordinator held the very id it was retrying. (It now goes through {@link refreshInFlight},
200
+ * which differs from `update()` only in reporting what the refresh saved — the field, not the
201
+ * method, is still what makes the refresh recognise the entry.)
202
+ *
203
+ * LIFETIME is the whole attempt CYCLE, not the latched span: it must survive the refresh
204
+ * BETWEEN a failed attempt and its retry, which is the only moment it is ever read. In
205
+ * {@link syncInternal} that cycle is contained inside the collection latch `sync()` holds; in
206
+ * `TransactionCoordinator.commit` the inter-attempt refresh runs OUTSIDE the commit latch
207
+ * span by design (`Latches` is non-reentrant), so the coordinator's clear necessarily runs
208
+ * latch-free. That is safe: this is a single field write, {@link beginInFlightAction}'s
209
+ * disposer only clears an id it still owns, and the only reader runs under the latch — so the
210
+ * worst a foreign concurrent refresh can see is a cleared field (it stops consuming), never a
211
+ * field it should not have consumed. */
212
+ private inFlightActionId?: ActionId;
213
+
214
+ /** The most recent FAILED attempt made under {@link inFlightActionId} — exactly what it sent,
215
+ * kept so a refresh that finds that attempt's own log entry can finish the action instead of
216
+ * assuming it is finished (see {@link completeOwnEntry}). Set by {@link retainInFlightAttempt},
217
+ * cleared with the mark by {@link beginInFlightAction}'s disposer, and meaningless without it.
218
+ *
219
+ * The transforms are retained VERBATIM, not rebuilt: a rebuilt attempt re-appends the log entry
220
+ * with a fresh timestamp, so its log tail would differ byte-for-byte from the one already
221
+ * stored under the same action id and revision — and any replica that had not yet stored the
222
+ * tail would then store the second version, leaving two contents under one `(action, revision)`.
223
+ *
224
+ * Only the latest attempt is kept. An own entry can only be visible at the revision of an
225
+ * attempt whose tail landed, and a later attempt at a DIFFERENT revision is only made after a
226
+ * refresh adopted somebody else's entry at the earlier one — which is proof the earlier tail did
227
+ * not land. {@link completeOwnEntry} still checks the revision and refuses on a mismatch. */
228
+ private inFlightAttempt?: InFlightAttempt;
229
+
230
+ /** The log tail block id the most recent header read named. A refresh asks for this block in the
231
+ * same request as the header ({@link readLogEnds}): the tail id only changes when the tail block
232
+ * fills, so an idle refresh is one request rather than two. A stale value costs one extra request
233
+ * and nothing else. */
234
+ private logTailId?: BlockId;
235
+
236
+ protected constructor(
237
+ public readonly id: CollectionId,
238
+ public readonly transactor: ITransactor,
239
+ private readonly handlers: Record<ActionType, ActionHandler<TAction>>,
240
+ private readonly source: TransactorSource<IBlock>,
241
+ /** Cache of unmodified blocks from the source */
242
+ private readonly sourceCache: CacheSource<IBlock>,
243
+ /** Tracked Changes */
244
+ public readonly tracker: Tracker<IBlock>,
245
+ /** What each block named by a walked log entry must be at least as new as — raised by
246
+ * {@link updateInternal}, and shared with every read source this handle builds (see
247
+ * {@link BlockFloors}). */
248
+ private readonly floors: BlockFloors,
249
+ private readonly filterConflict?: (action: Action<TAction>, potential: Action<TAction>[]) => Action<TAction> | undefined,
250
+ /** Short random tag naming THIS instance (see {@link newInstanceTag}). Open paths generate
251
+ * it BEFORE construction (so pre-construction diagnostics such as attachToLog can carry it);
252
+ * the default covers direct construction in tests. */
253
+ public readonly instanceTag: string = Collection.newInstanceTag(),
254
+ ) {
255
+ // Instance-scoped, deliberately NOT shared across instances of one collection id. The
256
+ // latch protects per-instance state only — the tracker, the pending queue, and
257
+ // source.actionContext, none of which two instances over the same id share — while
258
+ // cross-instance races are resolved by the transactor's optimistic concurrency (that is
259
+ // the design; the old process-global `Collection:${id}` key serialized instances by
260
+ // accident). Instance scope is also what lets TransactionCoordinator hold this latch
261
+ // across its whole commit span: `Latches` is a non-reentrant FIFO mutex, and a rival
262
+ // writer driving a SECOND instance of the same id from inside transactor.pend (see
263
+ // CompetingWriterTransactor) would otherwise wait on the very latch the parked commit
264
+ // holds — a deadlock, not contention.
265
+ this.latchId = `Collection:${this.id}#${this.instanceTag}`;
266
+ }
267
+
268
+ /** A fresh instance tag: four random bytes rendered base64url — six characters, enough that
269
+ * two instances over one collection id do not collide by accident, short enough to ride on
270
+ * every trace line (same shape as the node tag in quereus-plugin-optimystic's
271
+ * collection-factory). Scopes {@link latchId} per instance and labels diagnostics. */
272
+ private static newInstanceTag(): string {
273
+ return uint8ArrayToString(randomBytes(4), 'base64url');
274
+ }
275
+
276
+ /** Open an EXISTING collection.
277
+ *
278
+ * Resolves to `undefined` when the header block probe comes back empty — an
279
+ * authoritatively absent header, meaning nothing has ever been committed under this id.
280
+ * A header the storage layer could not RETRIEVE (a revision this node cannot
281
+ * reconstruct, an unreachable cohort) is not absent: the probe throws
282
+ * {@link BlockUnavailableError} instead of resolving `undefined`, so an unreachable
283
+ * collection can never be mistaken for a nonexistent one.
284
+ *
285
+ * Use this wherever reading — not creating — is what was meant. {@link createOrOpen}
286
+ * would instead stage a fresh empty collection, and reads through it would report an
287
+ * absent dataset as a legitimately empty one. */
288
+ static async open<TAction>(transactor: ITransactor, id: CollectionId, init: CollectionInitOptions<TAction>): Promise<Collection<TAction> | undefined> {
289
+ // Generated BEFORE anything reads, so every diagnostic of this handle — the floors the probe
290
+ // wires up and the log-attach-time lines included — names the instance the same way
291
+ // post-construction ones do.
292
+ const instanceTag = Collection.newInstanceTag();
293
+ const { source, sourceCache, tracker, floors, header } = await Collection.probeHeader(transactor, id, instanceTag);
294
+ if (!header) {
295
+ // Return before anything is staged: the tracker's transforms stay empty, so a caller
296
+ // that ignores the undefined cannot later sync a phantom collection into existence.
297
+ return undefined;
298
+ }
299
+ await Collection.attachToLog<TAction>(source, transactor, tracker, id, instanceTag, header);
300
+ const collection = new Collection(id, transactor, init.modules, source, sourceCache, tracker, floors, init.filterConflict, instanceTag);
301
+ collection.logTailId = header.tailId;
302
+ return collection;
303
+ }
304
+
305
+ /** Open an existing collection, or stage a fresh empty one in the local tracker when the
306
+ * header is authoritatively absent. Nothing is written to storage until {@link sync}.
307
+ *
308
+ * Correct only where inventing a collection is genuinely intended — a first write, a
309
+ * bootstrap path. The create branch logs `collection:invented`; prefer {@link open} on
310
+ * any pure read path. */
311
+ static async createOrOpen<TAction>(transactor: ITransactor, id: CollectionId, init: CollectionInitOptions<TAction>): Promise<Collection<TAction>> {
312
+ // Pre-construction for the same reason as in open(): see the comment there.
313
+ const instanceTag = Collection.newInstanceTag();
314
+ const { source, sourceCache, tracker, floors, header } = await Collection.probeHeader(transactor, id, instanceTag);
315
+
316
+ if (header) { // Collection already exists
317
+ await Collection.attachToLog<TAction>(source, transactor, tracker, id, instanceTag, header);
318
+ } else { // Collection does not exist
319
+ log('collection:invented id=%s — no committed header found; staging a fresh empty collection', id);
320
+ const headerBlock = init.createHeaderBlock(id, tracker);
321
+ tracker.insert(headerBlock);
322
+ source.actionContext = undefined;
323
+ await Log.open<Action<TAction>>(tracker, id);
324
+ }
325
+
326
+ const collection = new Collection(id, transactor, init.modules, source, sourceCache, tracker, floors, init.filterConflict, instanceTag);
327
+ collection.logTailId = header?.tailId;
328
+ return collection;
329
+ }
330
+
331
+ /** The per-instance read wiring every open path needs, plus the header probe result.
332
+ * Shared by {@link open} and {@link createOrOpen} so the two cannot drift. */
333
+ private static async probeHeader(transactor: ITransactor, id: CollectionId, instanceTag: string): Promise<{
334
+ source: TransactorSource<IBlock>,
335
+ sourceCache: CacheSource<IBlock>,
336
+ tracker: Tracker<IBlock>,
337
+ floors: BlockFloors,
338
+ header: CollectionHeaderBlock | undefined,
339
+ }> {
340
+ // Start with a context that has an infinite revision number to ensure that we always fetch the latest log information.
341
+ // One shared read-dependency collector feeds both the source (direct structural reads) and the cache (every
342
+ // cache hit/miss), so a block read from either layer records a dependency — cache hits included.
343
+ const collector = new ReadDependencyCollector();
344
+ const floors = Collection.newFloors(id, instanceTag);
345
+ const source = new TransactorSource(id, transactor, undefined, collector, floors);
346
+ const sourceCache = new CacheSource(source, undefined, collector);
347
+ const tracker = new Tracker(sourceCache);
348
+ const header = await source.tryGet(id) as CollectionHeaderBlock | undefined;
349
+ return { source, sourceCache, tracker, floors, header };
350
+ }
351
+
352
+ /** A new handle's floors: none yet (opening walks no entries), wired to report every below-floor
353
+ * answer any of the handle's read sources receives. The line is the only trace such an answer
354
+ * leaves — the read itself succeeds (see the accepted-tradeoff NOTE at
355
+ * `TransactorSource.mayRetain`) — so repeated lines for one block with `servedRev` short of
356
+ * `floorRev` are how an operator sees a machine that has not caught up, and lines that never
357
+ * stop are how they see a log entry whose blocks never landed. */
358
+ private static newFloors(id: CollectionId, instanceTag: string): BlockFloors {
359
+ return new BlockFloors(({ blockId, floor, servedRev }) => {
360
+ log('collection:block-below-floor id=%s tag=%s block=%s floorRev=%d floorAction=%s servedRev=%d',
361
+ id, instanceTag, blockId, floor.rev, floor.actionId, servedRev);
362
+ });
363
+ }
364
+
365
+ /** Walk an existing collection's log and point the source at its latest action context.
366
+ * A header we just probed successfully but whose log will not open is a fault, not an
367
+ * absence — throw rather than let the collection read as empty. (The re-read goes through
368
+ * the tracker/cache, so it can disagree with the probe when storage is flaky mid-open.) */
369
+ private static async attachToLog<TAction>(
370
+ source: TransactorSource<IBlock>,
371
+ transactor: ITransactor,
372
+ tracker: Tracker<IBlock>,
373
+ id: CollectionId,
374
+ /** The tag the calling open path minted for the Collection it is ABOUT to construct, so a
375
+ * diagnostic emitted here carries the same instance name as every post-construction one. */
376
+ instanceTag: string,
377
+ header: CollectionHeaderBlock,
378
+ ): Promise<void> {
379
+ // Bootstrap ActionContext from the committed tail before walking the chain.
380
+ // This allows the transactor to serve pending non-tail blocks during Log.open.
381
+ Collection.bootstrapContext(source, header.tailId === undefined ? undefined : await Collection.readLogTail(transactor, header.tailId));
382
+
383
+ const collectionLog = await Log.open<Action<TAction>>(tracker, id);
384
+ if (!collectionLog) {
385
+ throw new Error(`Log not found for collection ${id}`);
386
+ }
387
+ // Monotonic, not an overwrite: getActionContext resolves undefined when the chain has no
388
+ // tail or the tail block carries zero entries, and that must not erase the revision
389
+ // bootstrapContext just read off the committed tail.
390
+ Collection.advanceContext(source, id, instanceTag, 'attach', await collectionLog.getActionContext());
391
+ }
392
+
393
+ /** Adopt a freshly-read action context WITHOUT ever lowering the revision already held.
394
+ *
395
+ * The revision a collection last committed at is knowledge it earned; a read that found
396
+ * nothing — or found an older view of the log — cannot un-earn it. Silently accepting the
397
+ * lower value makes the next sync ask for a revision that is long gone, and every retry
398
+ * repeats the same doomed request because each retry re-runs the same losing read.
399
+ *
400
+ * Equal revisions still adopt `next`: the rev is unchanged but its `committed` list may be
401
+ * more complete than what we hold.
402
+ *
403
+ * This is also the one seam where lineage divergence is observable: at every revision BOTH
404
+ * sides name an action for, the two ids must agree. Revision
405
+ * numbers are per-collection counters, so two separately-built copies under one id can each
406
+ * occupy the same revision with DIFFERENT actions while each stays internally self-consistent
407
+ * — {@link reportShortfall} structurally cannot see that (its two numbers come from one
408
+ * chain), and this is the only place two `committed` lists meet. Naming different actions at
409
+ * one revision proves the two sides are different lineages (`collection:lineage-divergence`;
410
+ * see docs/debugging.md § "Did the refresh itself fail to close the gap?"). WHICH two sides
411
+ * depends on the caller, and the line says so in `site=` — see {@link DivergenceSite}, which
412
+ * defines the two values and what each one indicts.
413
+ *
414
+ * Every line from here also carries `tag=`, the {@link Collection.instanceTag} of the handle
415
+ * reporting. One process routinely holds several handles on one collection id; without the
416
+ * tag, two handles' lines interleave into what reads like one handle contradicting itself.
417
+ *
418
+ * Logs, does not throw — same reasoning as {@link reportShortfall}: `update()` runs
419
+ * blanket-style over every registered collection between commit retries, and aborting here
420
+ * would promote a diagnosis to production behaviour before the line has ever been seen to
421
+ * fire in the wild. Adoption then proceeds unchanged, which means the line is a PER-DISCOVERY
422
+ * report, not a per-refresh one: adopting `next` overwrites the held lineage marker with the
423
+ * log's, so the next refresh of this instance compares log-to-log and stays silent — even
424
+ * though block content materialized under the old lineage may still be in caches. The line
425
+ * marks the refresh that first observed the disagreement.
426
+ *
427
+ * NOTE: adoption resolves the CONTEXT disagreement, not the content one — the read caches on
428
+ * this instance still hold blocks materialized under the old lineage, and since the revision
429
+ * did not change nothing re-reads them. Conditional today: no fork has been reproduced (see
430
+ * the still-open upstream reproducer), so this instrument exists to find out whether one
431
+ * happens at all. If the line is ever seen firing in the field, decide then whether a
432
+ * divergence should also drop the read cache (and whether to keep re-reporting per refresh)
433
+ * — that is a behaviour change, and this seam deliberately makes none.
434
+ *
435
+ * The comparison is {@link earliestFork}, not a single lookup at the current revision: the
436
+ * two `committed` lists overlap across several revisions, and the LOWEST one they disagree at
437
+ * is where the lineages actually parted — a fork below the current revision was previously
438
+ * silent. `forkRev=` names it, `heldAction=`/`readAction=` are the two ids AT it, and
439
+ * `heldRev=`/`readRev=` are the two contexts' own revisions, so the line says both where the
440
+ * split began and how far each side has since travelled.
441
+ *
442
+ * The refusal line reports its two action ids at `readRev=` — the read's revision — on BOTH
443
+ * sides, because that is the only revision the two can be compared at: `next` never names an
444
+ * action above its own revision, so looking each side up at its own revision would compare
445
+ * different revisions and print two different ids for one honest lineage. Equal ids there mean
446
+ * the read is an older view of THIS lineage (ordinary lag, correctly refused); different ids
447
+ * mean a fork; `none` on the held side means this handle's own list does not reach back to the
448
+ * read's revision — the signature of a context bootstrapped from an over-claiming tail (see
449
+ * the NOTE in {@link bootstrapContext}), which is exactly the case {@link earliestFork} has no
450
+ * shared revision to report on.
451
+ *
452
+ * Gated on `log.enabled`, like every {@link actionIdAt} caller: the comparison buys nothing
453
+ * when the line has no sink, and the lists — one entry per commit between context reads,
454
+ * truncated at each checkpoint — are only walked on a run that has the namespace turned on.
455
+ * Silence proves nothing either way, because a revision is only comparable when BOTH sides
456
+ * name an action for it: an invented collection has no context at all, a revision slot the log
457
+ * gave to a checkpoint or invalidation entry names none, and a revision older than the read
458
+ * log's most recent checkpoint has already fallen off the read side's list. */
459
+ private static advanceContext(
460
+ source: TransactorSource<IBlock>,
461
+ id: CollectionId,
462
+ instanceTag: string,
463
+ site: DivergenceSite,
464
+ next: ActionContext | undefined,
465
+ ): void {
466
+ const current = source.actionContext;
467
+ if (next === undefined) {
468
+ return; // The read learned nothing — keep what we already know.
469
+ }
470
+ if (current !== undefined && log.enabled) {
471
+ const fork = Collection.earliestFork(current, next);
472
+ if (fork !== undefined) {
473
+ log('collection:lineage-divergence id=%s tag=%s site=%s forkRev=%d heldAction=%s readAction=%s heldRev=%d readRev=%d',
474
+ id, instanceTag, site, fork.rev, fork.heldAction, fork.readAction, current.rev, next.rev);
475
+ }
476
+ }
477
+ if (current !== undefined && next.rev < current.rev) {
478
+ // The refusal itself is unconditional; only the id lookups that explain it are gated.
479
+ if (log.enabled) {
480
+ log('collection:context-not-lowered id=%s tag=%s site=%s heldRev=%d readRev=%d heldAction=%s readAction=%s',
481
+ id, instanceTag, site, current.rev, next.rev,
482
+ actionIdAt(current, next.rev) ?? 'none', actionIdAt(next, next.rev) ?? 'none');
483
+ }
484
+ return;
485
+ }
486
+ source.actionContext = next;
487
+ }
488
+
489
+ /** The EARLIEST revision the two contexts provably disagree about: the lowest revision both
490
+ * `committed` lists name an action for, where the two ids differ.
491
+ *
492
+ * Comparing only at the holder's current revision — what this used to do — misses a fork that
493
+ * began earlier and has since been overtaken by same-numbered commits on both sides, which is
494
+ * the shape a replica that forked and kept writing actually has. Taking the lowest disagreeing
495
+ * revision instead names the split point rather than an arbitrary later symptom of it.
496
+ *
497
+ * Revisions only one side names are skipped, not treated as disagreement: {@link actionIdAt}'s
498
+ * `undefined` is legitimate (checkpoint/invalidation slots, and revisions that predate the
499
+ * other side's most recent checkpoint), so a one-sided entry is missing evidence, not evidence
500
+ * of a fork.
501
+ *
502
+ * NOTE: linear in the two lists, which hold one entry per commit between context reads and
503
+ * truncate at each checkpoint. Every caller is `log.enabled`-gated, so this does not run at
504
+ * all on a normal run; if a non-diagnostic caller ever appears, index by revision instead. */
505
+ private static earliestFork(held: ActionContext, read: ActionContext): LineageFork | undefined {
506
+ // NOTE: a `committed` list carrying TWO entries at one revision would be a defect in its own
507
+ // right, and this keeps the last of them arbitrarily. Harmless while every caller is a
508
+ // diagnostic; if such a list is ever seen, report the duplicate rather than silently
509
+ // picking one.
510
+ const readIds = new Map(read.committed.map(entry => [entry.rev, entry.actionId]));
511
+ let earliest: LineageFork | undefined;
512
+ for (const entry of held.committed) {
513
+ const readAction = readIds.get(entry.rev);
514
+ if (readAction === undefined || readAction === entry.actionId) {
515
+ continue;
516
+ }
517
+ if (earliest === undefined || entry.rev < earliest.rev) {
518
+ earliest = { rev: entry.rev, heldAction: entry.actionId, readAction };
519
+ }
520
+ }
521
+ return earliest;
522
+ }
523
+
524
+ /** Report a refresh that failed to move FORWARDS past a revision it had already read for
525
+ * itself — the sibling of {@link advanceContext}'s `collection:context-not-lowered`, which
526
+ * reports a collection declining to move BACKWARDS.
527
+ *
528
+ * `tailRev` is what the committed log tail claimed is committed under this id; `after` is
529
+ * where a SEPARATE read path (the chain walk) actually landed. Landing below the claim means
530
+ * this refresh closed nothing, which from outside the class is otherwise indistinguishable
531
+ * from "there was nothing newer to adopt".
532
+ *
533
+ * This detects LAG, and only lag. It CANNOT see lineage divergence: both of its numbers
534
+ * come from the same chain — `tailRev` off the tail block this collection's own header
535
+ * names, `after` from a walk of that same chain — and a forked replica is internally
536
+ * self-consistent, its tail claiming exactly what its own walk reaches. Two copies of one
537
+ * collection id holding the same revision under different actions therefore keep this line
538
+ * silent forever. That case is `collection:lineage-divergence`, reported from
539
+ * {@link advanceContext}, which compares action ids — the one value comparable across
540
+ * copies — rather than revision counters.
541
+ *
542
+ * Carries the same `tag=` as {@link advanceContext}'s lines, and for the same reason: several
543
+ * handles on one collection id otherwise read as one self-contradicting handle.
544
+ *
545
+ * Logs, does not throw: `update()` is called blanket-style over every registered collection
546
+ * between commit retries, and a shortfall is not yet known to be illegitimate — an abort here
547
+ * would promote an unproven diagnosis to production behaviour. Deliberately does NOT adopt
548
+ * `tailRev` either: the two numbers come from different read paths, and papering over the
549
+ * disagreement destroys the evidence this line exists to produce. */
550
+ private static reportShortfall(id: CollectionId, instanceTag: string, tailRev: number | undefined, before: number | undefined, after: number | undefined): void {
551
+ if (tailRev === undefined || (after !== undefined && after >= tailRev)) {
552
+ return;
553
+ }
554
+ log('collection:context-short-of-tail id=%s tag=%s before=%s after=%s tail=%d',
555
+ id, instanceTag, before ?? 'none', after ?? 'none', tailRev);
556
+ }
557
+
558
+ async act(...actions: Action<TAction>[]) {
559
+ const release = await Latches.acquire(this.latchId);
560
+ try {
561
+ await this.actInternal(...actions);
562
+ } finally {
563
+ release();
564
+ }
565
+ }
566
+
567
+ private async actInternal(...actions: Action<TAction>[]) {
568
+ await this.internalTransact(...actions);
569
+ this.pending.push(...actions);
570
+ }
571
+
572
+ private async internalTransact(...actions: Action<TAction>[]) {
573
+ const atomic = new Atomic(this.tracker);
574
+
575
+ for (const action of actions) {
576
+ const handler = this.handlers[action.type];
577
+ if (!handler) {
578
+ throw new Error(`No handler for action type ${action.type}`);
579
+ }
580
+ await handler(action, atomic);
581
+ }
582
+
583
+ atomic.commit();
584
+ }
585
+
586
+ /** Load external changes and update our context to the latest log revision - resolve any conflicts with our pending actions. */
587
+ async update() {
588
+ const release = await Latches.acquire(this.latchId);
589
+ try {
590
+ await this.updateInternal({});
591
+ } finally {
592
+ release();
593
+ }
594
+ }
595
+
596
+ /** The refresh `TransactionCoordinator.commit` runs between attempts: exactly {@link update},
597
+ * plus a report of whether it finished the write in flight on this instance's behalf (see
598
+ * {@link RefreshReport}). The coordinator needs that fact to tell its caller which participants
599
+ * are already saved when the commit later fails; a reader's `update()` has no write in flight
600
+ * and nothing to report.
601
+ *
602
+ * `report` is REQUIRED so a write path cannot refresh without learning what the refresh saved.
603
+ * It is filled in as the refresh goes, so it is accurate when this throws too.
604
+ *
605
+ * `lastChance` is REQUIRED for the same reason: only the caller knows whether its retry budget
606
+ * ends with this round, and a round that is the last must settle a half-landed write rather than
607
+ * ask for another (see {@link completeOwnEntry}). */
608
+ async refreshInFlight(report: RefreshReport, lastChance: boolean): Promise<void> {
609
+ const release = await Latches.acquire(this.latchId);
610
+ try {
611
+ await this.updateInternal(report, lastChance);
612
+ } finally {
613
+ release();
614
+ }
615
+ }
616
+
617
+ /** Finishes a half-landed write BEFORE its own log entry is consumed, so that consuming never
618
+ * reports a write as saved on the strength of its log entry alone.
619
+ *
620
+ * THE RULE: a write may be reported saved only if EVERY block its log entry names holds the
621
+ * write — at the write's own revision, or at a later revision that was BUILT FROM it. Finding the
622
+ * entry proves only that the log tail landed (see {@link inFlightActionId} for the two ways
623
+ * `NetworkTransactor.commit` answers failure over a stored tail). The writer's cancel has since
624
+ * removed the pending records of every block that did not land, so nothing else will ever land
625
+ * them — if this does not, the entry stands in the log, the blocks stay at their previous
626
+ * revision on every node, and readers materialize blocks, not log entries: the write is
627
+ * silently gone.
628
+ *
629
+ * Finishing is a plain re-send of the retained failed attempt ({@link inFlightAttempt}) — the
630
+ * SAME transforms, action id, revision and tail. Every storage tier treats a block that already
631
+ * holds exactly this action at exactly this revision as satisfied rather than as a rival
632
+ * (`isOwnRevision`: `StorageRepo.pend`/`.commit`, `ClusterMember`, `CoordinatorRepo`), so the
633
+ * re-send rolls forward precisely the blocks that are missing and is a no-op for the rest,
634
+ * including when nothing is missing at all. It is never sent at a new revision: the refresh has
635
+ * already seen the entry at this one, and a second revision would record the entry twice.
636
+ *
637
+ * A refused re-send is NOT yet an answer. Storage refuses it whenever any block has moved past
638
+ * the write's revision — and every later commit to the collection moves the log tail past it —
639
+ * which says a rival was there, not whether the rival built on this write or over it. That is
640
+ * asked separately, of the blocks' own history ({@link settleUnfinished}).
641
+ *
642
+ * Runs at the top of {@link updateInternal}, before that method has changed anything on this
643
+ * instance, so every throw from here leaves the collection exactly as the failed attempt left
644
+ * it — staged actions and transforms intact, revision not advanced.
645
+ *
646
+ * @param lastChance - the caller will not refresh again for this write (its retry budget ends
647
+ * with this round), so a refusal that could clear is settled now instead of asking for another
648
+ * round: the error that escapes then says whether the write can still land.
649
+ * @returns who holds the finished write, for the sync to report.
650
+ * @throws TornActionError — see {@link TornActionReason} for the three causes. Only an unsettled
651
+ * `completion-refused` (always `final: false`) is retried by the write paths.
652
+ *
653
+ * NOTE: the re-send costs a full pend and commit round even when every block had in fact landed
654
+ * (a lost or masked success). That is deliberate — it is the source of the durability the sync
655
+ * reports when nothing has superseded the write — and the case is rare: after a returned failure
656
+ * the network transactor has, by construction, NOT swept every block. If own-entry refreshes ever
657
+ * show up as a cost, ask `getLineage` first and skip the re-send when every block contains the
658
+ * write. */
659
+ private async completeOwnEntry(entry: ActionEntry<Action<TAction>>, entryRev: number | undefined, lastChance: boolean): Promise<WriteDurability | undefined> {
660
+ const attempt = this.inFlightAttempt;
661
+ const rev = entryRev ?? attempt?.rev;
662
+ if (attempt === undefined || rev === undefined || attempt.rev !== rev) {
663
+ // Nothing to finish the action WITH. That is only acceptable if there is nothing to
664
+ // finish. `getStatus` is the cheap first question; it judges a block by who holds its
665
+ // LATEST revision, so a block this action landed and a later action has since built on
666
+ // reads there as not committed — those are asked again, properly, by settleUnfinished.
667
+ // Both write paths retain an attempt before any refresh can run, so production only
668
+ // reaches this branch when the entry sits at a revision the retained attempt was not
669
+ // made at — a forked lineage.
670
+ const [status] = await this.transactor.getStatus([{ actionId: entry.actionId, blockIds: entry.blockIds }]);
671
+ if (entry.blockIds.every((_, i) => status?.statuses[i] === 'committed')) {
672
+ // Whole, and saved: the refresh reports it as finished (see RefreshReport), so the
673
+ // coordinator counts this participant as committed. Only WHO holds it is unknown.
674
+ // NOTE: a sync whose only commit was recognised here therefore answers `undefined` —
675
+ // the "nothing was written" answer — for a write that is saved, because the sync reads
676
+ // the report's durability, not the finished flag. Reachable only on the forked-lineage
677
+ // path above; if that path ever becomes ordinary, take the durability from
678
+ // `getLineage` (as settleUnfinished does) instead of stopping at the status read.
679
+ return undefined;
680
+ }
681
+ return await this.settleUnfinished(entry, rev, undefined, 'transforms-not-held',
682
+ attempt === undefined
683
+ ? 'no failed attempt is retained for this action'
684
+ : `the retained attempt was made at rev ${attempt.rev}`);
685
+ }
686
+
687
+ // NOTE: priority 0. The attempt's aged retry priority is a fairness hint for a race over a
688
+ // free revision; this revision is already this action's own, so there is no race to rank in.
689
+ // NOTE: this is a plain pend. When the failed attempt came from `TransactionCoordinator`, its
690
+ // pend carried `validation` (the transaction and its operations hash) and
691
+ // `superclusterNominees`; neither is retained, so the re-send carries neither. Harmless while
692
+ // no deployment hands members a transaction validator (and nothing reads the nominees on
693
+ // the receiving side at all). Once a validator is wired, members approve these
694
+ // blocks unchecked under `unvalidatablePendPolicy: 'accept'` and refuse them under `'reject'`
695
+ // (surfacing as a `completion-refused` TornActionError). Simply retaining and re-sending the
696
+ // pair is not obviously right either: a member re-executing the transaction after a sibling
697
+ // participant has landed no longer sees the state it was staged against. Tracked as an arm
698
+ // of tickets/backlog/feat-no-deployment-validates-transactions-at-pend.
699
+ const result = await this.source.transact(attempt.transforms, entry.actionId, rev, this.id, attempt.tailId, 0, attempt.blockDigests, attempt.baseRevs);
700
+ if (result.success) {
701
+ return result.durability;
702
+ }
703
+ // A refusal that CONFIRMS a committed revision under another action — `staleAt`, which every
704
+ // producer sets only after reading it out of its own storage and never for this action's own
705
+ // revision, or a non-empty `missing` list of committed rival transforms — will be repeated by
706
+ // every later re-send. Anything else (a rival merely PENDING on a block, a revision not yet
707
+ // held by a majority, a bare reason) can clear, and is worth the caller's next round.
708
+ // NOTE: `staleAt` is read here as "a rival committed", which is a second consumer of a field
709
+ // documented as never a retryability signal (docs/internals.md). It is the same kind of use
710
+ // `syncAttempts`' stall check makes: it can only END a retry, never start one.
711
+ const rivalConfirmed = result.staleAt !== undefined || (result.missing?.length ?? 0) > 0;
712
+ // NOTE: a confirmed rival settles at once, even with retry budget left: re-sending is
713
+ // pointless, but the lineage question is asked only this once, so a member that is silent
714
+ // just now makes the answer `final: false`. If unsettled outcomes ever show up often under
715
+ // contention, spend the remaining rounds re-asking `getLineage` before giving up.
716
+ if (!rivalConfirmed && !lastChance) {
717
+ // The refusal does not say which blocks lack the revision, so name every block the entry
718
+ // lists other than the tail — the entry being visible is what proves the tail holds it.
719
+ return this.throwTorn(entry, rev, entry.blockIds.filter(blockId => blockId !== attempt.tailId),
720
+ 'completion-refused', false, result.reason ?? 'the re-send was refused');
721
+ }
722
+ return await this.settleUnfinished(entry, rev, attempt.tailId,
723
+ rivalConfirmed ? 'rival-holds-revision' : 'completion-refused',
724
+ result.reason ?? (rivalConfirmed ? 'a different action holds a later revision' : 'the re-send was refused'),
725
+ result.staleAt);
726
+ }
727
+
728
+ /** Decides what a write that can no longer be finished BY RE-SENDING amounts to, and answers
729
+ * one of exactly three things — never a guess between them:
730
+ *
731
+ * - SAVED (returns): every block the entry names holds content built from the write, on a
732
+ * strict majority of its cohort (`ITransactor.getLineage`). The rival that refused the re-send
733
+ * had read this write and added to it. Returns who holds it.
734
+ * - TORN AND FINAL (throws, `final: true`): every block still missing the write answered, for its
735
+ * whole cohort, that it does not hold it — and the write's pending records were confirmed gone
736
+ * BEFORE the blocks were asked. The order is the point: a pending record left standing can be
737
+ * promoted by any later read that knows this write's log entry is committed
738
+ * (`StorageRepo.get`), so a block that answered "not reached" could still take the write
739
+ * afterwards. With the records gone first, nothing is left that could land it.
740
+ * - TORN, OUTCOME NOT ESTABLISHED (throws, `final: false`): anything else — the transactor
741
+ * cannot answer for lineage, a cohort did not all answer or contradicted itself, fewer than a
742
+ * majority hold the write, or the cancel could not be confirmed.
743
+ *
744
+ * `tailId` is the log block this write's entry was appended to, when known. It is never asked
745
+ * about: the refresh has just READ the entry out of it, which is direct evidence that the
746
+ * block's current content was built from the write, whatever any member's records can prove.
747
+ *
748
+ * NOTE: a write recognised as saved here reports the durability `getLineage` assembled. When
749
+ * the log block's own cohort could not vouch for it (members that took it as a replica), that is
750
+ * absent and the sync answers `undefined` for a saved write — the same wart the status-read
751
+ * branch of {@link completeOwnEntry} documents. If it is ever seen, fold the data blocks'
752
+ * reports alone rather than inventing one for the log block. */
753
+ private async settleUnfinished(
754
+ entry: ActionEntry<Action<TAction>>,
755
+ rev: number | undefined,
756
+ tailId: BlockId | undefined,
757
+ reason: TornActionReason,
758
+ detail: string,
759
+ staleAt?: { blockId: BlockId; rev: number },
760
+ ): Promise<WriteDurability | undefined> {
761
+ const discharged = await this.dischargeOwnPendings(entry);
762
+ const lineage = rev === undefined ? undefined : await this.lineageOfOwnEntry(entry, rev);
763
+ const unsaved = entry.blockIds.filter((blockId, i) => blockId !== tailId && lineage?.blocks[i] !== 'contains');
764
+ if (lineage !== undefined && unsaved.length === 0) {
765
+ log('collection:own-entry-superseded-but-saved id=%s tag=%s action=%s rev=%d blocks=%d',
766
+ this.id, this.instanceTag, entry.actionId, rev, entry.blockIds.length);
767
+ return lineage.durability;
768
+ }
769
+ const cannotLand = lineage !== undefined && unsaved.every(blockId => {
770
+ const answer = lineage.blocks[entry.blockIds.indexOf(blockId)];
771
+ return answer === 'excludes' || answer === 'behind';
772
+ });
773
+ return this.throwTorn(entry, rev ?? -1, unsaved, reason, discharged && cannotLand, detail, staleAt);
774
+ }
775
+
776
+ /** Cancels every pending record this write may have left, and says whether that is CONFIRMED.
777
+ * `ITransactor.cancel` returns only once the records are gone and throws otherwise; a throw here
778
+ * is reported as "not confirmed" rather than raised, because the caller is already reporting a
779
+ * torn write and that must not be displaced. Cancelling a block that holds no record is a no-op,
780
+ * so naming every block the entry lists is safe. */
781
+ private async dischargeOwnPendings(entry: ActionEntry<Action<TAction>>): Promise<boolean> {
782
+ try {
783
+ await this.transactor.cancel({ actionId: entry.actionId, blockIds: entry.blockIds });
784
+ return true;
785
+ } catch (err) {
786
+ log('collection:torn-cancel-unconfirmed id=%s tag=%s action=%s error=%s',
787
+ this.id, this.instanceTag, entry.actionId, err instanceof Error ? err.message : String(err));
788
+ return false;
789
+ }
790
+ }
791
+
792
+ /** What the blocks' own history says about this write, or `undefined` when nothing can say: the
793
+ * transactor (or a wrapper around it) does not offer `getLineage`, or the question failed. */
794
+ private async lineageOfOwnEntry(entry: ActionEntry<Action<TAction>>, rev: number): Promise<ActionLineage | undefined> {
795
+ if (this.transactor.getLineage === undefined) {
796
+ return undefined;
797
+ }
798
+ try {
799
+ return await this.transactor.getLineage({ actionId: entry.actionId, blockIds: entry.blockIds, rev });
800
+ } catch (err) {
801
+ log('collection:lineage-unanswered id=%s tag=%s action=%s rev=%d error=%s',
802
+ this.id, this.instanceTag, entry.actionId, rev, err instanceof Error ? err.message : String(err));
803
+ return undefined;
804
+ }
805
+ }
806
+
807
+ private throwTorn(
808
+ entry: ActionEntry<Action<TAction>>,
809
+ rev: number,
810
+ blockIds: BlockId[],
811
+ reason: TornActionReason,
812
+ final: boolean,
813
+ detail: string,
814
+ staleAt?: { blockId: BlockId; rev: number },
815
+ ): never {
816
+ log('collection:torn id=%s tag=%s action=%s rev=%d reason=%s final=%s blocks=%o',
817
+ this.id, this.instanceTag, entry.actionId, rev, reason, final, blockIds);
818
+ throw new TornActionError(this.id, entry.actionId, rev, blockIds, reason, final, detail, staleAt);
819
+ }
820
+
821
+ /** Drops the pending actions this sync's OWN committed entry made durable, instead of replaying
822
+ * them into a duplicate entry (see {@link inFlightActionId}). Only ever called once
823
+ * {@link completeOwnEntry} has returned for this entry — the entry alone is not proof the write
824
+ * is saved.
825
+ *
826
+ * `addActions` wrote exactly the snapshot pending list under this action id, and the entry's
827
+ * actions are therefore the LEADING `entry.actions.length` items of `this.pending` — anything
828
+ * staged since is behind them, because {@link actInternal} appends. Under
829
+ * {@link syncInternal} nothing can even be staged mid-cycle (`act()` shares the collection
830
+ * latch); under `TransactionCoordinator.commit` the mark spans a latch-free inter-attempt
831
+ * window, so an `act()` there CAN grow `pending` — still only at the tail, so the slice stays
832
+ * right.
833
+ *
834
+ * The guard below is the load-bearing part: `slice` fails SILENTLY if that correspondence ever
835
+ * breaks, dropping actions that were never committed, so an entry longer than `pending` throws
836
+ * instead of losing work. (See the sibling note on `syncInternal`'s post-commit replay, which
837
+ * rests on the same invariant.) */
838
+ private consumeOwnEntry(entry: ActionEntry<Action<TAction>>) {
839
+ if (entry.actions.length > this.pending.length) {
840
+ throw new Error(
841
+ `Collection ${this.id}: own committed entry for action ${entry.actionId} holds `
842
+ + `${entry.actions.length} actions but only ${this.pending.length} are pending; `
843
+ + `consuming it would drop actions that were never committed`);
844
+ }
845
+ // `mutated` is unconditional, even for a zero-action entry: the tracker still holds this
846
+ // action's staged transforms, and only the replay at the end of `updateInternal` — which
847
+ // resets the tracker and re-stages just what remains — drops them. That reset is what turns
848
+ // `hasUnsyncedChanges()` false so the sync loop exits reporting the success the writer is
849
+ // owed (the action IS durable).
850
+ return { after: this.pending.slice(entry.actions.length), mutated: true };
851
+ }
852
+
853
+ /** Maps each pending action to its effective form against a remote entry: the original, a
854
+ * replacement, or dropped. A replacement or a discard changes the pending set; the tracker still
855
+ * holds the pre-filter transforms, so report it as mutated to force a replay that re-stages
856
+ * against the effective actions. Identity comparison per the contract: keep => same instance,
857
+ * replace => new instance.
858
+ * NOTE: a filterConflict hook that always allocates a fresh (but equal) instance instead of
859
+ * returning the same one forces a replay on every update — if that ever shows up as a hot path,
860
+ * compare by value/id here instead of by reference. */
861
+ private filterAgainstEntry(entry: ActionEntry<Action<TAction>>) {
862
+ const before = this.pending;
863
+ const after = before
864
+ .map(p => this.doFilterConflict(p, entry.actions))
865
+ .filter((a): a is Action<TAction> => a !== undefined);
866
+ return { after, mutated: after.length !== before.length || after.some((a, i) => a !== before[i]) };
867
+ }
868
+
869
+ /** Refresh this instance against the stored log: adopt the latest committed revision, resolve
870
+ * pending actions against everything that landed since, and replay them if anything conflicts.
871
+ *
872
+ * Takes no in-flight action id — it reads {@link inFlightActionId} off `this`, which is set for
873
+ * exactly the write attempt cycles that own one (see that field). Callers cannot get this wrong
874
+ * by omission.
875
+ *
876
+ * @param report - Filled in as the refresh goes (see {@link RefreshReport}): its
877
+ * `ownEntryFinished` is set the moment the refresh has found this write's own log entry and
878
+ * finished it ({@link completeOwnEntry}) — before the entry is consumed, so a later throw from
879
+ * here still leaves it set. Never set on a reader's refresh. A caller with no use for it passes `{}`.
880
+ * @param lastChance - Whether the write in flight will get no further refresh (see
881
+ * {@link completeOwnEntry}). Meaningless, and left false, on a reader's refresh.
882
+ * @throws TornActionError when it found that entry and could not finish the action — thrown
883
+ * before anything on this instance changed, and with `report` untouched. */
884
+ private async updateInternal(report: RefreshReport, lastChance = false): Promise<void> {
885
+ // Start with a context that can see to the end of the log
886
+ const source = new TransactorSource(this.id, this.transactor, undefined);
887
+
888
+ // A header the storage layer could not retrieve throws BlockUnavailableError out of
889
+ // this read (it is not a StaleFailure, so sync's retry loop does not absorb it).
890
+ const ends = await Collection.readLogEnds(this.transactor, this.id, this.logTailId);
891
+ if (!ends) {
892
+ if (this.source.actionContext) {
893
+ // An absent header is only believable for a collection that has never committed.
894
+ // We hold a committed revision, so the two answers contradict each other — surface it
895
+ // as a fault instead of no-opping into a forgotten revision and a rev-1 retry spin.
896
+ // NOTE: this aborts every caller of update(), including TransactionCoordinator's
897
+ // blanket refresh of ALL registered collections between commit retries — a
898
+ // non-participant with a momentarily-absent header now fails the whole retry rather
899
+ // than being skipped (the coordinator still refreshes the remaining collections first,
900
+ // so a participant that can be finished is). That is the intended loud failure; if it ever shows up as
901
+ // otherwise-healthy transactions aborting, narrow that refresh to the transaction's
902
+ // participants (see the note at coordinator.ts's update loop) rather than softening
903
+ // this throw.
904
+ throw new CollectionHeaderVanishedError(this.id, this.source.actionContext.rev);
905
+ }
906
+ // The header is genuinely absent AND we hold no revision: nothing was ever committed under
907
+ // this id, so there is no log to walk and nothing to adopt — correct here, rather than a
908
+ // masked failure.
909
+ return;
910
+ }
911
+ this.logTailId = ends.header.tailId;
912
+ // Bootstrap context from committed tail so pending blocks are accessible.
913
+ Collection.bootstrapContext(source, ends.tail);
914
+
915
+ // The revision the committed tail just claimed, captured before anything else can touch
916
+ // the local source. This is the authoritative "latest committed under this id" number,
917
+ // read straight off the tail block's state; the chain walk below arrives at its own
918
+ // number by a different path, and the two disagreeing is worth saying out loud (see the
919
+ // {@link reportShortfall} call after advanceContext). Stays undefined when there is no
920
+ // tail, or a tail with no `latest` — both legitimate "nothing committed yet" states.
921
+ const tailRev = source.actionContext?.rev;
922
+
923
+ const actionContext = this.source.actionContext;
924
+ // A write's retry refresh always walks. Not needed for soundness — the write's own entry would
925
+ // sit above the held revision, which the tail test already refuses — but losing that entry
926
+ // loses the write, and a retry refresh is rare enough that the walk costs nothing that matters.
927
+ if (this.inFlightActionId === undefined && Collection.tailShowsNothingNewer(actionContext, ends.tail)) {
928
+ return;
929
+ }
930
+
931
+ // One block cache for the whole refresh, seeded with the header and tail just read: Log.open,
932
+ // the entry walk and the invalidation walk each start again from the header and the tail, and
933
+ // would otherwise fetch both every time. The seed came from UNPINNED reads and is served to
934
+ // reads pinned at the tail's claim, which is sound: the tail's content at its own claimed
935
+ // revision is that pinned view, and the header changes only when the tail block fills, which
936
+ // Chain.getTail already tolerates by following `nextId` from whichever tail the header names.
937
+ // NOTE: the cache keeps its default size (128 blocks, about 4,000 log entries). The entry walk
938
+ // reads every log block back to the head (no checkpoints), newest first, so past that size
939
+ // the newest blocks are evicted first and the invalidation walk fetches them again. If logs
940
+ // get that long before checkpoints land, walk entries and invalidations in one pass.
941
+ const tracker = new Tracker(new CacheSource(source, undefined, undefined, ends.served));
942
+
943
+ // Get the latest entries from the log, starting from where we left off
944
+ const collectionLog = await Log.open<Action<TAction>>(tracker, this.id);
945
+ const latest = collectionLog ? await collectionLog.getFrom(actionContext?.rev ?? 0) : undefined;
946
+
947
+ // This write's own entry, if its log tail landed despite the failure that sent us here.
948
+ // Decided ONCE, here: the mark can be cleared latch-free while the completion below is
949
+ // awaiting (see {@link inFlightActionId}), and an entry that was finished as our own must
950
+ // not then be run through the conflict filter as a stranger's and replayed.
951
+ const inFlightActionId = this.inFlightActionId;
952
+ const ownEntry = inFlightActionId === undefined
953
+ ? undefined
954
+ : latest?.entries.find(entry => entry.actionId === inFlightActionId);
955
+ // Finish it BEFORE anything below changes this instance — see {@link completeOwnEntry}. A
956
+ // throw from here (the action cannot be finished, or not yet) therefore abandons the refresh
957
+ // with the staged actions, the tracker and the held revision exactly as they were.
958
+ // The entry's revision comes from the context the same walk built; an entry older than a
959
+ // checkpoint is not restated there, and completeOwnEntry falls back to the attempt's own.
960
+ const entryRevs = Collection.revisionsByAction(latest?.context);
961
+ if (ownEntry !== undefined) {
962
+ const durability = await this.completeOwnEntry(ownEntry, entryRevs.get(ownEntry.actionId), lastChance);
963
+ // Saved from here on, whatever below throws — record it before anything else can.
964
+ report.ownEntryFinished = { durability };
965
+ }
966
+
967
+ // Process the entries and track the blocks they affect
968
+ let anyConflicts = false;
969
+ for (const entry of latest?.entries ?? []) {
970
+ const isOwnEntry = entry === ownEntry;
971
+ const { after, mutated } = isOwnEntry
972
+ ? this.consumeOwnEntry(entry)
973
+ : this.filterAgainstEntry(entry);
974
+ this.pending = after;
975
+ anyConflicts = anyConflicts || mutated || this.tracker.conflicts(new Set(entry.blockIds)).length > 0;
976
+ }
977
+
978
+ // React to durable invalidations that landed since we last synced. getFrom intentionally skips
979
+ // invalidation entries (they are not pending/committed actions), so surface them separately: an
980
+ // invalidation reverted committed content this client may have read, so treat it like a stale
981
+ // read — drop the reverted blocks from the read cache and replay pending work against the reverted
982
+ // base (docs/right-is-right.md §Client notification). De-duped across cascade children by reverted
983
+ // block; over-inclusive by design (over-invalidation just resubmits — it never wrongly retains).
984
+ const invalidations = collectionLog ? await collectionLog.getInvalidationsFrom(actionContext?.rev ?? 0) : [];
985
+ const revertedBlockIds = [...new Set(invalidations.flatMap(inv => inv.reverted.map(r => r.blockId)))];
986
+ if (invalidations.length > 0 && this.pending.length > 0) {
987
+ anyConflicts = true;
988
+ }
989
+
990
+ this.forgetAndAdopt(latest, entryRevs, revertedBlockIds, ends.served);
991
+
992
+ Collection.reportShortfall(this.id, this.instanceTag, tailRev, actionContext?.rev, this.source.actionContext?.rev);
993
+
994
+ // Re-stage the pending actions against the adopted revision. The affected
995
+ // blocks were already dropped from sourceCache above (per log entry / per invalidation),
996
+ // so the replay's reads re-materialize from the transactor.
997
+ // NOTE: a throw out of replayActions leaves the tracker holding only the transforms
998
+ // replayed so far while `pending` still lists them all; the caller's error handling is
999
+ // expected to abort/reset the collection rather than keep staging. The coordinator's partial
1000
+ // report after a refresh-saved sibling does not (backlog: debt-a-failed-refresh-can-leave-a-
1001
+ // collection-half-restaged). If replay ever gains a
1002
+ // routinely-throwing read path, rebuild into a scratch tracker and swap on success.
1003
+ if (this.mustReplay(anyConflicts, actionContext)) {
1004
+ await this.replayActions();
1005
+ }
1006
+ }
1007
+
1008
+ /** Forget every block the refresh saw change, floor the ones a log entry names, adopt the
1009
+ * revision the log is at, and keep what the refresh already read
1010
+ * ({@link keepWhatTheRefreshRead}) — ONE synchronous step, which must stay free of any `await`.
1011
+ *
1012
+ * Reads are not latched, so one can run while a refresh is under way. Forgetting a block while
1013
+ * this handle still reads at the revision it is LEAVING invites exactly the wrong re-read: the
1014
+ * block comes back as it was at that revision (correctly — that is what was asked for, and no
1015
+ * floor applies to a read below it), the cache keeps it, and the advance that follows turns it
1016
+ * into old content nothing will ever clear, the entry that named it having been consumed. That
1017
+ * needs no lagging machine; storage can be perfectly current. With no gap between the forgetting
1018
+ * and the adopting, a concurrent read either lands before both and is forgotten with the rest,
1019
+ * or lands after both and is judged at the adopted revision — against the floor
1020
+ * ({@link TransactorSource.tryGet} reads its context when the answer arrives, not when it was
1021
+ * asked for) and against the generation the clear moved (`CacheSource.stillWanted`).
1022
+ *
1023
+ * The advance is monotonic (see {@link advanceContext}): an empty or unopenable log yields no
1024
+ * context at all, and a log read that lags what this handle already committed yields an older
1025
+ * one; neither is grounds for forgetting the revision held. It also has to precede
1026
+ * {@link replayActions}, which re-reads blocks through `this.source` at whatever revision the
1027
+ * context names — replaying first would refill the cache at the revision being left. */
1028
+ private forgetAndAdopt(
1029
+ latest: GetFromResult<Action<TAction>> | undefined,
1030
+ entryRevs: ReadonlyMap<ActionId, number>,
1031
+ revertedBlockIds: BlockId[],
1032
+ served: LogEnds['served'],
1033
+ ): void {
1034
+ for (const entry of latest?.entries ?? []) {
1035
+ this.sourceCache.clear(entry.blockIds);
1036
+ this.raiseFloors(entry, entryRevs.get(entry.actionId));
1037
+ }
1038
+ this.sourceCache.clear(revertedBlockIds);
1039
+ Collection.advanceContext(this.source, this.id, this.instanceTag, 'refresh', latest?.context);
1040
+ this.keepWhatTheRefreshRead(served);
1041
+ }
1042
+
1043
+ /** Put the header and log tail block {@link readLogEnds} already fetched into this handle's own
1044
+ * cache, so the next read of either is a hit rather than a second fetch of a block just
1045
+ * received. Without it, a write that follows ANOTHER handle's commit fetches the log tail twice:
1046
+ * the refresh reads it, the walked entry names it (a commit's blocks include the log block its
1047
+ * entry was appended to) so the clear above drops it, and `Chain.add` then reads it again to
1048
+ * append this write's own entry.
1049
+ *
1050
+ * Last in {@link forgetAndAdopt}'s one synchronous step, after BOTH clears — offering before
1051
+ * either would simply be undone by it. An id that an invalidation reverted is therefore
1052
+ * re-offered rather than left forgotten, which is sound because the revert is itself a commit
1053
+ * and {@link readLogEnds} read unpinned: what it read already includes the revert.
1054
+ *
1055
+ * An offer is made only when all three hold:
1056
+ * - **This handle holds a context.** With none adopted, every later read is unpinned and may
1057
+ * legitimately be answered newer than what was read here, so there is nothing to judge the
1058
+ * offer against.
1059
+ * - **The served revision is at or below the adopted one.** {@link readLogEnds} reads unpinned,
1060
+ * so it can come back newer than the view this handle now reads at — exactly the
1061
+ * `collection:context-short-of-tail` case. Serving that to a read pinned lower would fabricate
1062
+ * a view that never existed. At or below the pin it is sound: a block whose newest content is
1063
+ * at `rev <= context.rev` has the same content at `context.rev`.
1064
+ * - **The served revision meets any applicable floor.** A cache must never keep a below-floor
1065
+ * answer.
1066
+ *
1067
+ * NOTE: the floor check lives here because {@link readLogEnds} reads around
1068
+ * {@link TransactorSource}, which is what applies floors on every other read — nothing else on
1069
+ * this path would apply one. It asks {@link BlockFloors.meetsApplicableFloor} rather than
1070
+ * spelling the comparison out, so it cannot drift from the one `answeredBelowFloor` makes, and
1071
+ * asks the non-reporting face because this is not an answer being served to a reader — the
1072
+ * `collection:block-below-floor` line would misdescribe it.
1073
+ *
1074
+ * Only this site seeds. The early return at {@link tailShowsNothingNewer} is deliberately left
1075
+ * alone: it fires when the log has not moved, and the cache is then already warm with the tail
1076
+ * (folded in by this handle's own commit, or kept from its last walk), so seeding there would
1077
+ * buy a fetch back only when the LRU had evicted it — at the cost of putting the one-synchronous-
1078
+ * step reasoning in a second place. */
1079
+ // NOTE: an offer whose revision equals what the cache already holds is still kept, so a walking
1080
+ // refresh bumps the header's generation even when the header was neither cleared nor changed.
1081
+ // Over-bumping is safe (see `CacheSource.bump`) and costs a re-materialize of that block on its
1082
+ // next read; if that ever shows up, skip the offer when the cache already RETAINS the id at the
1083
+ // same revision — but note it would then also stop correcting a cached block whose content was
1084
+ // folded forward locally at that revision.
1085
+ private keepWhatTheRefreshRead(served: LogEnds['served']): void {
1086
+ const context = this.source.actionContext;
1087
+ if (context === undefined) {
1088
+ return;
1089
+ }
1090
+ for (const [blockId, block, rev] of served) {
1091
+ if (rev <= context.rev && this.floors.meetsApplicableFloor(blockId, context, rev)) {
1092
+ this.sourceCache.offerServed(blockId, block, rev);
1093
+ }
1094
+ }
1095
+ }
1096
+
1097
+ /** The revision each action in `context` committed at, keyed by action id. `Log.getFrom` returns
1098
+ * entries without their revisions; the context the same walk built is where they are restated. */
1099
+ private static revisionsByAction(context: ActionContext | undefined): Map<ActionId, number> {
1100
+ return new Map((context?.committed ?? []).map(({ actionId, rev }) => [actionId, rev]));
1101
+ }
1102
+
1103
+ /** Records that the walked `entry` changed the blocks it names, so a later answer for one of
1104
+ * them that is older than the entry is recognised and never remembered (see {@link BlockFloors}).
1105
+ * Beside the cache clear on purpose (see {@link forgetAndAdopt}): the clear is what sends the next
1106
+ * read of these blocks back to storage, and that read is the one a lagging machine can answer too
1107
+ * old — after which nothing would clear the block again, this entry having been consumed.
1108
+ *
1109
+ * NOTE: an entry whose revision the walk did not restate sets no floor. That is an entry older
1110
+ * than the log's most recent checkpoint, and no checkpoint is written today
1111
+ * (tickets/backlog/debt-the-collection-log-never-writes-a-checkpoint). Once they are, such an
1112
+ * entry's blocks go unguarded unless `Log.getFrom` starts returning each entry's revision. */
1113
+ private raiseFloors(entry: ActionEntry<Action<TAction>>, rev: number | undefined): void {
1114
+ if (rev !== undefined) {
1115
+ this.floors.raise(entry.blockIds, { rev, actionId: entry.actionId });
1116
+ }
1117
+ }
1118
+
1119
+ /** Whether the log tail a refresh just read proves the log holds nothing newer than `held` —
1120
+ * the test that lets a refresh with nothing to find stop after one request instead of walking
1121
+ * the log.
1122
+ *
1123
+ * Sound because every commit and every invalidation appends a log entry, and entries only ever
1124
+ * go on the tail block. A tail block that is still the end of the chain (no `nextId`), whose
1125
+ * newest entry is at the held revision and names the action `held` names there — and whose own
1126
+ * claim (`state.latest`) says the same — has nothing above `held`. The walk would then find no
1127
+ * entries, no invalidations, and a context {@link advanceContext} adopts at an unchanged
1128
+ * revision, so it would change nothing.
1129
+ *
1130
+ * Says "no" — sending the refresh down the full walk — whenever the tail and the held context
1131
+ * disagree in any way, so everything that walk reports still fires:
1132
+ * - a claim above `held` (ordinary catch-up) or below it (a lagging read);
1133
+ * - a newest entry that is not the claimed action, or not an action at all: the entries lag the
1134
+ * claim (`collection:context-short-of-tail`, or `collection:context-not-lowered` for a handle
1135
+ * pinned by an over-claiming tail), or an invalidation or checkpoint took the newest slot;
1136
+ * - any action entry in the tail block naming a different action than `held` names at the same
1137
+ * revision (`collection:lineage-divergence`, and the walk's adoption of the log's list).
1138
+ *
1139
+ * NOTE: the lineage comparison sees only the entries the tail block holds. A fork below them, on
1140
+ * a log that has not moved, is not looked at again until a refresh finds something new — that
1141
+ * walk compares the whole list. Walking every time to look is exactly the cost this avoids.
1142
+ *
1143
+ * NOTE: an invalidation entry in the newest slot never matches (it names no action), so every
1144
+ * refresh after one walks the log until the next commit lands. Fine while disputes are rare; if
1145
+ * they are not, match an invalidation slot against the held revision too. */
1146
+ private static tailShowsNothingNewer(held: ActionContext | undefined, tail: GetBlockResult | undefined): boolean {
1147
+ const claim = tail?.state.latest;
1148
+ const block = tail?.block as LogBlock<unknown> | undefined;
1149
+ if (held === undefined || claim === undefined || !block || block.nextId !== undefined) {
1150
+ return false;
1151
+ }
1152
+ const newest = block.entries[block.entries.length - 1];
1153
+ return claim.rev === held.rev
1154
+ && newest?.rev === claim.rev
1155
+ && newest.action?.actionId === claim.actionId
1156
+ && actionIdAt(held, held.rev) === claim.actionId
1157
+ && !Collection.disagreesWithHeld(held, block.entries);
1158
+ }
1159
+
1160
+ /** Whether any action entry in `entries` names a different action than `held` names at the same
1161
+ * revision. A revision only one side names is missing evidence, not disagreement — the rule
1162
+ * {@link earliestFork} applies too. */
1163
+ private static disagreesWithHeld(held: ActionContext, entries: readonly LogEntry<unknown>[]): boolean {
1164
+ const logged = new Map(entries.flatMap(entry => entry.action ? [[entry.rev, entry.action.actionId] as const] : []));
1165
+ return held.committed.some(entry => {
1166
+ const loggedAction = logged.get(entry.rev);
1167
+ return loggedAction !== undefined && loggedAction !== entry.actionId;
1168
+ });
1169
+ }
1170
+
1171
+ /** Whether {@link updateInternal} must re-stage `pending` after adopting `latest`, given the
1172
+ * context it held BEFORE the refresh.
1173
+ *
1174
+ * The answer is deliberately NOT just "a conflict was found". A conflict is detected by an
1175
+ * incoming log entry naming a block this tracker already holds a transform for — so an action
1176
+ * that changed NO block can never register one. A staged delete of a key this instance cannot
1177
+ * see is exactly that action: the tree's `replace` handler misses on `find` and `deleteAt`
1178
+ * returns false without writing. Gated on conflicts alone, such an action stayed in `pending`
1179
+ * unapplied until the commit wrote a log entry listing it whose transforms did nothing — and
1180
+ * readers materialize blocks, not log entries, so the action was lost on every node, silently
1181
+ * and permanently. The invariant that has to hold is that **a pending action was applied
1182
+ * against the revision it commits over**, so a mere revision advance is reason enough.
1183
+ *
1184
+ * The advance test rests on {@link advanceContext} being monotonic (it refuses to lower), which
1185
+ * is what makes "the adopted rev differs from the held one" mean "it went up".
1186
+ *
1187
+ * The second conjunct is `pending.length` and deliberately NOT {@link hasUnsyncedChanges},
1188
+ * which also counts tracker transforms. A collection {@link createOrOpen} just INVENTED holds
1189
+ * its staged header/root in the tracker with NO pending action naming them, and
1190
+ * {@link replayActions} resets the tracker before re-staging — so counting transforms here
1191
+ * would drop those blocks and leave a brand-new collection unreadable (the same hazard
1192
+ * {@link snapshotPending} documents). Nothing to re-stage means nothing to replay.
1193
+ *
1194
+ * NOTE: this makes a refresh that adopts a newer revision O(pending) rather than free, so a
1195
+ * read taken mid-transaction while a rival keeps committing re-stages every action staged so
1196
+ * far, on each such read. Measured as no change to the storage-op budgets
1197
+ * (`index-backfill-cost.spec.ts`, `cold-apply-cost.spec.ts`), because the replay's reads are
1198
+ * served from `sourceCache`. If a long transaction's reads ever show up as slow, narrow the
1199
+ * replay to the actions whose reads the adopted entries actually invalidated — NOT back to
1200
+ * conflicts alone, which is the defect above. */
1201
+ private mustReplay(anyConflicts: boolean, priorContext: ActionContext | undefined): boolean {
1202
+ if (anyConflicts) {
1203
+ return true;
1204
+ }
1205
+ const adoptedRev = this.source.actionContext?.rev;
1206
+ if (adoptedRev !== undefined && adoptedRev !== priorContext?.rev && this.pending.length > 0) {
1207
+ return true;
1208
+ }
1209
+ // Third reason: a staged block's base has MOVED — the read cache no longer describes it at
1210
+ // the revision its operations were computed against (see `Tracker.movedBases`). A refresh
1211
+ // that adopted nothing and found no conflict can still be standing over such a block: a
1212
+ // concurrent read served content the cache then kept at a newer revision, say. Re-staging
1213
+ // is the only repair; declaring the pinned revision on the pend would merely get it refused.
1214
+ return this.pending.length > 0 && this.logMovedBases(this.tracker.movedBases()) > 0;
1215
+ }
1216
+
1217
+ /** Re-judge every staged update's pinned base against the read cache and re-stage the pending
1218
+ * queue if any has moved. Runs immediately before each pend attempt — {@link syncAttempts} and
1219
+ * the coordinator's commit span both call it — because that is the last moment before the
1220
+ * base the pend declares is put on the wire, and nothing between a refused attempt and the
1221
+ * re-pend reads the block again (the refresh may have moved nothing and so replayed nothing).
1222
+ *
1223
+ * Two steps. First, one read for each pinned block the cache does NOT retain — content handed
1224
+ * through unkept (a below-floor answer, which the cache re-asks for on every read anyway): the
1225
+ * pin names the revision last served, storage may have caught up since without the log moving,
1226
+ * and only a read can tell. An evicted-but-kept base needs no read: its content met every floor
1227
+ * the handle knows, so storage cannot have moved it without a log entry the next refresh will
1228
+ * walk (and the pend would be refused as stale on revision alone). Then, one cache probe per
1229
+ * pinned block ({@link Tracker.movedBases}) and a replay if any moved.
1230
+ *
1231
+ * Latch-free by contract, like {@link snapshotPending}: the caller holds this instance's latch
1232
+ * ({@link replayActions} is always run under it).
1233
+ *
1234
+ * NOTE: reads are not latched, so a concurrent read can still move a base between this check
1235
+ * and the pend. The pend then declares the pin's (old) revision, storage refuses it, and the
1236
+ * next attempt's call here re-stages — one wasted round trip, never a wrong base. If that
1237
+ * refusal ever shows up in practice, the closure is to hold the pend under the read latch.
1238
+ *
1239
+ * @returns whether the pending queue was re-staged. */
1240
+ async restageIfBasesMoved(): Promise<boolean> {
1241
+ for (const id of this.tracker.unretainedBases()) {
1242
+ // `navigation` never upgrades the purpose the original read recorded (value-wins), and
1243
+ // the revision recorded is the one returned, which the replay below re-records anyway.
1244
+ await this.sourceCache.tryGet(id, 'navigation');
1245
+ }
1246
+ if (this.pending.length === 0 || this.logMovedBases(this.tracker.movedBases()) === 0) {
1247
+ return false;
1248
+ }
1249
+ await this.replayActions();
1250
+ return true;
1251
+ }
1252
+
1253
+ /** Report each moved base, naming the revision the operations were computed against and the
1254
+ * one the cache describes now, and return how many there were. */
1255
+ private logMovedBases(moved: readonly BlockId[]): number {
1256
+ if (log.enabled) {
1257
+ for (const blockId of moved) {
1258
+ log('collection:restage-moved-base id=%s tag=%s block=%s pinnedRev=%s currentRev=%s',
1259
+ this.id, this.instanceTag, blockId,
1260
+ this.tracker.stagedBaseRevs([blockId])[blockId] ?? 'none',
1261
+ this.sourceCache.getCachedRevision(blockId) ?? 'none');
1262
+ }
1263
+ }
1264
+ return moved.length;
1265
+ }
1266
+
1267
+ /** Capture the current staged state — tracker transforms plus the pending
1268
+ * action queue — so it can be restored later via {@link restorePending}.
1269
+ *
1270
+ * Use to bracket a unit of staged DML that may need to be rolled back. Unlike
1271
+ * a blanket "reset to empty", restoring this snapshot preserves any structural
1272
+ * baseline that predates the staged DML — most importantly a brand-new
1273
+ * collection's header/root blocks, which live in the tracker (uncommitted)
1274
+ * until the first sync. Resetting such a collection to empty would leave it
1275
+ * unreadable; restoring the snapshot returns it to its prior (readable) state.
1276
+ *
1277
+ * The returned snapshot is deep-cloned and independent of subsequent mutations.
1278
+ * Synchronous and latch-free: intended to bracket transaction-scoped staging,
1279
+ * when no concurrent act/sync is in flight. */
1280
+ snapshotPending(): CollectionSnapshot<TAction> {
1281
+ return {
1282
+ transforms: copyTransforms(this.tracker.transforms),
1283
+ pins: this.tracker.pins.copy(),
1284
+ pending: [...this.pending],
1285
+ context: structuredClone(this.source.actionContext),
1286
+ };
1287
+ }
1288
+
1289
+ /** Restore the staged state captured by {@link snapshotPending}, discarding any
1290
+ * mutations staged since. Reads through the collection then observe exactly the
1291
+ * snapshot state again; storage is untouched because nothing was ever synced.
1292
+ *
1293
+ * A snapshot is only restorable VERBATIM onto the committed boundary it was captured
1294
+ * on. If this collection has since ADOPTED a newer committed revision — a rival's
1295
+ * commit folded in by a refresh while the snapshot's transaction was in flight, e.g.
1296
+ * the conflict replay that refused a guarded insert (TreeKeyTakenError) — the
1297
+ * snapshot's transforms describe block state at the OLD boundary, and reinstalling
1298
+ * them would shadow committed blocks with stale structure. The observed case: an
1299
+ * INVENTED collection's pre-commit header/root transforms restored over the rival's
1300
+ * now-committed collection make every later read descend an empty tree, silently
1301
+ * hiding the committed rows. When the snapshot's pending queue is empty (the
1302
+ * transaction-rollback shape: the capture predates the transaction's first stage),
1303
+ * the correct restore target IS the committed state — reset the tracker empty and
1304
+ * let reads flow through to the adopted revision.
1305
+ *
1306
+ * NOTE: a snapshot that carries PENDING actions across a moved boundary (a
1307
+ * mid-transaction savepoint captured before a mid-transaction refresh adopted a
1308
+ * rival's commit) still restores verbatim below — rebasing it would require an async
1309
+ * replay this synchronous method cannot run. That shape predates this guard and
1310
+ * keeps its old behaviour; if it is ever observed producing stale reads, the rebase
1311
+ * belongs in an async caller that can replay the pending queue (see replayActions).
1312
+ * The WRITE side of that shape is closed here: the snapshot's base pins are restored with
1313
+ * its transforms, so the restored operations name the bases they were computed on, and the
1314
+ * pre-pend re-validation (restageIfBasesMoved) finds those bases moved and re-stages. */
1315
+ restorePending(snapshot: CollectionSnapshot<TAction>): void {
1316
+ const capturedRev = snapshot.context?.rev;
1317
+ const currentRev = this.source.actionContext?.rev;
1318
+ const boundaryMoved = currentRev !== undefined && (capturedRev === undefined || currentRev > capturedRev);
1319
+ if (boundaryMoved && snapshot.pending.length === 0) {
1320
+ this.tracker.reset();
1321
+ this.pending = [];
1322
+ return;
1323
+ }
1324
+ this.tracker.reset(copyTransforms(snapshot.transforms));
1325
+ if (snapshot.pins) this.tracker.pins.replaceWith(snapshot.pins);
1326
+ this.pending = [...snapshot.pending];
1327
+ }
1328
+
1329
+ /** A read-only {@link Tracker} pinned to this collection's committed state AS OF the
1330
+ * moment of this call, seeded with a (deep-copied) set of pre-transaction transforms.
1331
+ * Reads through it observe exactly that revision plus exactly those transforms — NOT
1332
+ * the mutations staged into this collection's live tracker afterward, and NOT commits
1333
+ * that fold into (or clear) the live read cache while the view is being walked. Used
1334
+ * to build a committed read view (see {@link Tree.readView}) that a scan can trust
1335
+ * from first row to last.
1336
+ *
1337
+ * The pinning has three legs, built in ONE synchronous block so they all describe the
1338
+ * same instant (do not introduce an await between them):
1339
+ * - a private {@link TransactorSource} whose action context is a deep copy FROZEN at
1340
+ * view-creation time, so a block first read after a later commit still materializes
1341
+ * at the pinned revision (the transactor honours `context.rev` on get);
1342
+ * - a private {@link CacheSource} nothing else references, so the live collection's
1343
+ * `transformCache`/`clear` cannot reach it;
1344
+ * - that private cache is seeded from the shared cache's current entries, so the
1345
+ * common committed read (deferred CHECK over a warm cache) stays warm instead of
1346
+ * refetching every block over the network.
1347
+ *
1348
+ * By default the view records NO read dependencies — it is not part of any
1349
+ * transaction's conflict set (see {@link ReadViewOptions.recordReads}).
1350
+ *
1351
+ * NOTE: each view holds up to the cache LRU budget (128) of cloned blocks plus
1352
+ * whatever it faults in. Views are per-scan and dropped when the scan ends; a very
1353
+ * long-lived committed scan pins that much memory.
1354
+ *
1355
+ * NOTE: an INVENTED collection (createOrOpen found no header, so `actionContext` is
1356
+ * undefined) pins to no revision — its private source asks the transactor for the
1357
+ * latest. Harmless today because such a collection's blocks all live in the tracker
1358
+ * transforms, so a view never reaches storage; if invented collections ever gain
1359
+ * committed blocks not covered by their transforms, this view would follow storage
1360
+ * forward instead of staying pinned.
1361
+ *
1362
+ * When {@link ReadViewOptions.pinContext} is supplied, the view pins to THAT
1363
+ * boundary instead of the current context, and cache entries committed at a newer
1364
+ * revision are dropped from the seed (they would otherwise be served blindly — the
1365
+ * cache never checks a hit against the requested context). Dropped entries are
1366
+ * refetched from the transactor, which resolves the highest committed revision at
1367
+ * or below the pin. This is what lets a snapshot captured BEFORE a commit still
1368
+ * yield a coherent pre-commit view AFTER that commit folded into the shared cache
1369
+ * (a mid-sweep multi-tree commit being the motivating case). */
1370
+ createReadTracker(transforms: Transforms, options?: ReadViewOptions): Tracker<IBlock> {
1371
+ const collector = options?.recordReads ? this.source.getCollector() : undefined;
1372
+ const pinContext = options?.pinContext ?? this.source.actionContext;
1373
+ const pinRev = options?.pinContext?.rev;
1374
+ let seed = this.sourceCache.snapshotEntries();
1375
+ if (pinRev !== undefined) {
1376
+ // NOTE: entries whose revision the cache never learned read as 0 and pass this
1377
+ // filter. Committed blocks always carry a real revision (the transactor reports
1378
+ // it on load; transformCache stamps it on fold), so a rev-0 entry newer than the
1379
+ // pin does not occur on the paths that reach here.
1380
+ seed = seed.filter(([, , revision]) => revision <= pinRev);
1381
+ }
1382
+ // The view shares this handle's floors, so a block the last refresh saw change is not kept
1383
+ // too old by the view either. A view pinned below a floor is untouched by it.
1384
+ const pinnedSource = new TransactorSource<IBlock>(
1385
+ this.id, this.transactor, structuredClone(pinContext), collector, this.floors);
1386
+ const pinnedCache = new CacheSource<IBlock>(
1387
+ pinnedSource, undefined, collector, seed);
1388
+ return new Tracker(pinnedCache, copyTransforms(transforms));
1389
+ }
1390
+
1391
+ /** The staged (not-yet-synced) actions queued by {@link act}.
1392
+ *
1393
+ * Exposed so a {@link TransactionCoordinator} can append them to the log at
1394
+ * commit time — mirroring what {@link sync} does internally — when the actions
1395
+ * were staged directly into this collection (e.g. through a Tree's stage())
1396
+ * rather than applied via the coordinator's own action path. */
1397
+ getPendingActions(): Action<TAction>[] {
1398
+ return this.pending;
1399
+ }
1400
+
1401
+ /** Drop the staged actions after they have been committed through a
1402
+ * coordinator. Counterpart to {@link getPendingActions}; {@link sync} clears
1403
+ * its own pending inline, so this is only needed when commit was orchestrated
1404
+ * externally. */
1405
+ clearPendingActions(): void {
1406
+ this.pending = [];
1407
+ }
1408
+
1409
+ /** Whether {@link sync} has anything to push: staged actions, or tracker transforms
1410
+ * that were never committed (an INVENTED collection's header/root blocks live there
1411
+ * until its first sync, with no pending action to name them).
1412
+ *
1413
+ * This is the exact predicate {@link syncInternal} loops on, so `false` means a sync
1414
+ * would commit nothing. Exposed so a caller that would otherwise flush
1415
+ * unconditionally can skip the round trip — note that {@link Tree.sync} routes
1416
+ * through {@link updateAndSync}, so an unnecessary flush still pays a full,
1417
+ * cache-bypassing {@link update} before discovering it has nothing to do. */
1418
+ hasUnsyncedChanges(): boolean {
1419
+ return this.pending.length > 0 || !isTransformsEmpty(this.tracker.transforms);
1420
+ }
1421
+
1422
+ /** The committed revision this collection currently READS at, or `undefined` for an
1423
+ * INVENTED collection that has never adopted a committed revision
1424
+ * ({@link createOrOpen} found no header and staged a fresh empty one).
1425
+ *
1426
+ * Not the revision a pending write will land at: {@link getNextRev} is this plus one
1427
+ * (`undefined` counting as 0), so a diagnostic that prints this value BEFORE a commit
1428
+ * is naming the revision the commit will supersede, not the one it produces.
1429
+ *
1430
+ * DIAGNOSTIC ONLY — do not branch on this. Every block this collection reads is
1431
+ * materialized at this revision ({@link TransactorSource.tryGet} passes it as the
1432
+ * read context), and the revision advances ONLY through an explicit call on THIS
1433
+ * instance — {@link update} or {@link sync} on the single-node path, or
1434
+ * {@link recordCommitted} when a {@link TransactionCoordinator} commits this
1435
+ * collection (the session/consensus path, where no `update()` is involved at all).
1436
+ * Nothing moves it passively — not time, not another collection's
1437
+ * commit, not a peer's notification. So a collection sitting here at a lagging
1438
+ * revision silently serves an old root with no error, and two collections in one
1439
+ * process can be at different revisions at the same instant. That gap is invisible
1440
+ * from outside the class without this accessor, which is the whole reason it
1441
+ * exists: `docs/debugging.md` (§ "Which revision did a read descend?") explains
1442
+ * how an operator reads the difference.
1443
+ *
1444
+ * The one exception is `undefined` itself, which is a STATE rather than a revision:
1445
+ * "this instance invented the collection and has never adopted a committed revision".
1446
+ * For the same reason (only this instance moves it), it stays true until this instance
1447
+ * updates, syncs or records a commit, so a caller holding a freshly opened instance may
1448
+ * branch on it — the Quereus adapter does, to leave an invented, never-written index tree
1449
+ * unflushed exactly as an unwritten table tree is left. Never branch on the NUMBER. */
1450
+ committedRevision(): number | undefined {
1451
+ return this.source.actionContext?.rev;
1452
+ }
1453
+
1454
+ /** The id of the action that PRODUCED the revision {@link committedRevision} reports —
1455
+ * this collection's lineage marker at that revision — or `undefined` when the action
1456
+ * context holds no entry at the current revision.
1457
+ *
1458
+ * `undefined` is legitimate, not an error: an INVENTED collection has no context at
1459
+ * all, and otherwise {@link actionIdAt} resolves nothing at the current revision for
1460
+ * the reasons listed there. A caller printing this must therefore carry a placeholder
1461
+ * rather than invent an id. The contexts this class writes itself
1462
+ * ({@link recordCommitted}, the inline bump in `syncInternal`,
1463
+ * {@link bootstrapContext}) always do hold one.
1464
+ *
1465
+ * DIAGNOSTIC ONLY — do not branch on this. Its value is the one thing about a revision
1466
+ * that IS comparable across collections and across nodes: a revision number is
1467
+ * per-collection and says nothing on its own, so two nodes reporting the same
1468
+ * collection id at the same revision are indistinguishable between "one collection,
1469
+ * one node lagging" and "two separately-built collections each counting from 1". Equal
1470
+ * action ids mean one lineage; different action ids at the same revision mean two.
1471
+ * `docs/debugging.md` (§ "Which revision did a read descend?") spells out how an
1472
+ * operator reads the pair. */
1473
+ committedActionId(): ActionId | undefined {
1474
+ const context = this.source.actionContext;
1475
+ return context === undefined ? undefined : actionIdAt(context, context.rev);
1476
+ }
1477
+
1478
+ /** Fold a just-committed set of transforms into this collection's read cache
1479
+ * so subsequent reads (and stages) through THIS instance observe the committed
1480
+ * state, mirroring what {@link sync} does inline after a successful transact.
1481
+ *
1482
+ * Needed when commit was orchestrated externally (a coordinator): the tracker
1483
+ * is reset to empty, but the cache still holds the pre-commit blocks. Without
1484
+ * this, a collection that already had committed state (e.g. a pre-synced index
1485
+ * tree, or any collection on its second commit) keeps serving the stale prior
1486
+ * revision because {@link update} sees its rev is already current and refetches
1487
+ * nothing. Call BEFORE resetting the tracker (the transforms are read live).
1488
+ *
1489
+ * @param revision - the committed revision these transforms land at (from
1490
+ * {@link recordCommitted}), so cached read-dependency revisions advance to it. */
1491
+ applyCommittedToCache(transforms: Transforms, revision: number): void {
1492
+ this.sourceCache.transformCache(transforms, revision);
1493
+ }
1494
+
1495
+ /** Next revision this collection would commit at (current committed rev + 1). */
1496
+ getNextRev(): number {
1497
+ return (this.source.actionContext?.rev ?? 0) + 1;
1498
+ }
1499
+
1500
+ /** Record a just-committed action: append its ActionRev to the committed list
1501
+ * and advance the revision. Returns the new revision. Mirrors the inline bump
1502
+ * in {@link syncInternal} — which needs no such rev check because it computes and
1503
+ * uses its `newRev` inside one latched span.
1504
+ *
1505
+ * @param rev - the revision this action was PENDED at, captured once (at the log
1506
+ * append in `TransactionCoordinator.applyActionsToCollection`) and threaded through
1507
+ * the pend/commit round trips. Storage assigned the action THAT number; recording it
1508
+ * at any other would fork this instance's revision counter from storage permanently
1509
+ * (context adoption is one-way — see {@link advanceContext}). With the coordinator
1510
+ * holding this instance's latch across the whole commit span the mismatch cannot
1511
+ * happen; the throw is the tripwire for any path that still bypasses the latch. */
1512
+ recordCommitted(actionId: ActionId, rev: number): number {
1513
+ const expected = this.getNextRev();
1514
+ if (rev !== expected) {
1515
+ throw new Error(`Collection ${this.id}: action ${actionId} was pended at rev ${rev} ` +
1516
+ `but the collection now expects rev ${expected} — the collection was refreshed mid-commit`);
1517
+ }
1518
+ this.source.actionContext = {
1519
+ committed: [...(this.source.actionContext?.committed ?? []), { actionId, rev }],
1520
+ rev,
1521
+ };
1522
+ return rev;
1523
+ }
1524
+
1525
+ /** Acquire this instance's latch — the same mutex {@link act}, {@link update},
1526
+ * {@link sync}, and {@link updateAndSync} serialize behind — returning its release.
1527
+ * Exists so a TransactionCoordinator can hold the latch across its WHOLE commit span
1528
+ * (log append → pend → commit → local fold), keeping any refresh of this instance from
1529
+ * interleaving with a mid-flight commit. `Latches` is non-reentrant: while holding this,
1530
+ * the holder must not call any of those latched methods on this instance. The caller
1531
+ * MUST call the release exactly once, in a `finally`.
1532
+ *
1533
+ * NOTE: the commit paths are the ONLY callers — {@link act}, {@link update} and {@link sync}
1534
+ * take the same mutex through `Latches.acquire` directly, never through here. The acquisition
1535
+ * -order cases in `test/coordinator-latch-span.spec.ts` rely on that: they shadow this method
1536
+ * to record the span's order, so routing a refresh path through it would silently fold
1537
+ * refreshes into those recordings and weaken the assertions rather than fail them. Route a new
1538
+ * refresh caller here only alongside a filter in that spy. */
1539
+ acquireLatch(): Promise<() => void> {
1540
+ return Latches.acquire(this.latchId);
1541
+ }
1542
+
1543
+ /** Bracket a write attempt cycle on this instance under `actionId`, so a refresh taken between
1544
+ * a failed attempt and its retry recognises that action's own already-durable log entry (see
1545
+ * {@link inFlightActionId} for why, and for the lifetime this must span).
1546
+ *
1547
+ * The returned disposer clears the mark and MUST be called in a `finally` covering every exit
1548
+ * from the retry cycle — return, retry exhaustion, partial commit, hard error, abort. A mark
1549
+ * left behind would let a LATER, unrelated refresh consume a foreign entry that happens to
1550
+ * carry the same id. The clear is id-guarded, so a disposer whose mark has since been replaced
1551
+ * by another attempt is a no-op rather than wiping the newer one; disposers may therefore be
1552
+ * called out of order and more than once. Shaped like {@link acquireLatch} deliberately: a
1553
+ * disposer is harder to forget than a paired `end…` call.
1554
+ *
1555
+ * A concurrent READER's refresh landing in that latch-free window is fine, and is reachable
1556
+ * today (`OptimysticModule` declares `concurrencyMode = 'reentrant-reads'`, so scans inside a
1557
+ * transaction share one instance and only serialize on the latch). Consuming is a property of
1558
+ * the (instance, action id) pair, not of who calls: whoever refreshes first drops exactly the
1559
+ * durable entry's actions and the write's own later refresh then finds nothing new. The leading
1560
+ * slice stays right because {@link actInternal} APPENDS, so anything staged after the entry was
1561
+ * written survives it.
1562
+ *
1563
+ * NOTE: a concurrent WRITER is the hazard. The mark deliberately outlives the latch (see
1564
+ * {@link inFlightActionId}), so two writes overlapping on ONE instance can trample each other's:
1565
+ * a second write that acquires the latch between the first's failed attempt and its
1566
+ * retry-refresh replaces the id, and the first's refresh then reads the SECOND write's id —
1567
+ * consuming that write's durable entry and dropping pending actions of its own that never
1568
+ * landed. Not reachable today: a coordinator commit and a `sync()` both run on one session call
1569
+ * path, which is the same assumption the participant selection in
1570
+ * `TransactionCoordinator.commitOnce` and its `rollback` already rest on. If a second writer is
1571
+ * ever allowed to drive the SAME instance concurrently, this must become a per-attempt token (a
1572
+ * mark object compared by identity, refusing to replace a live one) rather than a bare id. */
1573
+ beginInFlightAction(actionId: ActionId): () => void {
1574
+ if (this.inFlightActionId !== actionId) {
1575
+ // A different action's failed attempt cannot finish this one. Re-marking the SAME id (a
1576
+ // coordinator re-marks on every attempt) keeps what the previous attempt retained — that
1577
+ // is the attempt the next refresh may need.
1578
+ this.inFlightAttempt = undefined;
1579
+ }
1580
+ this.inFlightActionId = actionId;
1581
+ return () => {
1582
+ if (this.inFlightActionId === actionId) {
1583
+ this.inFlightActionId = undefined;
1584
+ this.inFlightAttempt = undefined;
1585
+ }
1586
+ };
1587
+ }
1588
+
1589
+ /** Keep what a FAILED attempt under `actionId` sent, so that if the refresh before its retry
1590
+ * finds that attempt's own log entry it can finish the action rather than assume it finished
1591
+ * (see {@link inFlightAttempt} and {@link completeOwnEntry}). Every write path that marks an
1592
+ * action in flight must call this for each attempt that fails, BEFORE the refresh that follows —
1593
+ * a path that does not still never reports a half-landed write as saved (the refresh then
1594
+ * refuses with a `transforms-not-held` {@link TornActionError}), but it cannot recover from one.
1595
+ *
1596
+ * A no-op unless `actionId` is the action currently marked, so a late call from an attempt whose
1597
+ * cycle has already ended cannot plant transforms under somebody else's mark. Latch-free and
1598
+ * synchronous, like the mark itself: `TransactionCoordinator` calls it from inside its commit
1599
+ * span, where it already holds this instance's latch. */
1600
+ retainInFlightAttempt(actionId: ActionId, attempt: InFlightAttempt): void {
1601
+ if (this.inFlightActionId === actionId) {
1602
+ this.inFlightAttempt = attempt;
1603
+ }
1604
+ }
1605
+
1606
+ /** Push our pending actions to the transactor.
1607
+ *
1608
+ * @returns who holds what this sync committed, or `undefined` when NOTHING WAS WRITTEN — a sync
1609
+ * with no staged changes does no pend and no commit, so there is no durability to report and none
1610
+ * is fabricated. That is the one case a caller must handle; every other outcome either returns a
1611
+ * class or throws ({@link SyncRetryExhaustedError} for a write that never landed). Present a change
1612
+ * as saved only via `isFullyDurable`, never by comparing `quorum`. */
1613
+ async sync(options?: SyncOptions): Promise<WriteDurability | undefined> {
1614
+ const release = await Latches.acquire(this.latchId);
1615
+ try {
1616
+ return await this.syncInternal(options);
1617
+ } finally {
1618
+ release();
1619
+ }
1620
+ }
1621
+
1622
+ /** Mints the one action id this sync reuses across all of its retry attempts, and owns it for
1623
+ * the WHOLE cycle — including the inter-attempt refresh, which is the only thing that reads it
1624
+ * (see {@link inFlightActionId}). `sync()`/`updateAndSync()` hold the collection latch across
1625
+ * all of this, so the mark's lifetime is contained inside the latched span here; the disposer
1626
+ * runs on every exit, including a throw out of retry exhaustion or an abort. */
1627
+ private async syncInternal(options?: SyncOptions): Promise<WriteDurability | undefined> {
1628
+ const bytes = randomBytes(16);
1629
+ const actionId = uint8ArrayToString(bytes, 'base64url');
1630
+
1631
+ const endInFlight = this.beginInFlightAction(actionId);
1632
+ try {
1633
+ return await this.syncAttempts(actionId, options);
1634
+ } finally {
1635
+ endInFlight();
1636
+ }
1637
+ }
1638
+
1639
+ /** The retry loop behind {@link syncInternal}, run with `actionId` already marked in flight.
1640
+ *
1641
+ * @returns the durability of what this sync committed, or `undefined` when the loop never ran a
1642
+ * commit (nothing staged). See {@link sync}. */
1643
+ private async syncAttempts(actionId: ActionId, options?: SyncOptions): Promise<WriteDurability | undefined> {
1644
+ const maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
1645
+ const baseBackoffMs = options?.baseBackoffMs ?? PendingRetryDelayMs;
1646
+ const maxBackoffMs = options?.maxBackoffMs ?? DefaultMaxBackoffMs;
1647
+ const maxStalledAttempts = options?.maxStalledAttempts ?? DefaultMaxStalledAttempts;
1648
+ const deadlineMs = options?.deadlineMs;
1649
+ const signal = options?.signal;
1650
+ const startedAt = Date.now();
1651
+
1652
+ // Count of consecutive stale failures that made no forward progress. Reset to 0 on every
1653
+ // successful transact, so the cap bounds only a persistently-failing sync — a legitimate
1654
+ // large multi-batch sync (which iterates many times committing progress) never trips it.
1655
+ let consecutiveFailures = 0;
1656
+ let lastReason: string | undefined;
1657
+ // Highest confirmed revision any responder has reported holding, accumulated with the
1658
+ // codebase's single rule for picking among several candidates. Reported on the error, and
1659
+ // the evidence the stall check below reasons from.
1660
+ let lastStaleAt: { blockId: BlockId; rev: number } | undefined;
1661
+ // Whether the failure most recently handled carried its OWN staleAt. A strike needs the
1662
+ // responder to have re-confirmed the number this round, not merely an older observation
1663
+ // left standing in `lastStaleAt`.
1664
+ let lastFailureConfirmedStaleAt = false;
1665
+ // Consecutive refreshes that moved `getNextRev()` nowhere at all while a confirmed revision
1666
+ // stood at or above it.
1667
+ let consecutiveStalls = 0;
1668
+ // The revision the PREVIOUS iteration would have requested, so the stall check can tell a
1669
+ // refresh that moved nowhere from one that is still climbing toward the confirmed number.
1670
+ let previousRequestedRev: number | undefined;
1671
+ // Who holds what this sync has committed so far. Stays `undefined` while nothing has been
1672
+ // committed, which is also the answer when the loop never runs at all — a sync with nothing
1673
+ // staged writes nothing, and there is no durability to fabricate for it. Failed attempts never
1674
+ // contribute: a refused attempt left nothing behind, so the answer is the committing attempt's.
1675
+ let durability: WriteDurability | undefined;
1676
+
1677
+ while (this.hasUnsyncedChanges()) {
1678
+ if (signal?.aborted) {
1679
+ throw makeAbortError(signal);
1680
+ }
1681
+ // Progress-agnostic ceiling: give up if the wall-clock deadline passed. Deliberately
1682
+ // ahead of the stall check: the deadline is the documented outer bound, so a sync that
1683
+ // is both past it and stalled reports the deadline.
1684
+ if (deadlineMs !== undefined && Date.now() - startedAt >= deadlineMs) {
1685
+ throw new SyncRetryExhaustedError(this.id, consecutiveFailures, lastReason ?? 'deadline exceeded', lastStaleAt);
1686
+ }
1687
+
1688
+ // Can the attempt about to run possibly differ from the one that just failed? Only when
1689
+ // BOTH of these hold is the answer provably no:
1690
+ //
1691
+ // - The revision it would request is at or below one a responder CONFIRMED is taken. A
1692
+ // producer sets `staleAt` only after reading that revision as durably held by someone
1693
+ // else out of its own storage, revisions are one per-collection counter that every
1694
+ // commit touches, and a confirmed revision never becomes un-taken (invalidation takes
1695
+ // a NEW slot). So the request is already lost before it is sent.
1696
+ // - The refresh in between moved the collection nowhere. `advanceContext` never lowers
1697
+ // the held revision, so "nowhere" is exactly `requestedRev` unchanged since the last
1698
+ // iteration. A refresh that moved forward WITHOUT clearing the confirmed number is a
1699
+ // collection still climbing (it read a replica that lags the holder — the same
1700
+ // partial catch-up `reportShortfall` exists to report), and its next attempt is a
1701
+ // genuinely different request that may yet win.
1702
+ //
1703
+ // This is NOT a second answer to "is this failure retryable?" — `isConflictFailure`
1704
+ // remains the sole rule for that, untouched. It only ever stops a loop that rule had
1705
+ // already decided to continue.
1706
+ const requestedRev = this.getNextRev();
1707
+ const refreshMoved = previousRequestedRev !== undefined && requestedRev > previousRequestedRev;
1708
+ previousRequestedRev = requestedRev;
1709
+ if (lastStaleAt !== undefined) {
1710
+ if (requestedRev > lastStaleAt.rev || refreshMoved) {
1711
+ // Either the refresh adopted a revision above the confirmed one — ordinary
1712
+ // contention, where the rival's commit is exactly what we just read — or it made
1713
+ // partial forward progress. Both mean the next attempt differs. Not a stall.
1714
+ consecutiveStalls = 0;
1715
+ } else if (lastFailureConfirmedStaleAt) {
1716
+ consecutiveStalls++;
1717
+ if (log.enabled) {
1718
+ log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d',
1719
+ this.id, this.instanceTag, this.source.actionContext?.rev ?? 'none', requestedRev,
1720
+ lastStaleAt.blockId, lastStaleAt.rev, consecutiveStalls, maxStalledAttempts);
1721
+ }
1722
+ // Two strikes, not one: a legitimate loser can transiently read a view that has
1723
+ // not yet caught up with the rival's commit, which looks identical for one round.
1724
+ if (consecutiveStalls >= maxStalledAttempts) {
1725
+ throw new SyncRevisionStalledError(this.id, consecutiveFailures, lastStaleAt,
1726
+ requestedRev, this.source.actionContext?.rev, lastReason);
1727
+ }
1728
+ }
1729
+ // Else: the responder told us nothing new this round. No strike, and no reset either
1730
+ // — the budget stays bounded by maxAttempts.
1731
+ }
1732
+
1733
+ // A pending action is never pended over a base that has moved under it — before EVERY
1734
+ // attempt, first and retries alike (see restageIfBasesMoved for why the retry needs it).
1735
+ await this.restageIfBasesMoved();
1736
+
1737
+ // Snapshot the pending actions so that any new actions aren't assumed to be part of this action
1738
+ const pending = [...this.pending];
1739
+
1740
+ // Create a snapshot tracker for the action, so that we can ditch the log changes if we have to retry the action.
1741
+ // It SHARES the live tracker's pin store: the snapshot inherits transforms by copy and never
1742
+ // stages the data-block updates itself, so without the shared pins the digest pass below
1743
+ // could only describe bases still resident in the read cache.
1744
+ // NOTE: an attempt that ends in a stale failure is abandoned WITHOUT reset(), so the log
1745
+ // tail/header bases its append pinned stay in the shared store until the live tracker's
1746
+ // next reset. Bounded and harmless — a few log blocks per attempt, capped by maxAttempts,
1747
+ // each re-validated against the source generation before any use — but if this loop ever
1748
+ // grows a per-attempt footprint beyond the log blocks, reset the abandoned tracker on the
1749
+ // stale-failure branch instead of letting the residue ride to the end of the sync.
1750
+ const snapshot = copyTransforms(this.tracker.transforms);
1751
+ const tracker = new Tracker(this.sourceCache, snapshot, this.tracker.pins);
1752
+
1753
+ // Add the action to the log (in local tracking space)
1754
+ const collectionLog = await Log.open<Action<TAction>>(tracker, this.id);
1755
+ if (!collectionLog) {
1756
+ throw new Error(`Log not found for collection ${this.id}`);
1757
+ }
1758
+ const newRev = this.getNextRev();
1759
+ const addResult = await collectionLog.addActions(pending, actionId, newRev, () => tracker.transformedBlockIds());
1760
+
1761
+ // Declare what each touched block will contain once committed, computed from this snapshot
1762
+ // tracker (which layers over `this.sourceCache`, so the peek/getCachedRevision probes are
1763
+ // live). Purely local — an id whose base is not already cached is simply omitted and falls
1764
+ // back to corroboration on the member side. Computed AFTER the log append so the log tail
1765
+ // and header transforms this attempt just staged are digested too.
1766
+ // NOTE: recomputed from scratch on every retry attempt (the snapshot tracker is rebuilt each
1767
+ // iteration), so a sync that loses N races pays N full hashing passes over its touched
1768
+ // blocks. Unmeasured and cheap relative to the round trips it is retrying; if a
1769
+ // high-contention sync ever shows digest hashing in a profile, memoize per (id, staged ops).
1770
+ const blockDigests = await computeBlockContentDigests(tracker, tracker.transformedBlockIds());
1771
+ // The base each update-only block's operations were computed against, for the pend (see
1772
+ // `PendRequest.baseRevs`): the tracker's pinned revisions, which `restageIfBasesMoved` at the
1773
+ // top of this iteration has just re-judged, so a moved base was re-staged before it is named.
1774
+ const baseRevs = tracker.stagedBaseRevs(tracker.transformedBlockIds());
1775
+
1776
+ // Commit the action to the transactor. Carry the aged retry priority derived from the
1777
+ // consecutive-failure count so a sync that keeps losing concurrent races out-ranks fresh
1778
+ // (priority-0) rivals in the cluster's resolveRace (fairness-only; capped at MaxPriority).
1779
+ // First attempt has consecutiveFailures == 0, so priority 0 — the common pend is unchanged.
1780
+ const attempt = await this.source.transact(tracker.transforms, actionId, newRev, this.id, addResult.tailPath.block.header.id, clampPriority(consecutiveFailures), blockDigests, baseRevs);
1781
+ if (!attempt.success) {
1782
+ consecutiveFailures++;
1783
+ lastReason = attempt.reason ?? lastReason;
1784
+ // Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
1785
+ // responder reporting a LOWER number understates the binding constraint. Same rule the
1786
+ // producers and the transactor's aggregation already use.
1787
+ lastStaleAt = highestStaleAt([lastStaleAt, attempt.staleAt]);
1788
+ lastFailureConfirmedStaleAt = attempt.staleAt !== undefined;
1789
+ // Give up once the consecutive no-progress budget is exhausted, so a transactor that
1790
+ // persistently rejects the sync can no longer hold the collection latch forever.
1791
+ // NOTE: this also bounds the legitimate `pending`-wait case (retrying the same action
1792
+ // while another commit is in flight), which used to retry indefinitely. Default 10
1793
+ // attempts ≈ 21s of exponential backoff. If a high-contention workload legitimately
1794
+ // needs to wait longer for a pending commit to clear, raise maxAttempts for that caller.
1795
+ // NOTE: no refresh follows the LAST budgeted attempt, so if that attempt's log tail
1796
+ // landed nobody finds out: the caller gets plain exhaustion over a log that holds an
1797
+ // entry for this write. The write is still never reported saved, which is the rule; what
1798
+ // is lost is the more specific name (TornActionError). The leftover entry itself is
1799
+ // tracked in tickets/backlog/bug-a-refused-write-can-leave-its-log-entry-behind.
1800
+ if (consecutiveFailures >= maxAttempts) {
1801
+ throw new SyncRetryExhaustedError(this.id, consecutiveFailures, lastReason, lastStaleAt);
1802
+ }
1803
+ // Keep exactly what this attempt sent. A refused commit is not proof nothing landed:
1804
+ // the log tail is committed first and can be stored while the answer is still a
1805
+ // failure, and `transact` has just cancelled every block that did not land. If the
1806
+ // refresh below finds this attempt's own entry, these are the transforms that finish
1807
+ // the action (see completeOwnEntry). The snapshot tracker is abandoned after this
1808
+ // iteration, so its transforms are handed over as-is, uncopied.
1809
+ this.retainInFlightAttempt(actionId, {
1810
+ rev: newRev,
1811
+ transforms: tracker.transforms,
1812
+ tailId: addResult.tailPath.block.header.id,
1813
+ ...(blockDigests === undefined ? {} : { blockDigests }),
1814
+ ...baseRevsField(baseRevs),
1815
+ });
1816
+ // Refresh, and keep refreshing while it reports that this write's own half-landed
1817
+ // action could not be finished YET. It must not fall through to a new attempt in that
1818
+ // state: a new attempt rebuilds the log entry (fresh timestamp) and would send a second
1819
+ // version of a log tail that is already stored under this action and revision. Each
1820
+ // round is a failure against the same no-progress budget as a refused attempt.
1821
+ for (;;) {
1822
+ // Back off before every retry (any stale failure — reason/missing/pending), growing
1823
+ // exponentially from the base delay up to the cap, with proportional random jitter so a
1824
+ // herd of clients that lost the same race does not re-collide on the next tick (see
1825
+ // utility/backoff.ts). The abortable sleep lets an aborted sync reject promptly instead
1826
+ // of finishing the sleep.
1827
+ // NOTE: the `missing`/`reason` conflict paths now pay this backoff too (they previously
1828
+ // retried with zero delay); that is what stops the persistent-`reason` hot spin. If a
1829
+ // high-contention workload ever shows this base delay as recovery latency, lower
1830
+ // baseBackoffMs for that caller rather than reintroducing the zero-delay retry.
1831
+ const delay = jitteredBackoffMs(consecutiveFailures - 1, { baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
1832
+ await abortableDelay(delay, signal);
1833
+ try {
1834
+ // Fetch latest state - updateInternal() will call replayActions() if there are conflicts.
1835
+ // This sync's actionId is marked in flight for the whole cycle (see syncInternal), so
1836
+ // the refresh recognizes a log entry written by THIS action (its log tail landed but
1837
+ // the commit answered failure), FINISHES that action from the attempt retained
1838
+ // above, and only then consumes the entry rather than replaying it into a duplicate.
1839
+ // The round that would spend the last of the budget settles a half-landed write
1840
+ // instead of asking for another round, so the error that escapes below says
1841
+ // whether the write can still land. (A deadline cannot be foreseen the same
1842
+ // way; a write given up on it escapes unsettled, as `final: false`.)
1843
+ const report: RefreshReport = {};
1844
+ await this.updateInternal(report, consecutiveFailures + 1 >= maxAttempts);
1845
+ const completed = report.ownEntryFinished?.durability;
1846
+ if (completed !== undefined) {
1847
+ // The refresh made this sync's write durable: that is a commit, and it is
1848
+ // reported and counted exactly like one made by an attempt (see the success
1849
+ // branch below for why batches fold to the weakest).
1850
+ durability = durability === undefined ? completed : mergeDurability([durability, completed]);
1851
+ consecutiveFailures = 0;
1852
+ lastReason = undefined;
1853
+ lastStaleAt = undefined;
1854
+ lastFailureConfirmedStaleAt = false;
1855
+ consecutiveStalls = 0;
1856
+ }
1857
+ break;
1858
+ } catch (err) {
1859
+ if (!(err instanceof TornActionError) || err.reason !== 'completion-refused') {
1860
+ throw err;
1861
+ }
1862
+ // Refused for a cause that can clear. The error already names the write as torn,
1863
+ // which is the truth if the budget ends here — so it, not a plain exhaustion,
1864
+ // is what escapes: the log holds an entry for a write that was not saved.
1865
+ consecutiveFailures++;
1866
+ lastReason = err.detail;
1867
+ if (consecutiveFailures >= maxAttempts
1868
+ || (deadlineMs !== undefined && Date.now() - startedAt >= deadlineMs)) {
1869
+ throw err;
1870
+ }
1871
+ }
1872
+ }
1873
+ } else {
1874
+ // This attempt's commit landed, so its durability is the one to report. A sync that
1875
+ // commits ONCE — every sync that has a caller today — reports exactly that answer,
1876
+ // untouched. A sync whose loop commits more than one batch (the `hasUnsyncedChanges`
1877
+ // condition re-entering after a successful commit) folds the batches with
1878
+ // `mergeDurability`, whose scalar answer is the WEAKEST of them: one batch that only
1879
+ // reached the writer makes the whole sync only-on-the-writer, and reporting the last
1880
+ // batch's class instead would show such a write as saved.
1881
+ durability = durability === undefined ? attempt.durability : mergeDurability([durability, attempt.durability]);
1882
+ // Forward progress: reset the no-progress budget.
1883
+ consecutiveFailures = 0;
1884
+ lastReason = undefined;
1885
+ lastStaleAt = undefined;
1886
+ lastFailureConfirmedStaleAt = false;
1887
+ consecutiveStalls = 0;
1888
+ // Clear the pending actions that were part of this action
1889
+ this.pending = this.pending.slice(pending.length);
1890
+ // Reset cache and replay any actions that were added during the action
1891
+ const transforms = tracker.reset();
1892
+ // NOTE: this replay runs BEFORE the cache fold and the context bump below — the
1893
+ // inverse of the order `updateInternal` and `TransactionCoordinator.commitOnce`
1894
+ // both document as required, where the newly committed state must be visible
1895
+ // before anything re-reads. Dormant today: `act()` and `syncInternal` take the
1896
+ // same collection latch, so `this.pending` cannot grow during the transact above
1897
+ // and the slice always leaves it empty, making this replay a no-op reset. If a
1898
+ // path ever stages outside that latch, move the transformCache + actionContext
1899
+ // lines above this replay — otherwise the replay re-reads at the superseded
1900
+ // revision over a cache that has not yet seen the commit, and re-stages onto a
1901
+ // root the commit already replaced.
1902
+ await this.replayActions();
1903
+ this.sourceCache.transformCache(transforms, newRev);
1904
+ this.source.actionContext = this.source.actionContext
1905
+ ? { committed: [...this.source.actionContext.committed, { actionId, rev: newRev }], rev: newRev }
1906
+ : { committed: [{ actionId, rev: newRev }], rev: newRev };
1907
+ }
1908
+ }
1909
+ return durability;
1910
+ }
1911
+
1912
+ /** Refresh from the transactor, then push. Same return contract as {@link sync}: the durability of
1913
+ * what was committed, or `undefined` when nothing was staged and so nothing was written. */
1914
+ async updateAndSync(options?: SyncOptions): Promise<WriteDurability | undefined> {
1915
+ const release = await Latches.acquire(this.latchId);
1916
+ try {
1917
+ await this.updateInternal({});
1918
+ return await this.syncInternal(options);
1919
+ } finally {
1920
+ release();
1921
+ }
1922
+ }
1923
+
1924
+ /** Stage `actions` and flush them as one call that either takes effect or leaves NOTHING behind:
1925
+ * when the flush throws, the actions this call staged are unstaged again before the error escapes.
1926
+ *
1927
+ * For callers that own both halves of a write (`Tree.replace`, `Diary.append`). Staging with
1928
+ * {@link act} and flushing separately leaves a failed write's actions staged — deliberate for a
1929
+ * caller that means to call {@link sync} again, and a trap for one that does not: the actions
1930
+ * ride along, unasked, with its NEXT write, so a change the caller was told had failed shows up
1931
+ * later, and a caller that reacted to the failure by submitting it again stores it twice. That
1932
+ * is how a write reported torn was seen to "appear one write later" (the pending-record route
1933
+ * first suspected was ruled out: with the failed writer's collection discarded, 0 of 17 torn
1934
+ * rows ever appeared).
1935
+ *
1936
+ * One latch hold spans staging and flushing, so no other {@link act} on this instance can land
1937
+ * between them and the actions to take back are exactly the ones this call put there.
1938
+ *
1939
+ * The error is rethrown untouched. Unstaging says nothing about storage: a
1940
+ * {@link TornActionError} with `final: false` still means the write may be saved or may yet
1941
+ * land, and only the staged copy is gone. */
1942
+ async actAndSync(actions: Action<TAction>[], options?: SyncOptions): Promise<WriteDurability | undefined> {
1943
+ const release = await Latches.acquire(this.latchId);
1944
+ try {
1945
+ const stagedBefore = copyTransforms(this.tracker.transforms);
1946
+ const revBefore = this.source.actionContext?.rev;
1947
+ await this.actInternal(...actions);
1948
+ try {
1949
+ await this.updateInternal({});
1950
+ return await this.syncInternal(options);
1951
+ } catch (err) {
1952
+ await this.unstage(actions, stagedBefore, revBefore);
1953
+ throw err;
1954
+ }
1955
+ } finally {
1956
+ release();
1957
+ }
1958
+ }
1959
+
1960
+ /** Takes `actions` back out of the staged queue and rebuilds the tracker without them (always
1961
+ * called under latch). `stagedBefore` / `revBefore` are the tracker's transforms and the held
1962
+ * revision from before the actions were staged.
1963
+ *
1964
+ * While the held revision has not moved, the earlier transforms are reinstated verbatim rather
1965
+ * than rebuilt by replay: an INVENTED collection keeps its header and root in the tracker with
1966
+ * no staged action naming them, and a replay (which resets the tracker first) would drop them —
1967
+ * the hazard {@link mustReplay} and {@link snapshotPending} document. Once the revision has
1968
+ * moved those transforms describe blocks at a revision this handle has left, so what remains is
1969
+ * re-staged against the adopted one, exactly as a refresh would have done. */
1970
+ private async unstage(actions: Action<TAction>[], stagedBefore: Transforms, revBefore: number | undefined): Promise<void> {
1971
+ // NOTE: matched by identity, which is what `filterAgainstEntry` promises for a kept action.
1972
+ // A `filterConflict` hook that answers a REPLACEMENT instance for one of these actions would
1973
+ // leave the replacement staged here. No collection installs such a hook today (`Tree` and
1974
+ // `Diary` install none); if one ever does, carry a per-call token on the staged actions and
1975
+ // match on that instead.
1976
+ this.pending = this.pending.filter(staged => !actions.includes(staged));
1977
+ if (this.source.actionContext?.rev === revBefore) {
1978
+ this.tracker.reset(stagedBefore);
1979
+ } else {
1980
+ await this.replayActions();
1981
+ }
1982
+ }
1983
+
1984
+ async *selectLog(forward = true): AsyncIterableIterator<Action<TAction>> {
1985
+ const collectionLog = await Log.open<Action<TAction>>(this.tracker, this.id);
1986
+ if (!collectionLog) {
1987
+ throw new Error(`Log not found for collection ${this.id}`);
1988
+ }
1989
+ for await (const entry of collectionLog.select(undefined, forward)) {
1990
+ if (entry.action) {
1991
+ // NOTE: copy-then-reverse to avoid mutating the stored log entry array.
1992
+ // Once tsconfig targets ES2023, `entry.action.actions.toReversed()` is cleaner.
1993
+ yield* forward ? entry.action.actions : [...entry.action.actions].reverse();
1994
+ }
1995
+ }
1996
+ }
1997
+
1998
+ private async replayActions() {
1999
+ this.tracker.reset();
2000
+ // Replay pending actions against the fresh tracker state (always called under latch)
2001
+ for (const action of this.pending) {
2002
+ await this.internalTransact(action);
2003
+ }
2004
+ }
2005
+
2006
+ getReadDependencies(): ReadDependency[] {
2007
+ return this.source.getReadDependencies();
2008
+ }
2009
+
2010
+ clearReadDependencies(): void {
2011
+ this.source.clearReadDependencies();
2012
+ }
2013
+
2014
+ /** Called for each local action that may be in conflict with a remote action (always called under latch).
2015
+ * @param action - The local action to check
2016
+ * @param potential - The remote actions that are potentially in conflict
2017
+ * @returns The effective action to keep: the original (unchanged), a replacement
2018
+ * instance (applied instead of the original), or undefined to discard it.
2019
+ */
2020
+ protected doFilterConflict(action: Action<TAction>, potential: Action<TAction>[]): Action<TAction> | undefined {
2021
+ return this.filterConflict ? this.filterConflict(action, potential) : action;
2022
+ }
2023
+
2024
+ /** The two blocks every refresh starts from — the collection header and the log tail block it
2025
+ * names — read unpinned ("latest"), in ONE request when `knownTailId` is the tail the header
2026
+ * names. Only when the header names a different tail (the known one filled, or none was known)
2027
+ * is that tail fetched in a second request; the out-of-date block's answer is dropped rather
2028
+ * than kept, because nothing proves it is current at the revision the refresh will pin to.
2029
+ *
2030
+ * Both answers pass {@link answeredBlock}'s checks as unpinned reads, header first, so a doubted
2031
+ * header or tail throws exactly as it would through {@link TransactorSource.tryGet}.
2032
+ *
2033
+ * NOTE: a batched get fails as a whole when any block in it gets no answer
2034
+ * (`NetworkTransactor.get` throws on a missing id), so an unreachable out-of-date tail fails a
2035
+ * refresh that would not have needed it. Harmless today: the log has no checkpoints, so a
2036
+ * refresh that finds a new tail walks back through the old one anyway. If checkpoints start
2037
+ * letting that walk stop short, read the known tail in its own request instead.
2038
+ *
2039
+ * @returns undefined when the header is authoritatively absent. */
2040
+ private static async readLogEnds(transactor: ITransactor, id: CollectionId, knownTailId: BlockId | undefined): Promise<LogEnds | undefined> {
2041
+ const results = await transactor.get({ blockIds: knownTailId === undefined ? [id] : [id, knownTailId] });
2042
+ const headerEntry = results?.[id];
2043
+ if (headerEntry === undefined) {
2044
+ return undefined;
2045
+ }
2046
+ const header = answeredBlock(id, headerEntry, undefined) as CollectionHeaderBlock | undefined;
2047
+ if (!header) {
2048
+ return undefined;
2049
+ }
2050
+ const served: LogEnds['served'] = [[id, header, servedRevision(headerEntry)]];
2051
+ const tailId = header.tailId;
2052
+ if (tailId === undefined) {
2053
+ return { header, served };
2054
+ }
2055
+ const tail = tailId === knownTailId
2056
+ ? Collection.checkedLogTail(tailId, results[tailId])
2057
+ : await Collection.readLogTail(transactor, tailId);
2058
+ if (tail?.block) {
2059
+ served.push([tailId, tail.block, servedRevision(tail)]);
2060
+ }
2061
+ return { header, tail, served };
2062
+ }
2063
+
2064
+ /** An unpinned read of the log tail block, checked as {@link checkedLogTail} describes. */
2065
+ private static async readLogTail(transactor: ITransactor, tailId: BlockId): Promise<GetBlockResult | undefined> {
2066
+ return Collection.checkedLogTail(tailId, (await transactor.get({ blockIds: [tailId] }))?.[tailId]);
2067
+ }
2068
+
2069
+ /** The repo's answer for the log tail, once it has passed {@link answeredBlock}'s unpinned-read
2070
+ * checks. The raw entry, not just the block, is what a refresh needs — {@link bootstrapContext}
2071
+ * reads `state.latest` off it — which is why the tail is read around {@link TransactorSource}
2072
+ * and has to be checked here.
2073
+ *
2074
+ * Both checks matter at this seam in particular. A tail the repo could not retrieve must not
2075
+ * degrade into "no context", which would leave the chain walk unable to see pending non-tail
2076
+ * blocks and the collection reading as if they did not exist. And this unpinned tail read is
2077
+ * the ONE seam where a lagging collection can learn a newer revision exists — every later data
2078
+ * read is pinned to the context seeded from it — so seeding from a tail the repo could not
2079
+ * confirm is current would freeze the collection at the stale revision with nothing ever
2080
+ * reporting a problem. A tail with no `state.latest` and no flag is a real answer (nothing
2081
+ * committed yet). */
2082
+ private static checkedLogTail(tailId: BlockId, entry: GetBlockResult | undefined): GetBlockResult | undefined {
2083
+ if (entry) {
2084
+ answeredBlock(tailId, entry, undefined);
2085
+ }
2086
+ return entry;
2087
+ }
2088
+
2089
+ /** Bootstrap ActionContext from the committed tail block's state.
2090
+ * Every member applies an action's tail before its other blocks (commit protocol guarantee:
2091
+ * `StorageRepo.commit` orders the tail first and stops at the first failure), so wherever
2092
+ * committed data of an action exists its tail does too, readable with context=undefined.
2093
+ * Its state.latest contains the ActionRev of the most recent
2094
+ * committed action — exactly the proof needed for the transactor to serve pending
2095
+ * non-tail blocks during chain walks. A tail with no `latest` (or no tail) no-ops.
2096
+ *
2097
+ * NOTE: this number is adopted on trust, and adoption is one-way (advanceContext never
2098
+ * lowers it). A tail that over-claims therefore pins the collection at a revision its
2099
+ * own log can never reach, permanently: every later refresh walks the log (the claim's newest
2100
+ * entry never matches, so {@link tailShowsNothingNewer} never lets it skip), reads the
2101
+ * real (lower) revision, and is refused — so the instance emits
2102
+ * `collection:context-not-lowered` forever while `collection:context-short-of-tail`
2103
+ * stays silent (the held revision is at or above what the tail claims). No condition
2104
+ * that makes a real tail over-claim has been demonstrated; this was seen only through a
2105
+ * test double built to lie (see collection.spec.ts, 'a refresh that lands short of the
2106
+ * tail it just read'). If an over-claiming tail is ever observed in the field, the fix
2107
+ * belongs here — validate the claim against the log before pinning — not in the refresh.
2108
+ */
2109
+ private static bootstrapContext(source: TransactorSource<IBlock>, tail: GetBlockResult | undefined): void {
2110
+ const latest = tail?.state.latest;
2111
+ if (latest) {
2112
+ source.actionContext = {
2113
+ committed: [{ actionId: latest.actionId, rev: latest.rev }],
2114
+ rev: latest.rev,
2115
+ };
2116
+ }
2117
+ }
2118
+ }