@optimystic/db-core 1.18.0 → 1.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. package/README.md +2 -1
  2. package/dist/src/blocks/helpers.d.ts +7 -0
  3. package/dist/src/blocks/helpers.d.ts.map +1 -1
  4. package/dist/src/blocks/helpers.js +7 -1
  5. package/dist/src/blocks/helpers.js.map +1 -1
  6. package/dist/src/btree/btree.d.ts +100 -4
  7. package/dist/src/btree/btree.d.ts.map +1 -1
  8. package/dist/src/btree/btree.js +252 -90
  9. package/dist/src/btree/btree.js.map +1 -1
  10. package/dist/src/btree/node-division.d.ts +37 -0
  11. package/dist/src/btree/node-division.d.ts.map +1 -0
  12. package/dist/src/btree/node-division.js +133 -0
  13. package/dist/src/btree/node-division.js.map +1 -0
  14. package/dist/src/chain/chain.d.ts +23 -0
  15. package/dist/src/chain/chain.d.ts.map +1 -1
  16. package/dist/src/chain/chain.js +43 -6
  17. package/dist/src/chain/chain.js.map +1 -1
  18. package/dist/src/cluster/structs.d.ts +41 -8
  19. package/dist/src/cluster/structs.d.ts.map +1 -1
  20. package/dist/src/cluster/structs.js +18 -7
  21. package/dist/src/cluster/structs.js.map +1 -1
  22. package/dist/src/collection/action.d.ts +29 -0
  23. package/dist/src/collection/action.d.ts.map +1 -1
  24. package/dist/src/collection/action.js +6 -0
  25. package/dist/src/collection/action.js.map +1 -1
  26. package/dist/src/collection/collection.d.ts +566 -120
  27. package/dist/src/collection/collection.d.ts.map +1 -1
  28. package/dist/src/collection/collection.js +1084 -495
  29. package/dist/src/collection/collection.js.map +1 -1
  30. package/dist/src/collection/head-commit.d.ts +110 -0
  31. package/dist/src/collection/head-commit.d.ts.map +1 -0
  32. package/dist/src/collection/head-commit.js +65 -0
  33. package/dist/src/collection/head-commit.js.map +1 -0
  34. package/dist/src/collection/index.d.ts +2 -0
  35. package/dist/src/collection/index.d.ts.map +1 -1
  36. package/dist/src/collection/index.js +2 -0
  37. package/dist/src/collection/index.js.map +1 -1
  38. package/dist/src/collection/segments.d.ts +77 -0
  39. package/dist/src/collection/segments.d.ts.map +1 -0
  40. package/dist/src/collection/segments.js +110 -0
  41. package/dist/src/collection/segments.js.map +1 -0
  42. package/dist/src/collection/struct.d.ts +176 -1
  43. package/dist/src/collection/struct.d.ts.map +1 -1
  44. package/dist/src/collection/struct.js +70 -0
  45. package/dist/src/collection/struct.js.map +1 -1
  46. package/dist/src/collection/write-attempts.d.ts +245 -0
  47. package/dist/src/collection/write-attempts.d.ts.map +1 -0
  48. package/dist/src/collection/write-attempts.js +439 -0
  49. package/dist/src/collection/write-attempts.js.map +1 -0
  50. package/dist/src/collections/tree/struct.d.ts +100 -17
  51. package/dist/src/collections/tree/struct.d.ts.map +1 -1
  52. package/dist/src/collections/tree/struct.js +38 -7
  53. package/dist/src/collections/tree/struct.js.map +1 -1
  54. package/dist/src/collections/tree/tree.d.ts +17 -5
  55. package/dist/src/collections/tree/tree.d.ts.map +1 -1
  56. package/dist/src/collections/tree/tree.js +117 -29
  57. package/dist/src/collections/tree/tree.js.map +1 -1
  58. package/dist/src/index.d.ts +2 -0
  59. package/dist/src/index.d.ts.map +1 -1
  60. package/dist/src/index.js +2 -0
  61. package/dist/src/index.js.map +1 -1
  62. package/dist/src/log/log.d.ts +5 -1
  63. package/dist/src/log/log.d.ts.map +1 -1
  64. package/dist/src/log/log.js +7 -1
  65. package/dist/src/log/log.js.map +1 -1
  66. package/dist/src/network/block-size.d.ts +67 -0
  67. package/dist/src/network/block-size.d.ts.map +1 -0
  68. package/dist/src/network/block-size.js +82 -0
  69. package/dist/src/network/block-size.js.map +1 -0
  70. package/dist/src/network/i-repo.d.ts +3 -0
  71. package/dist/src/network/i-repo.d.ts.map +1 -1
  72. package/dist/src/network/index.d.ts +5 -0
  73. package/dist/src/network/index.d.ts.map +1 -1
  74. package/dist/src/network/index.js +5 -0
  75. package/dist/src/network/index.js.map +1 -1
  76. package/dist/src/network/promise-shortfall-error.d.ts +40 -0
  77. package/dist/src/network/promise-shortfall-error.d.ts.map +1 -0
  78. package/dist/src/network/promise-shortfall-error.js +67 -0
  79. package/dist/src/network/promise-shortfall-error.js.map +1 -0
  80. package/dist/src/network/refuse-solo.d.ts +8 -0
  81. package/dist/src/network/refuse-solo.d.ts.map +1 -0
  82. package/dist/src/network/refuse-solo.js +8 -0
  83. package/dist/src/network/refuse-solo.js.map +1 -0
  84. package/dist/src/network/stale-failure.d.ts +20 -3
  85. package/dist/src/network/stale-failure.d.ts.map +1 -1
  86. package/dist/src/network/stale-failure.js +24 -3
  87. package/dist/src/network/stale-failure.js.map +1 -1
  88. package/dist/src/network/struct.d.ts +161 -13
  89. package/dist/src/network/struct.d.ts.map +1 -1
  90. package/dist/src/network/struct.js.map +1 -1
  91. package/dist/src/network/transaction-size.d.ts +59 -0
  92. package/dist/src/network/transaction-size.d.ts.map +1 -0
  93. package/dist/src/network/transaction-size.js +74 -0
  94. package/dist/src/network/transaction-size.js.map +1 -0
  95. package/dist/src/network/validity-rejected-error.d.ts +28 -0
  96. package/dist/src/network/validity-rejected-error.d.ts.map +1 -0
  97. package/dist/src/network/validity-rejected-error.js +51 -0
  98. package/dist/src/network/validity-rejected-error.js.map +1 -0
  99. package/dist/src/offline/entry.d.ts +72 -0
  100. package/dist/src/offline/entry.d.ts.map +1 -0
  101. package/dist/src/offline/entry.js +73 -0
  102. package/dist/src/offline/entry.js.map +1 -0
  103. package/dist/src/offline/index.d.ts +5 -0
  104. package/dist/src/offline/index.d.ts.map +1 -0
  105. package/dist/src/offline/index.js +5 -0
  106. package/dist/src/offline/index.js.map +1 -0
  107. package/dist/src/offline/journal.d.ts +61 -0
  108. package/dist/src/offline/journal.d.ts.map +1 -0
  109. package/dist/src/offline/journal.js +2 -0
  110. package/dist/src/offline/journal.js.map +1 -0
  111. package/dist/src/offline/queue.d.ts +484 -0
  112. package/dist/src/offline/queue.d.ts.map +1 -0
  113. package/dist/src/offline/queue.js +1236 -0
  114. package/dist/src/offline/queue.js.map +1 -0
  115. package/dist/src/offline/reachability.d.ts +25 -0
  116. package/dist/src/offline/reachability.d.ts.map +1 -0
  117. package/dist/src/offline/reachability.js +13 -0
  118. package/dist/src/offline/reachability.js.map +1 -0
  119. package/dist/src/offline/status.d.ts +148 -0
  120. package/dist/src/offline/status.d.ts.map +1 -0
  121. package/dist/src/offline/status.js +63 -0
  122. package/dist/src/offline/status.js.map +1 -0
  123. package/dist/src/offline/wake.d.ts +30 -0
  124. package/dist/src/offline/wake.d.ts.map +1 -0
  125. package/dist/src/offline/wake.js +68 -0
  126. package/dist/src/offline/wake.js.map +1 -0
  127. package/dist/src/reactivity/config.d.ts +8 -4
  128. package/dist/src/reactivity/config.d.ts.map +1 -1
  129. package/dist/src/reactivity/config.js +7 -3
  130. package/dist/src/reactivity/config.js.map +1 -1
  131. package/dist/src/reactivity/rotation.d.ts +9 -3
  132. package/dist/src/reactivity/rotation.d.ts.map +1 -1
  133. package/dist/src/reactivity/rotation.js +8 -2
  134. package/dist/src/reactivity/rotation.js.map +1 -1
  135. package/dist/src/testing/test-transactor.d.ts +31 -0
  136. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  137. package/dist/src/testing/test-transactor.js +60 -0
  138. package/dist/src/testing/test-transactor.js.map +1 -1
  139. package/dist/src/transaction/commit-failure-class.d.ts +103 -0
  140. package/dist/src/transaction/commit-failure-class.d.ts.map +1 -0
  141. package/dist/src/transaction/commit-failure-class.js +96 -0
  142. package/dist/src/transaction/commit-failure-class.js.map +1 -0
  143. package/dist/src/transaction/coordinator.d.ts +224 -31
  144. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  145. package/dist/src/transaction/coordinator.js +841 -362
  146. package/dist/src/transaction/coordinator.js.map +1 -1
  147. package/dist/src/transaction/entry-commit.d.ts +139 -0
  148. package/dist/src/transaction/entry-commit.d.ts.map +1 -0
  149. package/dist/src/transaction/entry-commit.js +199 -0
  150. package/dist/src/transaction/entry-commit.js.map +1 -0
  151. package/dist/src/transaction/index.d.ts +4 -1
  152. package/dist/src/transaction/index.d.ts.map +1 -1
  153. package/dist/src/transaction/index.js +1 -0
  154. package/dist/src/transaction/index.js.map +1 -1
  155. package/dist/src/transaction/transaction.d.ts +20 -0
  156. package/dist/src/transaction/transaction.d.ts.map +1 -1
  157. package/dist/src/transaction/validator.d.ts +7 -0
  158. package/dist/src/transaction/validator.d.ts.map +1 -1
  159. package/dist/src/transaction/validator.js +21 -12
  160. package/dist/src/transaction/validator.js.map +1 -1
  161. package/dist/src/transactor/change-notifier.d.ts +13 -4
  162. package/dist/src/transactor/change-notifier.d.ts.map +1 -1
  163. package/dist/src/transactor/change-notifier.js.map +1 -1
  164. package/dist/src/transactor/network-transactor.d.ts +58 -8
  165. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  166. package/dist/src/transactor/network-transactor.js +244 -50
  167. package/dist/src/transactor/network-transactor.js.map +1 -1
  168. package/dist/src/transactor/transactor-source.d.ts +22 -11
  169. package/dist/src/transactor/transactor-source.d.ts.map +1 -1
  170. package/dist/src/transactor/transactor-source.js +9 -13
  171. package/dist/src/transactor/transactor-source.js.map +1 -1
  172. package/dist/src/transactor/transactor.d.ts +14 -0
  173. package/dist/src/transactor/transactor.d.ts.map +1 -1
  174. package/dist/src/transform/digest.d.ts +20 -0
  175. package/dist/src/transform/digest.d.ts.map +1 -1
  176. package/dist/src/transform/digest.js +23 -3
  177. package/dist/src/transform/digest.js.map +1 -1
  178. package/dist/src/utility/visit-every.d.ts +14 -0
  179. package/dist/src/utility/visit-every.d.ts.map +1 -0
  180. package/dist/src/utility/visit-every.js +27 -0
  181. package/dist/src/utility/visit-every.js.map +1 -0
  182. package/package.json +1 -1
  183. package/src/blocks/helpers.ts +8 -1
  184. package/src/btree/btree.ts +266 -88
  185. package/src/btree/node-division.ts +155 -0
  186. package/src/chain/chain.ts +55 -6
  187. package/src/cluster/structs.ts +42 -7
  188. package/src/collection/action.ts +34 -0
  189. package/src/collection/collection.ts +1167 -540
  190. package/src/collection/head-commit.ts +148 -0
  191. package/src/collection/index.ts +2 -0
  192. package/src/collection/segments.ts +149 -0
  193. package/src/collection/struct.ts +201 -1
  194. package/src/collection/write-attempts.ts +506 -0
  195. package/src/collections/tree/struct.ts +97 -17
  196. package/src/collections/tree/tree.ts +129 -31
  197. package/src/index.ts +2 -0
  198. package/src/log/log.ts +7 -1
  199. package/src/network/block-size.ts +81 -0
  200. package/src/network/i-repo.ts +3 -0
  201. package/src/network/index.ts +5 -0
  202. package/src/network/promise-shortfall-error.ts +69 -0
  203. package/src/network/refuse-solo.ts +7 -0
  204. package/src/network/stale-failure.ts +27 -4
  205. package/src/network/struct.ts +169 -15
  206. package/src/network/transaction-size.ts +73 -0
  207. package/src/network/validity-rejected-error.ts +55 -0
  208. package/src/offline/entry.ts +127 -0
  209. package/src/offline/index.ts +4 -0
  210. package/src/offline/journal.ts +60 -0
  211. package/src/offline/queue.ts +1342 -0
  212. package/src/offline/reachability.ts +32 -0
  213. package/src/offline/status.ts +183 -0
  214. package/src/offline/wake.ts +67 -0
  215. package/src/reactivity/config.ts +8 -4
  216. package/src/reactivity/rotation.ts +9 -3
  217. package/src/testing/test-transactor.ts +65 -0
  218. package/src/transaction/commit-failure-class.ts +133 -0
  219. package/src/transaction/coordinator.ts +969 -393
  220. package/src/transaction/entry-commit.ts +285 -0
  221. package/src/transaction/index.ts +4 -0
  222. package/src/transaction/transaction.ts +21 -0
  223. package/src/transaction/validator.ts +22 -12
  224. package/src/transactor/change-notifier.ts +13 -4
  225. package/src/transactor/network-transactor.ts +252 -55
  226. package/src/transactor/transactor-source.ts +29 -13
  227. package/src/transactor/transactor.ts +16 -0
  228. package/src/transform/digest.ts +36 -3
  229. package/src/utility/visit-every.ts +25 -0
@@ -0,0 +1,506 @@
1
+ import type { IBlock, BlockId, Action, ActionId, BlockContentDigests, BlockStore, Transforms, WriteDurability, WriteFailure } from "../index.js";
2
+ import type { BlockBaseRevs } from "../network/struct.js";
3
+ import { Log } from "../log/log.js";
4
+ import { Tracker } from "../transform/tracker.js";
5
+ import type { BasePins } from "../transform/base-pins.js";
6
+ import type { CacheSource } from "../transform/cache-source.js";
7
+ import { copyTransforms } from "../transform/helpers.js";
8
+ import { describeTouchedBlocks, baseRevsField } from "../transform/digest.js";
9
+ import { oversizedBlock } from "../network/block-size.js";
10
+ import { mergeDurability } from "../network/durability.js";
11
+ import { highestStaleAt, isExpiryFailure, isRejectedFailure, isShortfallFailure } from "../network/stale-failure.js";
12
+ import { TransactionExpiredError } from "../network/transaction-expired-error.js";
13
+ import { PromiseShortfallError } from "../network/promise-shortfall-error.js";
14
+ import { ValidityRejectedError } from "../network/validity-rejected-error.js";
15
+ import { SyncRetryExhaustedError, SyncRevisionStalledError, TornActionError } from "./struct.js";
16
+ import type { CollectionId, InFlightAttempt, SyncOptions } from "./struct.js";
17
+ import type { RefreshReport } from "./collection.js";
18
+ import { clampPriority, SlotHoldAfterLosses } from "../transaction/transaction.js";
19
+ import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
20
+ import type { JitteredBackoffConfig, RandFn } from "../utility/backoff.js";
21
+ import { createLogger } from "../logger.js";
22
+
23
+ /**
24
+ * The attempt machinery a collection's two write paths share — `Collection.syncAttempts`, which
25
+ * commits the open segment, and `Collection.commitHead`, which commits the head sealed segment
26
+ * (docs/offline-sync.md §One entry, one network transaction). Each path decides WHAT an attempt
27
+ * sends and what to do with a commit; everything about HOW an attempt is built and accounted for
28
+ * is here, so the two cannot drift: the per-attempt tracker and log append
29
+ * ({@link stageAttempt}), the retained attempt a refresh may finish from
30
+ * ({@link retainedAttempt}), and the retry accounting with its stall check and its
31
+ * refresh-until-settled loop ({@link AttemptCycle}). The rules each step enforces are spelled out
32
+ * in docs/internals.md under *Key Invariants* ("The writer's retry finishes its own half-landed
33
+ * action", "A write's log append is a fixed function of the write and the revision it is
34
+ * requesting", "Monotonicity is also what makes a wedged view detectable").
35
+ */
36
+
37
+ const log = createLogger('collection');
38
+
39
+ /** Default base backoff (and historical fixed delay) between write retries, in ms. */
40
+ const PendingRetryDelayMs = 100;
41
+ /** Default max consecutive no-progress stale-failure retries before a write gives up. */
42
+ const DefaultMaxAttempts = 10;
43
+ /** Default ceiling on a single exponential-backoff sleep, in ms. */
44
+ const DefaultMaxBackoffMs = 5000;
45
+ /** Default consecutive stalled refreshes — ones that moved this collection's revision NOWHERE
46
+ * while a responder CONFIRMED a revision at or above the one being requested — before a write
47
+ * gives up. Two, not one, so a single transiently-lagging read is absorbed.
48
+ *
49
+ * NOTE: two is a judgement call, not a measurement — nothing here counts how often a legitimate
50
+ * loser reads a view that moves nowhere for two consecutive rounds. The check only strikes when the
51
+ * refresh made NO progress at all, so a client merely catching up is already excluded; if a
52
+ * spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
53
+ const DefaultMaxStalledAttempts = 2;
54
+
55
+ /**
56
+ * The sleep before the refresh that follows a write's `failures`-th consecutive loss.
57
+ *
58
+ * The jittered backoff exists to separate a herd of writers that lost the SAME race, so they do not
59
+ * re-collide on the next tick. A loss to a rival that had already committed (the refusal carried a
60
+ * confirmed `staleAt`) is not a herd, and once this write has taken `SlotHoldAfterLosses` such losses
61
+ * (`confirmedLosses`, counted over the whole write as a member counts its stale refusals of one
62
+ * action) the members that refused it hold the next slot for it — so sleeping only spends that hold.
63
+ * Both sides count the same refusals against the same constant, so the writer stops sleeping on the
64
+ * loss after which a member holds, with nothing said over the wire. A loss the latest refusal did not
65
+ * confirm (`latestConfirmed` false — a race against something still in flight) keeps the jitter.
66
+ *
67
+ * NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member the
68
+ * refused pends did not reach) this retries without sleeping and so spends the same attempt budget
69
+ * sooner, at about twice one writer's pend rate (measured in the plan stage of
70
+ * `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it only
71
+ * delayed the same give-up.
72
+ */
73
+ function retryBackoffMs(failures: number, confirmedLosses: number, latestConfirmed: boolean,
74
+ config: JitteredBackoffConfig, rand?: RandFn): number {
75
+ if (latestConfirmed && confirmedLosses >= SlotHoldAfterLosses) {
76
+ return 0;
77
+ }
78
+ return jitteredBackoffMs(failures - 1, config, rand);
79
+ }
80
+
81
+ /** What a write is identified as on the wire: the action id every attempt of it reuses, and the
82
+ * log entry's timestamp. Both are minted once per WRITE, never per attempt, so a retry at the same
83
+ * revision appends the same entry (docs/internals.md, "A write's log append is a fixed function of
84
+ * the write and the revision it is requesting"). */
85
+ export type WriteIdentity = {
86
+ readonly actionId: ActionId;
87
+ readonly timestamp: number;
88
+ };
89
+
90
+ /** What one attempt is built from: the actions its log entry records, the transforms they produced
91
+ * over the committed state, and the bases those transforms were computed against. For a sync these
92
+ * are the open segment and the live tracker's; for a head commit the head segment and the head
93
+ * tracker's (see `Collection.commitHead`). */
94
+ export type AttemptTarget<TAction> = {
95
+ readonly actions: readonly Action<TAction>[];
96
+ readonly transforms: Transforms;
97
+ readonly pins: BasePins;
98
+ };
99
+
100
+ /** One attempt as it goes on the wire: a per-attempt tracker holding the target's transforms plus
101
+ * the log append, the revision it requests, and the declarations the pend and commit carry. */
102
+ export type StagedAttempt = {
103
+ readonly tracker: Tracker<IBlock>;
104
+ readonly rev: number;
105
+ /** The log tail block the entry was appended to, committed first. */
106
+ readonly tailId: BlockId;
107
+ readonly blockDigests: BlockContentDigests | undefined;
108
+ readonly baseRevs: BlockBaseRevs;
109
+ };
110
+
111
+ /**
112
+ * Build one attempt: a tracker seeded with a COPY of the target's transforms, so the target is
113
+ * untouched whatever becomes of the attempt, with the log entry for `target.actions` appended at
114
+ * `rev` under `identity`, then the content declarations and the size check.
115
+ *
116
+ * The per-attempt tracker SHARES the target's pin store: it inherits the transforms by copy and
117
+ * never stages the data-block updates itself, so without the shared pins the digest pass could only
118
+ * describe bases still resident in the read cache. NOTE: an attempt that ends in a stale failure is
119
+ * abandoned WITHOUT reset(), so the log tail/header bases its append pinned stay in the shared
120
+ * store until the target tracker's next reset. Bounded and harmless — a few log blocks per
121
+ * attempt, capped by `maxAttempts`, each re-validated against the source generation before any use
122
+ * — but if an attempt ever grows a per-attempt footprint beyond the log blocks, reset the abandoned
123
+ * tracker on the stale-failure branch instead of letting the residue ride to the end of the write.
124
+ *
125
+ * The append is a fixed function of the write and `rev`: the timestamp is the identity's, and any
126
+ * data block the append has to mint takes the id `newDataBlockId` hands out — the caller's
127
+ * `Collection.logAppendBlockIds`, which replays the ids an earlier attempt at this revision minted.
128
+ *
129
+ * The declarations are computed AFTER the append so the log tail and header transforms the attempt
130
+ * just staged are digested too, and purely locally — an id whose base is not cached is omitted and
131
+ * falls back to corroboration on the member side. NOTE: recomputed from scratch on every attempt,
132
+ * so a write that loses N races pays N full hashing passes over its touched blocks. Unmeasured and
133
+ * cheap relative to the round trips it is retrying; if a high-contention write ever shows digest
134
+ * hashing in a profile, memoize per (id, staged ops).
135
+ *
136
+ * @throws BlockTooLargeError when a touched block would be left larger than one message can carry —
137
+ * before anything is sent, judged apart from the declaration (which never throws), and unretried by
138
+ * either write path, since a later attempt would leave the same block. */
139
+ export async function stageAttempt<TAction>(
140
+ collectionId: CollectionId,
141
+ cache: CacheSource<IBlock>,
142
+ target: AttemptTarget<TAction>,
143
+ identity: WriteIdentity,
144
+ rev: number,
145
+ newDataBlockId: (store: BlockStore<IBlock>) => () => BlockId,
146
+ ): Promise<StagedAttempt> {
147
+ const tracker = new Tracker(cache, copyTransforms(target.transforms), target.pins);
148
+ const collectionLog = await Log.open<Action<TAction>>(tracker, collectionId, { newDataBlockId: newDataBlockId(tracker) });
149
+ if (!collectionLog) {
150
+ throw new Error(`Log not found for collection ${collectionId}`);
151
+ }
152
+ const added = await collectionLog.addActions([...target.actions], identity.actionId, rev,
153
+ () => tracker.transformedBlockIds(), { timestamp: identity.timestamp });
154
+ const touched = await describeTouchedBlocks(tracker, tracker.transformedBlockIds());
155
+ const tooLarge = oversizedBlock(touched.bytes);
156
+ if (tooLarge) throw tooLarge;
157
+ return {
158
+ tracker,
159
+ rev,
160
+ tailId: added.tailPath.block.header.id,
161
+ blockDigests: touched.digests,
162
+ // The base each update-only block's operations were computed against, for the pend (see
163
+ // `PendRequest.baseRevs`): the pinned revisions, which the caller re-judged before staging.
164
+ baseRevs: tracker.stagedBaseRevs(tracker.transformedBlockIds()),
165
+ };
166
+ }
167
+
168
+ /** Exactly what an attempt sends under `actionId`, for `Collection.retainInFlightAttempt` and for
169
+ * the write-ahead hook (`HeadCommitOptions.recordAttempt`): a refused commit is not proof nothing
170
+ * landed, and if the refresh that follows finds the attempt's own log entry these are the
171
+ * transforms that finish the action. The attempt's tracker is abandoned once this is taken, so its
172
+ * transforms are handed over as they are, uncopied.
173
+ * NOTE: the same object reaches the write-ahead hook and is retained for the re-send; nothing in
174
+ * the tree mutates it (a fold clones the inserts and applies the updates to the cache's own blocks),
175
+ * but a hook that mutated its argument would corrupt the re-send. If a hook ever needs to, copy
176
+ * here (`copyTransforms`) rather than at the hook. */
177
+ export function retainedAttempt(staged: StagedAttempt, actionId: ActionId): InFlightAttempt {
178
+ return {
179
+ actionId,
180
+ rev: staged.rev,
181
+ transforms: staged.tracker.transforms,
182
+ tailId: staged.tailId,
183
+ ...(staged.blockDigests === undefined ? {} : { blockDigests: staged.blockDigests }),
184
+ ...baseRevsField(staged.baseRevs),
185
+ };
186
+ }
187
+
188
+ /**
189
+ * The inverse of {@link retainedAttempt}, for a head commit opened with `resume`
190
+ * (`HeadCommitOptions.resume`): the attempt an earlier cycle recorded, as a {@link StagedAttempt}
191
+ * that sends exactly what that cycle sent, while `rev` — the revision the next attempt would
192
+ * request — equals the recorded one; `undefined` otherwise, and the attempt is built as usual.
193
+ * The tracker is over the handle's read cache, with a COPY of the recorded transforms (a fold
194
+ * resets it and folds what it holds, and the record stays the caller's) and pins of its own, since
195
+ * it stages nothing.
196
+ *
197
+ * The recorded attempt IS the attempt that may have landed: it can sit on a minority of its cohort,
198
+ * hidden from the refresh by a coordinator lagging that cohort, and storage accepts a retry of one
199
+ * `(action id, revision)` as the same write, so anything but these transforms under that pair forks
200
+ * the block on the machines that hold the first (docs/offline-sync.md §Identity across attempts).
201
+ * Three departures from how an attempt is otherwise built are deliberate:
202
+ * - **Keyed on `resume`, not on the retained attempt.** Within one cycle a retry at an unchanged
203
+ * revision reuses the head tracker and the minted log block ids, so it is byte-identical already,
204
+ * and keeps the re-judging of its base pins before each attempt (`Collection.headTrackerFor`).
205
+ * The recorded attempt is re-sent as it was, unjudged: a base that moved under it is judged by
206
+ * storage's own base guard, which refuses it on every machine whose base moved — the exception
207
+ * `Collection.completeOwnEntry`'s re-send of a retained attempt already takes.
208
+ * - **No invented baseline.** A head tracker over a collection with no committed revision starts
209
+ * from the invented baseline so the header, log block and root ride in the head's commit; the
210
+ * recorded transforms already carry those inserts.
211
+ * - **Only equality switches it on.** A refresh that moves the revision past `attempt.rev` adopted a
212
+ * rival's entry there, which proves the recorded attempt lost at that revision, and the write is
213
+ * rebuilt at the higher one. A revision BELOW it is not reachable inside one process (the held
214
+ * revision never lowers); a new handle after a restart that reaches one has read a view behind the
215
+ * attempt, and the ordinary build handles it as it would without `resume` — the recorded revision
216
+ * is not adopted to get there.
217
+ */
218
+ export function resumedAttempt(cache: CacheSource<IBlock>, attempt: InFlightAttempt | undefined, rev: number): StagedAttempt | undefined {
219
+ if (attempt === undefined || attempt.rev !== rev) {
220
+ return undefined;
221
+ }
222
+ return {
223
+ tracker: new Tracker(cache, copyTransforms(attempt.transforms)),
224
+ rev: attempt.rev,
225
+ tailId: attempt.tailId,
226
+ blockDigests: attempt.blockDigests,
227
+ baseRevs: attempt.baseRevs ?? {},
228
+ };
229
+ }
230
+
231
+ /** The refresh a write path runs between attempts — `Collection.updateInternal`, under the
232
+ * collection latch, with the write's action marked in flight so the refresh recognises the write's
233
+ * own log entry. `lastChance` says the budget ends with this round; `tailFloor` is the highest
234
+ * revision a responder confirmed is taken, for the tail read. */
235
+ export type Refresh = (report: RefreshReport, lastChance: boolean, tailFloor: number | undefined) => Promise<void>;
236
+
237
+ /**
238
+ * The retry accounting of one write's attempt cycle: how many consecutive attempts have been
239
+ * refused, what the responders confirmed, whether the refresh in between moved the collection, and
240
+ * who holds what the write has committed so far. One instance per write; the write path drives it
241
+ * in order — {@link beforeAttempt}, then either {@link recordRefusal} followed by
242
+ * {@link refreshUntilSettled}, or {@link recordCommit}.
243
+ */
244
+ export class AttemptCycle {
245
+ private readonly maxAttempts: number;
246
+ private readonly maxStalledAttempts: number;
247
+ private readonly backoff: JitteredBackoffConfig;
248
+ private readonly deadlineMs: number | undefined;
249
+ private readonly signal: AbortSignal | undefined;
250
+ private readonly rand: RandFn | undefined;
251
+ private readonly startedAt = Date.now();
252
+
253
+ // Count of consecutive stale failures that made no forward progress. Reset to 0 on every
254
+ // commit, so the cap bounds only a persistently-failing write — a legitimate large multi-batch
255
+ // sync (which iterates many times committing progress) never trips it.
256
+ private consecutiveFailures = 0;
257
+ private lastReason: string | undefined;
258
+ // Highest confirmed revision any responder has reported holding, accumulated with the
259
+ // codebase's single rule for picking among several candidates. Reported on the error, and
260
+ // the evidence the stall check reasons from.
261
+ private lastStaleAt: { blockId: BlockId; rev: number } | undefined;
262
+ // Whether the failure most recently handled carried its OWN staleAt. A strike needs the
263
+ // responder to have re-confirmed the number this round, not merely an older observation
264
+ // left standing in `lastStaleAt`.
265
+ private lastFailureConfirmedStaleAt = false;
266
+ // Refused attempts of this write that carried their own staleAt. Never reset: a member counts
267
+ // its stale refusals per action for the action's whole life, and this is the writer's view of
268
+ // that count (see retryBackoffMs).
269
+ private confirmedLosses = 0;
270
+ // Consecutive refreshes that moved the requested revision nowhere at all while a confirmed
271
+ // revision stood at or above it.
272
+ private consecutiveStalls = 0;
273
+ // The revision the PREVIOUS attempt would have requested, so the stall check can tell a
274
+ // refresh that moved nowhere from one that is still climbing toward the confirmed number.
275
+ private previousRequestedRev: number | undefined;
276
+
277
+ /** Who holds what the write has committed so far. Stays `undefined` while nothing has been
278
+ * committed, which is also the answer when no attempt runs at all — a write with nothing staged
279
+ * writes nothing, and there is no durability to fabricate for it. Refused attempts never
280
+ * contribute: a refused attempt left nothing behind, so the answer is the committing one's. A
281
+ * write that commits more than once folds them with `mergeDurability`, whose scalar answer is
282
+ * the WEAKEST: one batch that only reached the writer makes the whole write only-on-the-writer,
283
+ * and reporting the last batch's class instead would show such a write as saved. */
284
+ durability: WriteDurability | undefined;
285
+
286
+ constructor(
287
+ private readonly collectionId: CollectionId,
288
+ private readonly instanceTag: string,
289
+ options?: SyncOptions,
290
+ ) {
291
+ this.maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
292
+ this.maxStalledAttempts = options?.maxStalledAttempts ?? DefaultMaxStalledAttempts;
293
+ this.backoff = {
294
+ baseMs: options?.baseBackoffMs ?? PendingRetryDelayMs,
295
+ capMs: options?.maxBackoffMs ?? DefaultMaxBackoffMs,
296
+ };
297
+ this.deadlineMs = options?.deadlineMs;
298
+ this.signal = options?.signal;
299
+ this.rand = options?.rand;
300
+ }
301
+
302
+ /** The aged retry priority for the next attempt, from the consecutive-failure count: a write
303
+ * that keeps losing concurrent races out-ranks fresh (priority-0) rivals in the cluster's
304
+ * `resolveRace` (fairness-only, capped at `MaxPriority`). A first attempt is priority 0, so the
305
+ * common pend is unchanged. */
306
+ priority(): number {
307
+ return clampPriority(this.consecutiveFailures);
308
+ }
309
+
310
+ /**
311
+ * The checks that run before an attempt is built: the abort signal, the wall-clock deadline, and
312
+ * the stall check. The deadline comes before the stall check deliberately: it is the documented
313
+ * outer bound, so a write that is both past it and stalled reports the deadline.
314
+ *
315
+ * The stall check asks: can the attempt about to run possibly differ from the one that just
316
+ * failed? Only when BOTH of these hold is the answer provably no:
317
+ *
318
+ * - `requestedRev` is at or below one a responder CONFIRMED is taken. A producer sets `staleAt`
319
+ * only after reading that revision as durably held by someone else out of its own storage,
320
+ * revisions are one per-collection counter that every commit touches, and a confirmed
321
+ * revision never becomes un-taken (invalidation takes a NEW slot). So the request is already
322
+ * lost before it is sent.
323
+ * - The refresh in between moved the collection nowhere. `advanceContext` never lowers the
324
+ * held revision, so "nowhere" is exactly `requestedRev` unchanged since the last attempt. A
325
+ * refresh that moved forward WITHOUT clearing the confirmed number is a collection still
326
+ * climbing (it read a replica that lags the holder — the same partial catch-up
327
+ * `reportShortfall` exists to report), and its next attempt is a genuinely different request
328
+ * that may yet win.
329
+ *
330
+ * This is NOT a second answer to "is this failure retryable?" — `isConflictFailure` remains the
331
+ * sole rule for that, untouched. It only ever stops a loop that rule had already decided to
332
+ * continue.
333
+ *
334
+ * @param requestedRev the revision the attempt would request (`Collection.getNextRev`).
335
+ * @param heldRev the revision the collection holds, for the stall report.
336
+ * @throws AbortError, {@link SyncRetryExhaustedError} (deadline), {@link SyncRevisionStalledError}. */
337
+ beforeAttempt(requestedRev: number, heldRev: number | undefined): void {
338
+ if (this.signal?.aborted) {
339
+ throw makeAbortError(this.signal);
340
+ }
341
+ if (this.pastDeadline()) {
342
+ throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason ?? 'deadline exceeded', this.lastStaleAt);
343
+ }
344
+ const refreshMoved = this.previousRequestedRev !== undefined && requestedRev > this.previousRequestedRev;
345
+ this.previousRequestedRev = requestedRev;
346
+ if (this.lastStaleAt === undefined) {
347
+ return;
348
+ }
349
+ if (requestedRev > this.lastStaleAt.rev || refreshMoved) {
350
+ // Either the refresh adopted a revision above the confirmed one — ordinary contention,
351
+ // where the rival's commit is exactly what we just read — or it made partial forward
352
+ // progress. Both mean the next attempt differs. Not a stall.
353
+ this.consecutiveStalls = 0;
354
+ } else if (this.lastFailureConfirmedStaleAt) {
355
+ this.consecutiveStalls++;
356
+ if (log.enabled) {
357
+ log('collection:sync-stalled id=%s tag=%s heldRev=%s requestedRev=%d staleBlock=%s staleRev=%d strike=%d of=%d',
358
+ this.collectionId, this.instanceTag, heldRev ?? 'none', requestedRev,
359
+ this.lastStaleAt.blockId, this.lastStaleAt.rev, this.consecutiveStalls, this.maxStalledAttempts);
360
+ }
361
+ // Two strikes, not one: a legitimate loser can transiently read a view that has not yet
362
+ // caught up with the rival's commit, which looks identical for one round.
363
+ if (this.consecutiveStalls >= this.maxStalledAttempts) {
364
+ throw new SyncRevisionStalledError(this.collectionId, this.consecutiveFailures, this.lastStaleAt,
365
+ requestedRev, heldRev, this.lastReason);
366
+ }
367
+ }
368
+ // Else: the responder told us nothing new this round. No strike, and no reset either — the
369
+ // budget stays bounded by maxAttempts.
370
+ }
371
+
372
+ /**
373
+ * Account for a refused attempt. The three refusals a retry can never get past are thrown by
374
+ * name, with nothing on the collection touched (the attempt ran on its own tracker), so the
375
+ * staged actions are still staged for a later write: an expiry, because the next attempt's
376
+ * expiration comes from the same clock; a promise shortfall, because every coordinator of the
377
+ * cohort meets the same silent members (only a pend answers this way, so nothing of the attempt
378
+ * was stored); a validity rejection, because the next attempt sends the same content to the same
379
+ * checks. Everything else is a stale failure counted against the no-progress budget, so a
380
+ * transactor that persistently rejects can no longer hold the collection latch forever.
381
+ *
382
+ * NOTE: an expiry refusing a commit SWEEP (`NetworkTransactor.commitTailThenSweep`) arrives after
383
+ * the log tail was stored, and no refresh follows this throw, so that attempt's log entry stays
384
+ * behind with none of its data — the state the budget-exhausted exit leaves too. Needs a sweep
385
+ * cohort whose clocks disagree while the tail's agree; tracked with the other leftover-entry
386
+ * exits in tickets/backlog/bug-a-refused-write-can-leave-its-log-entry-behind.
387
+ *
388
+ * NOTE: the budget also bounds the legitimate `pending`-wait case (retrying the same action while
389
+ * another commit is in flight), which used to retry indefinitely. Default 10 attempts ≈ 21s of
390
+ * exponential backoff. If a high-contention workload legitimately needs to wait longer for a
391
+ * pending commit to clear, raise maxAttempts for that caller.
392
+ *
393
+ * NOTE: no refresh follows the LAST budgeted attempt, so if that attempt's log tail landed nobody
394
+ * finds out: the caller gets plain exhaustion over a log that holds an entry for this write. The
395
+ * write is still never reported saved, which is the rule; what is lost is the more specific name
396
+ * (TornActionError). The leftover entry itself is tracked in
397
+ * tickets/backlog/bug-a-refused-write-can-leave-its-log-entry-behind.
398
+ *
399
+ * @throws TransactionExpiredError, PromiseShortfallError, ValidityRejectedError, SyncRetryExhaustedError */
400
+ recordRefusal(refusal: WriteFailure): void {
401
+ if (isExpiryFailure(refusal)) {
402
+ throw new TransactionExpiredError(refusal.expired);
403
+ }
404
+ if (isShortfallFailure(refusal)) {
405
+ throw new PromiseShortfallError(refusal.shortfall);
406
+ }
407
+ if (isRejectedFailure(refusal)) {
408
+ throw new ValidityRejectedError(refusal.rejected);
409
+ }
410
+ this.consecutiveFailures++;
411
+ this.lastReason = refusal.reason ?? this.lastReason;
412
+ // Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
413
+ // responder reporting a LOWER number understates the binding constraint. Same rule the
414
+ // producers and the transactor's aggregation already use.
415
+ this.lastStaleAt = highestStaleAt([this.lastStaleAt, refusal.staleAt]);
416
+ this.lastFailureConfirmedStaleAt = refusal.staleAt !== undefined;
417
+ if (this.lastFailureConfirmedStaleAt) this.confirmedLosses++;
418
+ if (this.consecutiveFailures >= this.maxAttempts) {
419
+ throw new SyncRetryExhaustedError(this.collectionId, this.consecutiveFailures, this.lastReason, this.lastStaleAt);
420
+ }
421
+ }
422
+
423
+ /** Account for a commit — by an attempt, or by a refresh that finished the write's own entry,
424
+ * which is a commit too and is reported and counted exactly like one (see {@link durability}).
425
+ * Forward progress: the no-progress budget and the stall evidence start over. */
426
+ recordCommit(durability: WriteDurability): void {
427
+ this.durability = this.durability === undefined ? durability : mergeDurability([this.durability, durability]);
428
+ this.consecutiveFailures = 0;
429
+ this.lastReason = undefined;
430
+ this.lastStaleAt = undefined;
431
+ this.lastFailureConfirmedStaleAt = false;
432
+ this.consecutiveStalls = 0;
433
+ }
434
+
435
+ /**
436
+ * Back off, refresh, and keep refreshing while the refresh reports that this write's own
437
+ * half-landed action could not be finished YET. The write path must not fall through to a new
438
+ * attempt in that state: the action's log entry is already durable, and as soon as a refresh
439
+ * adopts a newer revision a new attempt appends a SECOND entry under the same action id — the
440
+ * duplicate `Collection.consumeOwnEntry` exists to prevent. (At an unchanged revision the rebuild
441
+ * is byte-identical — see `Collection.logAppendBlockIds` — so what is at stake is the second
442
+ * entry, not two versions of one tail block.) Each round is a failure against the same
443
+ * no-progress budget as a refused attempt, and the round that would spend the last of it is told
444
+ * so (`lastChance`), so it settles a half-landed write instead of asking for another round and
445
+ * the error that escapes says whether the write can still land. (A deadline cannot be foreseen
446
+ * the same way; a write given up on it escapes unsettled, as `final: false`.)
447
+ *
448
+ * The backoff before each round grows exponentially from the base delay up to the cap, with
449
+ * proportional random jitter so a herd of clients that lost the same race does not re-collide on
450
+ * the next tick (see utility/backoff.ts) — except once this write has lost to committed rivals
451
+ * often enough that a member holds the next slot for it ({@link retryBackoffMs}). The abortable
452
+ * sleep lets an aborted write reject promptly instead of finishing the sleep. A refresh that
453
+ * follows NO refused attempt — the one `Collection.commitHead` opens its cycle with — sleeps not
454
+ * at all: there is no race to space out.
455
+ * NOTE: the `missing`/`reason` conflict paths pay this backoff too (they used to retry with zero
456
+ * delay); that is what stops the persistent-`reason` hot spin. If a high-contention workload ever
457
+ * shows this base delay as recovery latency, lower baseBackoffMs for that caller rather than
458
+ * reintroducing the zero-delay retry.
459
+ *
460
+ * The highest revision a responder confirmed is taken rides out as the floor of the refresh's
461
+ * tail read, so a machine that has not caught up to it is not the last one asked (see
462
+ * `Collection.updateInternal`).
463
+ *
464
+ * @returns whether the refresh FINISHED the write's own log entry (`RefreshReport.ownEntryFinished`)
465
+ * — the write is saved and the entry consumed, whether or not the refresh learned who holds it.
466
+ * @throws TornActionError with any reason but an unsettled `completion-refused`, and that one too
467
+ * once the budget or the deadline is spent; whatever else the refresh throws. */
468
+ async refreshUntilSettled(refresh: Refresh): Promise<boolean> {
469
+ // Whether the failure just handled was a loss to a rival already durably committed. Only the
470
+ // attempt's own refusal can say so; a refresh refused for a cause that can clear (below) is a
471
+ // race against something still in flight, and clears it.
472
+ let latestConfirmed = this.lastFailureConfirmedStaleAt;
473
+ for (;;) {
474
+ const delay = this.consecutiveFailures === 0
475
+ ? 0
476
+ : retryBackoffMs(this.consecutiveFailures, this.confirmedLosses, latestConfirmed, this.backoff, this.rand);
477
+ await abortableDelay(delay, this.signal);
478
+ try {
479
+ const report: RefreshReport = {};
480
+ await refresh(report, this.consecutiveFailures + 1 >= this.maxAttempts, this.lastStaleAt?.rev);
481
+ const completed = report.ownEntryFinished?.durability;
482
+ if (completed !== undefined) {
483
+ this.recordCommit(completed);
484
+ }
485
+ return report.ownEntryFinished !== undefined;
486
+ } catch (err) {
487
+ if (!(err instanceof TornActionError) || err.reason !== 'completion-refused') {
488
+ throw err;
489
+ }
490
+ // Refused for a cause that can clear. The error already names the write as torn, which
491
+ // is the truth if the budget ends here — so it, not a plain exhaustion, is what escapes:
492
+ // the log holds an entry for a write that was not saved.
493
+ this.consecutiveFailures++;
494
+ this.lastReason = err.detail;
495
+ latestConfirmed = false;
496
+ if (this.consecutiveFailures >= this.maxAttempts || this.pastDeadline()) {
497
+ throw err;
498
+ }
499
+ }
500
+ }
501
+ }
502
+
503
+ private pastDeadline(): boolean {
504
+ return this.deadlineMs !== undefined && Date.now() - this.startedAt >= this.deadlineMs;
505
+ }
506
+ }