@optimystic/db-core 0.24.2 → 0.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/blocks/helpers.d.ts +5 -0
- package/dist/src/blocks/helpers.d.ts.map +1 -1
- package/dist/src/blocks/helpers.js +12 -0
- package/dist/src/blocks/helpers.js.map +1 -1
- package/dist/src/cluster/membership.d.ts +7 -0
- package/dist/src/cluster/membership.d.ts.map +1 -1
- package/dist/src/cluster/membership.js +12 -8
- package/dist/src/cluster/membership.js.map +1 -1
- package/dist/src/cluster/structs.d.ts +27 -1
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/cluster/structs.js.map +1 -1
- package/dist/src/collection/action.d.ts +14 -0
- package/dist/src/collection/action.d.ts.map +1 -1
- package/dist/src/collection/action.js +16 -1
- package/dist/src/collection/action.js.map +1 -1
- package/dist/src/collection/collection.d.ts +273 -4
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +427 -30
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts +16 -1
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +19 -0
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/network/i-repo.d.ts +11 -1
- package/dist/src/network/i-repo.d.ts.map +1 -1
- package/dist/src/network/stale-failure.d.ts +21 -0
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +22 -0
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/network/struct.d.ts +66 -21
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/testing/test-transactor.d.ts +22 -0
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +44 -5
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +20 -0
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +256 -113
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transaction/operations-hash.d.ts +1 -1
- package/dist/src/transaction/operations-hash.js +1 -1
- package/dist/src/transaction/transaction.d.ts +4 -2
- package/dist/src/transaction/transaction.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +21 -1
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +122 -29
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor-source.d.ts +11 -5
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +16 -8
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/dist/src/transform/cache-source.d.ts +13 -0
- package/dist/src/transform/cache-source.d.ts.map +1 -1
- package/dist/src/transform/cache-source.js +18 -0
- package/dist/src/transform/cache-source.js.map +1 -1
- package/dist/src/transform/digest.d.ts +18 -0
- package/dist/src/transform/digest.d.ts.map +1 -0
- package/dist/src/transform/digest.js +65 -0
- package/dist/src/transform/digest.js.map +1 -0
- package/dist/src/transform/index.d.ts +1 -0
- package/dist/src/transform/index.d.ts.map +1 -1
- package/dist/src/transform/index.js +1 -0
- package/dist/src/transform/index.js.map +1 -1
- package/dist/src/transform/tracker.d.ts +16 -0
- package/dist/src/transform/tracker.d.ts.map +1 -1
- package/dist/src/transform/tracker.js +40 -1
- package/dist/src/transform/tracker.js.map +1 -1
- package/dist/src/utility/canonical-json.d.ts +11 -0
- package/dist/src/utility/canonical-json.d.ts.map +1 -0
- package/dist/src/utility/canonical-json.js +15 -0
- package/dist/src/utility/canonical-json.js.map +1 -0
- package/dist/src/utility/lru-map.d.ts +2 -0
- package/dist/src/utility/lru-map.d.ts.map +1 -1
- package/dist/src/utility/lru-map.js +4 -0
- package/dist/src/utility/lru-map.js.map +1 -1
- package/package.json +2 -1
- package/src/blocks/helpers.ts +26 -13
- package/src/cluster/membership.ts +87 -85
- package/src/cluster/structs.ts +28 -1
- package/src/collection/action.ts +17 -0
- package/src/collection/collection.ts +1129 -688
- package/src/collections/tree/tree.ts +341 -320
- package/src/index.ts +24 -23
- package/src/network/i-repo.ts +59 -46
- package/src/network/stale-failure.ts +67 -43
- package/src/network/struct.ts +332 -270
- package/src/testing/test-transactor.ts +680 -638
- package/src/transaction/coordinator.ts +1266 -1110
- package/src/transaction/operations-hash.ts +1 -1
- package/src/transaction/transaction.ts +4 -2
- package/src/transactor/network-transactor.ts +123 -32
- package/src/transactor/transactor-source.ts +17 -9
- package/src/transform/cache-source.ts +20 -0
- package/src/transform/digest.ts +72 -0
- package/src/transform/index.ts +1 -0
- package/src/transform/tracker.ts +42 -1
- package/src/utility/canonical-json.ts +16 -0
- package/src/utility/lru-map.ts +5 -0
|
@@ -1,1110 +1,1266 @@
|
|
|
1
|
-
import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, CommitRequest, ActionId } from "../index.js";
|
|
2
|
-
import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions, ReadDependency } from "./transaction.js";
|
|
3
|
-
import type { PeerId } from "../network/types.js";
|
|
4
|
-
import { isConflictFailure } from "../network/stale-failure.js";
|
|
5
|
-
import type { Collection } from "../collection/collection.js";
|
|
6
|
-
import type { SyncOptions } from "../collection/index.js";
|
|
7
|
-
import { isTransactionExpired, clampPriority } from "./transaction.js";
|
|
8
|
-
import { Log } from "../log/log.js";
|
|
9
|
-
import { blockIdsForTransforms } from "../transform/helpers.js";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
*
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
* -
|
|
56
|
-
* -
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
private readonly
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
* @param
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
* `
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
* @param
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
const
|
|
147
|
-
const
|
|
148
|
-
const
|
|
149
|
-
const
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
//
|
|
154
|
-
//
|
|
155
|
-
|
|
156
|
-
let
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
//
|
|
181
|
-
//
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
//
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
//
|
|
267
|
-
//
|
|
268
|
-
//
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
*
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
const
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
//
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
//
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
//
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
//
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
//
|
|
829
|
-
//
|
|
830
|
-
//
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
);
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
const
|
|
878
|
-
|
|
879
|
-
//
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
}
|
|
1006
|
-
|
|
1007
|
-
if
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
}
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1
|
+
import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, CommitRequest, ActionId } from "../index.js";
|
|
2
|
+
import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions, ReadDependency } from "./transaction.js";
|
|
3
|
+
import type { PeerId } from "../network/types.js";
|
|
4
|
+
import { isConflictFailure } from "../network/stale-failure.js";
|
|
5
|
+
import type { Collection } from "../collection/collection.js";
|
|
6
|
+
import type { SyncOptions } from "../collection/index.js";
|
|
7
|
+
import { isTransactionExpired, clampPriority } from "./transaction.js";
|
|
8
|
+
import { Log } from "../log/log.js";
|
|
9
|
+
import { blockIdsForTransforms } from "../transform/helpers.js";
|
|
10
|
+
import { computeBlockContentDigests, blockDigestsField } from "../transform/digest.js";
|
|
11
|
+
import { collectOperations, hashOperations } from "./operations-hash.js";
|
|
12
|
+
import { CoordinatorPartialCommitError, CoordinatorStaleLossError } from "./errors.js";
|
|
13
|
+
import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
|
|
14
|
+
import { createLogger } from "../logger.js";
|
|
15
|
+
|
|
16
|
+
const log = createLogger('trx:coordinator');
|
|
17
|
+
|
|
18
|
+
/** Default max consecutive clean-stale-loss retries before {@link TransactionCoordinator.commit}
|
|
19
|
+
* gives up. Mirrors the single-collection sync default so the two retry loops share one policy. */
|
|
20
|
+
const DefaultMaxAttempts = 10;
|
|
21
|
+
/** Default base backoff (ms) before the first commit retry. */
|
|
22
|
+
const DefaultBaseBackoffMs = 100;
|
|
23
|
+
/** Default ceiling (ms) on a single commit-retry backoff sleep. */
|
|
24
|
+
const DefaultMaxBackoffMs = 5000;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* A pend that failed. `conflict` marks the retryable class — an optimistic-concurrency collision that
|
|
28
|
+
* a re-read + re-pend can clear — as decided by `isConflictFailure` over the failure response. A hard
|
|
29
|
+
* rejection (storage full, policy) is NOT a conflict and is not worth re-driving. Thrown by
|
|
30
|
+
* {@link TransactionCoordinator.pendCollection} so the fan-out in pendPhase can settle it and read
|
|
31
|
+
* the flag off the rejection.
|
|
32
|
+
*/
|
|
33
|
+
class PendRejectedError extends Error {
|
|
34
|
+
constructor(
|
|
35
|
+
collectionId: CollectionId,
|
|
36
|
+
readonly conflict: boolean,
|
|
37
|
+
reason?: string,
|
|
38
|
+
/** Confirmed revision the responder holds, from `StaleFailure.staleAt`. Folded into the
|
|
39
|
+
* message because pendPhase collapses this error to its `.message` string, which is the only
|
|
40
|
+
* form that reaches an embedder through the transaction result's `error` field. */
|
|
41
|
+
staleAt?: { blockId: BlockId; rev: number },
|
|
42
|
+
) {
|
|
43
|
+
super(`Pend failed for collection ${collectionId}: ${reason ?? (conflict ? 'stale conflict' : 'rejected')}`
|
|
44
|
+
+ (staleAt ? ` (block ${staleAt.blockId} at rev ${staleAt.rev})` : ''));
|
|
45
|
+
this.name = 'PendRejectedError';
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Coordinates multi-collection transactions.
|
|
51
|
+
*
|
|
52
|
+
* This is the ONLY interface for all mutations (single or multi-collection).
|
|
53
|
+
*
|
|
54
|
+
* Responsibilities:
|
|
55
|
+
* - Manage collections (create as needed)
|
|
56
|
+
* - Apply actions to collections (run handlers, write to logs)
|
|
57
|
+
* - Commit transactions by running consensus phases (GATHER, PEND, COMMIT)
|
|
58
|
+
*/
|
|
59
|
+
export class TransactionCoordinator {
|
|
60
|
+
/** Per-stampId tracking: snapshot before first apply + accumulated actions for replay */
|
|
61
|
+
private stampData = new Map<string, {
|
|
62
|
+
order: number;
|
|
63
|
+
preSnapshot: Map<CollectionId, Transforms>;
|
|
64
|
+
actionBatches: CollectionActions[][];
|
|
65
|
+
}>();
|
|
66
|
+
private nextStampOrder = 0;
|
|
67
|
+
|
|
68
|
+
constructor(
|
|
69
|
+
private readonly transactor: ITransactor,
|
|
70
|
+
private readonly collections: Map<CollectionId, Collection<any>>
|
|
71
|
+
) {}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Apply actions to collections (called by engines during statement execution).
|
|
75
|
+
*
|
|
76
|
+
* This is the core method that engines call to apply actions to collections.
|
|
77
|
+
* Actions are tagged with the stamp ID and executed immediately through collections
|
|
78
|
+
* to update the local snapshot.
|
|
79
|
+
*
|
|
80
|
+
* @param actions - The actions to apply (per collection)
|
|
81
|
+
* @param stampId - The transaction stamp ID to tag actions with
|
|
82
|
+
*/
|
|
83
|
+
async applyActions(
|
|
84
|
+
actions: CollectionActions[],
|
|
85
|
+
stampId: string
|
|
86
|
+
): Promise<void> {
|
|
87
|
+
// On first call for this stampId, snapshot all collections for potential rollback
|
|
88
|
+
if (!this.stampData.has(stampId)) {
|
|
89
|
+
const snapshot = new Map<CollectionId, Transforms>();
|
|
90
|
+
for (const [id, col] of this.collections) {
|
|
91
|
+
snapshot.set(id, structuredClone(col.tracker.transforms));
|
|
92
|
+
}
|
|
93
|
+
this.stampData.set(stampId, {
|
|
94
|
+
order: this.nextStampOrder++,
|
|
95
|
+
preSnapshot: snapshot,
|
|
96
|
+
actionBatches: []
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
this.stampData.get(stampId)!.actionBatches.push(actions);
|
|
100
|
+
|
|
101
|
+
await this.applyActionsRaw(actions, stampId);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Apply actions without tracking (used internally and for replay during rollback).
|
|
106
|
+
*/
|
|
107
|
+
private async applyActionsRaw(
|
|
108
|
+
actions: CollectionActions[],
|
|
109
|
+
stampId: string
|
|
110
|
+
): Promise<void> {
|
|
111
|
+
for (const { collectionId, actions: collectionActions } of actions) {
|
|
112
|
+
const collection = this.collections.get(collectionId);
|
|
113
|
+
if (!collection) {
|
|
114
|
+
throw new Error(`Collection not found: ${collectionId}`);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
for (const action of collectionActions) {
|
|
118
|
+
const taggedAction = { ...(action as any), transaction: stampId };
|
|
119
|
+
await collection.act(taggedAction);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Commit a transaction with a bounded, jittered backoff retry around a CLEAN stale loss.
|
|
126
|
+
*
|
|
127
|
+
* The single-attempt work lives in {@link commitOnce}; this wrapper re-drives it when the attempt
|
|
128
|
+
* fails as a clean optimistic-concurrency loss ({@link CoordinatorStaleLossError} — nothing
|
|
129
|
+
* durably committed, every tracker restored to its pre-append state). Before each re-attempt it
|
|
130
|
+
* re-reads each collection to fresh revisions (so the retry pends against current state rather
|
|
131
|
+
* than immediately re-failing stale), then backs off with the same jitter policy as
|
|
132
|
+
* {@link Collection.sync}. Retry is bounded by `maxAttempts` and an optional wall-clock
|
|
133
|
+
* `deadlineMs`, and honours an abort `signal`.
|
|
134
|
+
*
|
|
135
|
+
* A {@link CoordinatorPartialCommitError} (a partial landing — some collection durably committed)
|
|
136
|
+
* is NOT retryable and escapes immediately: blindly retrying would re-log already-durable actions.
|
|
137
|
+
* Any other failure (expired transaction, unavailable transactor, unreachable cluster) also
|
|
138
|
+
* propagates without retry — only genuine clean stale losses are re-driven.
|
|
139
|
+
*
|
|
140
|
+
* Defaults are safe out of the box: a caller that passes no options gets bounded, jittered retry.
|
|
141
|
+
*
|
|
142
|
+
* @param transaction - The transaction to commit
|
|
143
|
+
* @param options - Retry knobs; shares the {@link SyncOptions} vocabulary with `Collection.sync`.
|
|
144
|
+
*/
|
|
145
|
+
async commit(transaction: Transaction, options?: SyncOptions): Promise<void> {
|
|
146
|
+
const maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
|
|
147
|
+
const baseBackoffMs = options?.baseBackoffMs ?? DefaultBaseBackoffMs;
|
|
148
|
+
const maxBackoffMs = options?.maxBackoffMs ?? DefaultMaxBackoffMs;
|
|
149
|
+
const deadlineMs = options?.deadlineMs;
|
|
150
|
+
const signal = options?.signal;
|
|
151
|
+
const startedAt = Date.now();
|
|
152
|
+
|
|
153
|
+
// Count of consecutive clean stale losses. There is no forward-progress notion here (a
|
|
154
|
+
// single commit either lands or it does not), so this simply bounds how many times we
|
|
155
|
+
// re-drive a losing transaction before surfacing a terminal error.
|
|
156
|
+
let staleLosses = 0;
|
|
157
|
+
let lastLoss: CoordinatorStaleLossError | undefined;
|
|
158
|
+
|
|
159
|
+
// Disposers for the in-flight marks {@link commitOnce} sets on each participant it latches.
|
|
160
|
+
// They must outlive the individual attempt: the inter-attempt refresh below is the ONLY
|
|
161
|
+
// reader of the mark, and it deliberately runs after the commit span released its latches
|
|
162
|
+
// (`Latches` is non-reentrant), so a clear tied to the latch would already have run. Hence
|
|
163
|
+
// the finally spans the WHOLE retry loop — every exit clears: return, stale-loss exhaustion,
|
|
164
|
+
// a partial landing, a hard error, an abort. Each disposer is id-guarded, so re-marking on a
|
|
165
|
+
// later attempt is harmless and a stale disposer cannot wipe a newer mark.
|
|
166
|
+
const inFlightDisposers: (() => void)[] = [];
|
|
167
|
+
try {
|
|
168
|
+
for (;;) {
|
|
169
|
+
if (signal?.aborted) {
|
|
170
|
+
throw makeAbortError(signal);
|
|
171
|
+
}
|
|
172
|
+
// Progress-agnostic ceiling: once we've taken at least one loss, give up if the
|
|
173
|
+
// wall-clock deadline passed (independent of the attempt cap).
|
|
174
|
+
if (deadlineMs !== undefined && lastLoss && Date.now() - startedAt >= deadlineMs) {
|
|
175
|
+
throw lastLoss;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Age the transaction's advisory priority by the number of losses taken so far, so a
|
|
179
|
+
// repeatedly-losing transaction out-ranks fresh (priority-0) rivals in the cluster's
|
|
180
|
+
// resolveRace. Fairness-only and capped at MaxPriority; excluded from the tx id / client
|
|
181
|
+
// signature, so bumping it here does not churn identity. Left untouched on the first
|
|
182
|
+
// attempt (staleLosses == 0) so the initial pend serializes exactly as before.
|
|
183
|
+
if (staleLosses > 0) {
|
|
184
|
+
transaction.priority = clampPriority(staleLosses);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
try {
|
|
188
|
+
await this.commitOnce(transaction, inFlightDisposers);
|
|
189
|
+
return;
|
|
190
|
+
} catch (err) {
|
|
191
|
+
// Only a CLEAN stale loss is retryable. A partial landing, an expired transaction, an
|
|
192
|
+
// unavailable transactor, etc. all propagate unchanged.
|
|
193
|
+
if (!(err instanceof CoordinatorStaleLossError)) {
|
|
194
|
+
throw err;
|
|
195
|
+
}
|
|
196
|
+
lastLoss = err;
|
|
197
|
+
staleLosses++;
|
|
198
|
+
if (staleLosses >= maxAttempts) {
|
|
199
|
+
throw err;
|
|
200
|
+
}
|
|
201
|
+
const delay = jitteredBackoffMs(staleLosses - 1, { baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
|
|
202
|
+
await abortableDelay(delay, signal);
|
|
203
|
+
// Re-read fresh state before re-attempting so the next commit pends against current
|
|
204
|
+
// revisions (mirrors how Collection.sync calls updateInternal() before retrying).
|
|
205
|
+
// NOTE: refreshes EVERY registered collection, not only the participants of this
|
|
206
|
+
// transaction. Not free: a non-participant's update() throws CollectionHeaderVanishedError
|
|
207
|
+
// if its header momentarily reads absent while it holds a committed revision, aborting
|
|
208
|
+
// this retry. The registered set is small today; if that (or retry latency) ever bites,
|
|
209
|
+
// narrow this to the transaction's participating collections.
|
|
210
|
+
for (const collection of this.collections.values()) {
|
|
211
|
+
await collection.update();
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
} finally {
|
|
216
|
+
for (const dispose of inFlightDisposers) {
|
|
217
|
+
dispose();
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Commit a transaction (single attempt): materialise a log entry from each collection's staged
|
|
224
|
+
* pending actions, then orchestrate the distributed consensus (GATHER/PEND/COMMIT).
|
|
225
|
+
*
|
|
226
|
+
* Called by {@link commit} (which wraps it in the backoff+jitter retry loop). The
|
|
227
|
+
* staged mutations already live in each collection's tracker — applied either via
|
|
228
|
+
* applyActions() (engine-driven path) or directly via Collection.act()/Tree.stage
|
|
229
|
+
* (the vtab's deferred-DML path) — but in BOTH cases without a log entry yet, so
|
|
230
|
+
* this method appends that entry here (see the inline note below) before pending,
|
|
231
|
+
* and folds the committed transforms back into each collection's read cache.
|
|
232
|
+
*
|
|
233
|
+
* On a clean stale loss (nothing durable, every tracker restored) it throws
|
|
234
|
+
* {@link CoordinatorStaleLossError} so the caller can retry; on a partial landing it throws
|
|
235
|
+
* {@link CoordinatorPartialCommitError} (not retryable).
|
|
236
|
+
*
|
|
237
|
+
* @param transaction - The transaction to commit
|
|
238
|
+
* @param inFlightDisposers - Collects one disposer per participant marked in flight under this
|
|
239
|
+
* transaction's id (see {@link Collection.beginInFlightAction}). REQUIRED, so a future caller
|
|
240
|
+
* cannot silently reintroduce the unmarked refresh this parameter exists to prevent: a caller
|
|
241
|
+
* that never refreshes between attempts passes a throwaway array and simply ignores it. The
|
|
242
|
+
* caller owns clearing them, because the mark has to survive past this attempt — see the
|
|
243
|
+
* array's declaration in {@link commit}.
|
|
244
|
+
*/
|
|
245
|
+
private async commitOnce(transaction: Transaction, inFlightDisposers: (() => void)[]): Promise<void> {
|
|
246
|
+
if (isTransactionExpired(transaction.stamp)) {
|
|
247
|
+
throw new Error(`Transaction expired at ${transaction.stamp.expiration}`);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Collect collections with staged (un-synced) changes.
|
|
251
|
+
const collectionData = Array.from(this.collections.entries())
|
|
252
|
+
.map(([collectionId, collection]) => ({
|
|
253
|
+
collectionId,
|
|
254
|
+
collection,
|
|
255
|
+
transforms: collection.tracker.transforms
|
|
256
|
+
}))
|
|
257
|
+
.filter(({ transforms }) =>
|
|
258
|
+
Object.keys(transforms.inserts ?? {}).length +
|
|
259
|
+
Object.keys(transforms.updates ?? {}).length +
|
|
260
|
+
(transforms.deletes?.length ?? 0) > 0
|
|
261
|
+
);
|
|
262
|
+
|
|
263
|
+
if (collectionData.length === 0) {
|
|
264
|
+
return; // Nothing to commit
|
|
265
|
+
}
|
|
266
|
+
// NOTE: this selection reads each tracker BEFORE the latches below are held, so a stage
|
|
267
|
+
// that lands between the filter and the acquisition is simply not part of this commit.
|
|
268
|
+
// Harmless today — a session stages and commits on one call path, so nothing races its
|
|
269
|
+
// own commit. If a caller ever stages a collection concurrently with committing it,
|
|
270
|
+
// re-derive the participant set inside the held span instead of filtering out here.
|
|
271
|
+
|
|
272
|
+
// Hold every participating collection's instance latch for the WHOLE commit span —
|
|
273
|
+
// snapshot, log append, the pend/commit round trips, and the local fold — so a
|
|
274
|
+
// reader-driven update()/sync() on the same instance cannot interleave: without this, a
|
|
275
|
+
// refresh could adopt the newly committed revision mid-flight and recordCommitted would
|
|
276
|
+
// land the action at a revision storage never assigned it. Acquisition is in sorted
|
|
277
|
+
// collection-id order, mirroring StorageRepo.commit's sorted block-id latch discipline
|
|
278
|
+
// (db-p2p/src/storage/block-latch.ts), so two concurrent commits over overlapping
|
|
279
|
+
// participant sets cannot deadlock. `Latches` is non-reentrant, so nothing inside the
|
|
280
|
+
// held span may call a latched Collection method (act/update/sync/updateAndSync) on a
|
|
281
|
+
// participant — the retry loop's blanket collection.update() in commit() runs OUTSIDE
|
|
282
|
+
// this span, after release.
|
|
283
|
+
// NOTE: the span covers the pend/commit consensus round trips, so every latched method on
|
|
284
|
+
// a participant instance (act/update/sync) queues for as long as the transactor takes.
|
|
285
|
+
// Accepted: correctness needs the whole span, and the transactor's own timeouts bound it.
|
|
286
|
+
// If a stalled peer is ever observed wedging unrelated readers, bound the hold instead —
|
|
287
|
+
// e.g. acquire with a deadline and fail the commit rather than queueing indefinitely.
|
|
288
|
+
const latchReleases: (() => void)[] = [];
|
|
289
|
+
try {
|
|
290
|
+
const latchOrder = [...collectionData].sort((a, b) =>
|
|
291
|
+
a.collectionId < b.collectionId ? -1 : a.collectionId > b.collectionId ? 1 : 0);
|
|
292
|
+
for (const { collection } of latchOrder) {
|
|
293
|
+
latchReleases.push(await collection.acquireLatch());
|
|
294
|
+
// Mark THIS attempt's action id on each participant while its latch is held, so the
|
|
295
|
+
// inter-attempt refresh in commit() recognises a log entry this transaction itself
|
|
296
|
+
// made durable (a torn commit: header and log tail committed, a later sweep block
|
|
297
|
+
// reported the conflict) and consumes it instead of replaying it into a second entry
|
|
298
|
+
// under the same id. `transaction.id` is stable across retries, so re-marking on a
|
|
299
|
+
// later attempt re-states the same fact. Only participants are marked; a registered
|
|
300
|
+
// non-participant is left unmarked and its refresh behaves exactly as a reader's.
|
|
301
|
+
inFlightDisposers.push(collection.beginInFlightAction(transaction.id));
|
|
302
|
+
}
|
|
303
|
+
await this.commitOnceLatched(transaction, collectionData);
|
|
304
|
+
} finally {
|
|
305
|
+
for (const release of latchReleases.reverse()) {
|
|
306
|
+
release();
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* The body of {@link commitOnce}, run with every participating collection's instance latch
|
|
313
|
+
* held by the caller (see the acquisition comment there). Nothing in here may re-acquire a
|
|
314
|
+
* participant's latch — every Collection member it touches (snapshotPending, getPendingActions,
|
|
315
|
+
* recordCommitted, applyCommittedToCache, restorePending, clearPendingActions, tracker.reset)
|
|
316
|
+
* is latch-free by contract.
|
|
317
|
+
*/
|
|
318
|
+
private async commitOnceLatched(
|
|
319
|
+
transaction: Transaction,
|
|
320
|
+
collectionData: { collectionId: CollectionId; collection: Collection<any> }[]
|
|
321
|
+
): Promise<void> {
|
|
322
|
+
// Append each collection's staged actions to its log, then collect the
|
|
323
|
+
// resulting transforms + critical (log-tail) block for consensus.
|
|
324
|
+
//
|
|
325
|
+
// The actions were staged directly into the trackers (Collection.act, e.g.
|
|
326
|
+
// via Tree.stage) WITHOUT first appending a log entry, so — exactly as
|
|
327
|
+
// execute()/applyActionsToCollection does — we materialise the log entry
|
|
328
|
+
// here from each collection's pending actions. Reading raw tracker
|
|
329
|
+
// transforms without a fresh log entry only ever "worked" for a
|
|
330
|
+
// collection's pristine first commit (where the initial empty log block is
|
|
331
|
+
// itself an uncommitted tracker insert); it broke for any collection with
|
|
332
|
+
// prior committed state — a pre-synced index tree, or a second commit on
|
|
333
|
+
// the same collection — whose log tail lives in storage, not the tracker.
|
|
334
|
+
const allCollectionIds = collectionData.map(({ collectionId }) => collectionId);
|
|
335
|
+
const collectionTransforms = new Map<CollectionId, Transforms>();
|
|
336
|
+
const criticalBlocks = new Map<CollectionId, BlockId>();
|
|
337
|
+
// The revision each collection's log entry was stamped with. Captured ONCE — at the log
|
|
338
|
+
// append in applyActionsToCollection, the single legitimate capture point — and threaded
|
|
339
|
+
// through pend, commit, and the local recordCommitted, so all four name the same number.
|
|
340
|
+
const pendedRevs = new Map<CollectionId, number>();
|
|
341
|
+
|
|
342
|
+
// Snapshot EVERY participating collection's staged state (transforms + pending
|
|
343
|
+
// queue) BEFORE the append loop mutates any tracker. The loop appends log
|
|
344
|
+
// entries sequentially, so a failure on the Nth collection must also undo the
|
|
345
|
+
// 0..N-1 collections that already appended — and coordinateTransaction can fail
|
|
346
|
+
// after ALL of them appended. On any throw below we restore every snapshot, so a
|
|
347
|
+
// failed commit leaves each tracker exactly as it was: a retry re-appends cleanly
|
|
348
|
+
// (no duplicate log entry) and a directly-staged tree's rollback (which no-ops
|
|
349
|
+
// when the stamp was never tracked via applyActions) has nothing poisoned to undo.
|
|
350
|
+
const preCommitSnapshots = new Map<CollectionId, ReturnType<Collection<any>['snapshotPending']>>();
|
|
351
|
+
for (const { collectionId, collection } of collectionData) {
|
|
352
|
+
preCommitSnapshots.set(collectionId, collection.snapshotPending());
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
let coordResult: {
|
|
356
|
+
success: boolean;
|
|
357
|
+
error?: string;
|
|
358
|
+
committedCollections?: Set<CollectionId>;
|
|
359
|
+
failedCollections?: Set<CollectionId>;
|
|
360
|
+
staleLoss?: boolean;
|
|
361
|
+
};
|
|
362
|
+
try {
|
|
363
|
+
for (const { collectionId, collection } of collectionData) {
|
|
364
|
+
const applyResult = await this.applyActionsToCollection(
|
|
365
|
+
{ collectionId, actions: collection.getPendingActions() },
|
|
366
|
+
transaction,
|
|
367
|
+
allCollectionIds
|
|
368
|
+
);
|
|
369
|
+
if (!applyResult.success) {
|
|
370
|
+
throw new Error(`Transaction commit failed: ${applyResult.error}`);
|
|
371
|
+
}
|
|
372
|
+
collectionTransforms.set(collectionId, applyResult.transforms!);
|
|
373
|
+
criticalBlocks.set(collectionId, applyResult.logTailBlockId!);
|
|
374
|
+
pendedRevs.set(collectionId, applyResult.rev!);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// Compute hash of ALL operations across ALL collections (post-log-append).
|
|
378
|
+
// Validators re-execute the transaction and compare their computed hash.
|
|
379
|
+
// The shared operations-hash module canonicalises (sort + canonical JSON) so
|
|
380
|
+
// this order-independent fingerprint matches what a validator recomputes.
|
|
381
|
+
const operationsHash = await hashOperations(collectOperations(collectionTransforms));
|
|
382
|
+
|
|
383
|
+
// Execute consensus phases (GATHER, PEND, COMMIT)
|
|
384
|
+
coordResult = await this.coordinateTransaction(
|
|
385
|
+
transaction,
|
|
386
|
+
operationsHash,
|
|
387
|
+
collectionTransforms,
|
|
388
|
+
criticalBlocks,
|
|
389
|
+
pendedRevs
|
|
390
|
+
);
|
|
391
|
+
} catch (err) {
|
|
392
|
+
// A throw here means the failure happened BEFORE any collection could
|
|
393
|
+
// durably commit (a log-append failure, or coordinateTransaction rejecting
|
|
394
|
+
// unexpectedly). Nothing landed on the cluster, so roll every tracker back
|
|
395
|
+
// to its pre-append snapshot — a genuinely clean rollback that leaves each
|
|
396
|
+
// tracker pristine for retry (see txn-failed-commit-leaves-staged-log-entry).
|
|
397
|
+
for (const { collectionId, collection } of collectionData) {
|
|
398
|
+
collection.restorePending(preCommitSnapshots.get(collectionId)!);
|
|
399
|
+
}
|
|
400
|
+
throw err;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
if (!coordResult.success) {
|
|
404
|
+
const committed = coordResult.committedCollections ?? new Set<CollectionId>();
|
|
405
|
+
if (committed.size > 0) {
|
|
406
|
+
// PARTIAL COMMIT: at least one collection durably committed via consensus
|
|
407
|
+
// while another failed permanently. A uniform pre-append restore would
|
|
408
|
+
// corrupt the committed half — re-staging its already-durable actions as
|
|
409
|
+
// still-pending, so tracker memory would disagree with cluster storage.
|
|
410
|
+
// Split the local handling instead:
|
|
411
|
+
for (const { collectionId, collection } of collectionData) {
|
|
412
|
+
if (committed.has(collectionId)) {
|
|
413
|
+
// Committed → the success-path local treatment (see below): fold the
|
|
414
|
+
// committed transforms into the read cache BEFORE resetting the tracker,
|
|
415
|
+
// then drop the now-durable pending actions so a retry cannot re-log them.
|
|
416
|
+
// NOTE: no-double-apply on retry depends on clearPendingActions() running for
|
|
417
|
+
// EVERY committed collection here before any re-drive of commit(). If a committed
|
|
418
|
+
// collection kept its pending queue, a subsequent commit() would re-append and
|
|
419
|
+
// re-log its already-durable actions — a duplicate log entry on the winner. The
|
|
420
|
+
// no-double-apply-on-retry test in transaction.spec.ts locks this.
|
|
421
|
+
const rev = collection.recordCommitted(transaction.id, pendedRevs.get(collectionId)!);
|
|
422
|
+
collection.applyCommittedToCache(collectionTransforms.get(collectionId)!, rev);
|
|
423
|
+
collection.tracker.reset();
|
|
424
|
+
collection.clearPendingActions();
|
|
425
|
+
} else {
|
|
426
|
+
// Failed / never-committed → restore the pre-append snapshot so a retry
|
|
427
|
+
// re-appends cleanly (no duplicate log entry).
|
|
428
|
+
collection.restorePending(preCommitSnapshots.get(collectionId)!);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
// The transaction half-landed, so it is neither cleanly retryable nor
|
|
432
|
+
// cleanly abortable: drop its stamp tracking (the success path does the
|
|
433
|
+
// same at the end) and surface the structured signal for reconciliation.
|
|
434
|
+
this.stampData.delete(transaction.stamp.id);
|
|
435
|
+
throw new CoordinatorPartialCommitError(
|
|
436
|
+
[...committed],
|
|
437
|
+
[...(coordResult.failedCollections ?? new Set<CollectionId>())],
|
|
438
|
+
coordResult.error
|
|
439
|
+
);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
// EMPTY committed set: PEND failed, or the whole commit failed cleanly with
|
|
443
|
+
// nothing durable. Restore every tracker so each is pristine for retry.
|
|
444
|
+
for (const { collectionId, collection } of collectionData) {
|
|
445
|
+
collection.restorePending(preCommitSnapshots.get(collectionId)!);
|
|
446
|
+
}
|
|
447
|
+
// Distinguish a genuine optimistic-concurrency conflict (a stale loss / pending
|
|
448
|
+
// contention — retryable after a re-read) from a hard failure (unavailable transactor,
|
|
449
|
+
// storage rejection, expired). Only the former is worth re-driving; the retry wrapper in
|
|
450
|
+
// commit() catches CoordinatorStaleLossError and re-attempts, while a plain Error escapes
|
|
451
|
+
// immediately (preserving the historical fail-fast behaviour for hard failures).
|
|
452
|
+
if (coordResult.staleLoss) {
|
|
453
|
+
throw new CoordinatorStaleLossError([...(coordResult.failedCollections ?? new Set(allCollectionIds))], coordResult.error);
|
|
454
|
+
}
|
|
455
|
+
throw new Error(`Transaction commit failed: ${coordResult.error}`);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
// Advance actionContext, fold the committed transforms into each
|
|
459
|
+
// collection's read cache, reset the tracker, and drop the now-committed
|
|
460
|
+
// pending actions. Order matters: cache the committed blocks BEFORE
|
|
461
|
+
// resetting the tracker (the transforms are read live), so a collection
|
|
462
|
+
// with prior committed state (a pre-synced index, or any second commit)
|
|
463
|
+
// serves the new revision instead of the stale cached one. Clearing
|
|
464
|
+
// pending keeps a subsequent commit from re-logging these actions.
|
|
465
|
+
// NOTE: this fold loop must stay await-free — session-mode publish relies on it being
|
|
466
|
+
// event-loop-atomic across collections (see OptimysticModule's readCommittedSnapshot audit).
|
|
467
|
+
for (const { collectionId, collection } of collectionData) {
|
|
468
|
+
const rev = collection.recordCommitted(transaction.id, pendedRevs.get(collectionId)!);
|
|
469
|
+
collection.applyCommittedToCache(collectionTransforms.get(collectionId)!, rev);
|
|
470
|
+
collection.tracker.reset();
|
|
471
|
+
collection.clearPendingActions();
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
// Clean up stamp tracking data
|
|
475
|
+
this.stampData.delete(transaction.stamp.id);
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/**
|
|
479
|
+
* Rollback a transaction (undo only the given stampId's applied actions).
|
|
480
|
+
*
|
|
481
|
+
* Restores tracker state to the snapshot taken before the stampId's first
|
|
482
|
+
* applyActions call, then replays any later stamps' actions to preserve
|
|
483
|
+
* other sessions' transforms.
|
|
484
|
+
*
|
|
485
|
+
* @param stampId - The transaction stamp ID to rollback
|
|
486
|
+
*/
|
|
487
|
+
async rollback(stampId: string): Promise<void> {
|
|
488
|
+
// NOTE: unlike the commit path, this resets and replays into participant trackers WITHOUT
|
|
489
|
+
// holding their instance latches. Safe today because a session drives abort and commit from
|
|
490
|
+
// one call path, so a rollback cannot overlap a commit span on the same collections. If
|
|
491
|
+
// rollback ever becomes reachable concurrently with a commit (a background abort, a second
|
|
492
|
+
// session sharing collection instances), latch the participants here the way commitOnce does.
|
|
493
|
+
const data = this.stampData.get(stampId);
|
|
494
|
+
if (!data) return;
|
|
495
|
+
|
|
496
|
+
this.stampData.delete(stampId);
|
|
497
|
+
|
|
498
|
+
// Collect all remaining stamps to replay
|
|
499
|
+
const toReplay = [...this.stampData.entries()]
|
|
500
|
+
.sort(([, a], [, b]) => a.order - b.order);
|
|
501
|
+
|
|
502
|
+
// Find the earliest snapshot among the rolled-back stamp and all remaining stamps.
|
|
503
|
+
// This is necessary because interleaved execution means a lower-order stamp
|
|
504
|
+
// may have batches applied after a higher-order stamp's snapshot was taken.
|
|
505
|
+
let earliestSnapshot = data.preSnapshot;
|
|
506
|
+
let earliestOrder = data.order;
|
|
507
|
+
for (const [, d] of toReplay) {
|
|
508
|
+
if (d.order < earliestOrder) {
|
|
509
|
+
earliestSnapshot = d.preSnapshot;
|
|
510
|
+
earliestOrder = d.order;
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
// Restore to the earliest snapshot
|
|
515
|
+
for (const [collectionId, transforms] of earliestSnapshot) {
|
|
516
|
+
const collection = this.collections.get(collectionId);
|
|
517
|
+
if (collection) {
|
|
518
|
+
collection.tracker.reset(structuredClone(transforms));
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// Replay all remaining stamps' batches in order
|
|
523
|
+
for (const [replayStampId, replayData] of toReplay) {
|
|
524
|
+
// Update the snapshot to reflect current (post-replay) state
|
|
525
|
+
const newSnapshot = new Map<CollectionId, Transforms>();
|
|
526
|
+
for (const [id, col] of this.collections) {
|
|
527
|
+
newSnapshot.set(id, structuredClone(col.tracker.transforms));
|
|
528
|
+
}
|
|
529
|
+
replayData.preSnapshot = newSnapshot;
|
|
530
|
+
|
|
531
|
+
for (const actionBatch of replayData.actionBatches) {
|
|
532
|
+
await this.applyActionsRaw(actionBatch, replayStampId);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
/**
|
|
538
|
+
* Get current transforms from all collections.
|
|
539
|
+
*
|
|
540
|
+
* This collects transforms from each collection's tracker. Useful for
|
|
541
|
+
* validation scenarios where transforms need to be extracted after
|
|
542
|
+
* engine execution.
|
|
543
|
+
*/
|
|
544
|
+
getTransforms(): Map<CollectionId, Transforms> {
|
|
545
|
+
const transforms = new Map<CollectionId, Transforms>();
|
|
546
|
+
for (const [collectionId, collection] of this.collections.entries()) {
|
|
547
|
+
const collectionTransforms = collection.tracker.transforms;
|
|
548
|
+
const hasChanges =
|
|
549
|
+
Object.keys(collectionTransforms.inserts ?? {}).length > 0 ||
|
|
550
|
+
Object.keys(collectionTransforms.updates ?? {}).length > 0 ||
|
|
551
|
+
(collectionTransforms.deletes?.length ?? 0) > 0;
|
|
552
|
+
if (hasChanges) {
|
|
553
|
+
transforms.set(collectionId, collectionTransforms);
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
return transforms;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
/**
|
|
560
|
+
* Reset all collection trackers.
|
|
561
|
+
*
|
|
562
|
+
* This clears pending transforms from all collections. Useful for
|
|
563
|
+
* cleaning up after validation or when starting a new transaction.
|
|
564
|
+
*/
|
|
565
|
+
resetTransforms(): void {
|
|
566
|
+
for (const collection of this.collections.values()) {
|
|
567
|
+
collection.tracker.reset();
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* Collect read dependencies from all participating collections.
|
|
573
|
+
*/
|
|
574
|
+
getReadDependencies(): ReadDependency[] {
|
|
575
|
+
const reads: ReadDependency[] = [];
|
|
576
|
+
for (const collection of this.collections.values()) {
|
|
577
|
+
reads.push(...collection.getReadDependencies());
|
|
578
|
+
}
|
|
579
|
+
return reads;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* Clear read dependencies from all collections.
|
|
584
|
+
*/
|
|
585
|
+
clearReadDependencies(): void {
|
|
586
|
+
for (const collection of this.collections.values()) {
|
|
587
|
+
collection.clearReadDependencies();
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
/**
|
|
592
|
+
* Execute a fully-formed transaction.
|
|
593
|
+
*
|
|
594
|
+
* This is called with a complete transaction (e.g., from Quereus).
|
|
595
|
+
*
|
|
596
|
+
* @param transaction - The transaction to execute
|
|
597
|
+
* @param engine - The engine to use for executing the transaction
|
|
598
|
+
* @returns Execution result with actions and results
|
|
599
|
+
*/
|
|
600
|
+
async execute(transaction: Transaction, engine: ITransactionEngine): Promise<ExecutionResult> {
|
|
601
|
+
const trxId = transaction.id;
|
|
602
|
+
const t0 = Date.now();
|
|
603
|
+
|
|
604
|
+
if (isTransactionExpired(transaction.stamp)) {
|
|
605
|
+
return { success: false, error: `Transaction expired at ${transaction.stamp.expiration}` };
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
// 1. Validate engine matches transaction
|
|
609
|
+
// Note: We don't enforce this strictly since the engine is passed in explicitly
|
|
610
|
+
// The caller is responsible for ensuring the correct engine is used
|
|
611
|
+
|
|
612
|
+
const tEngine = Date.now();
|
|
613
|
+
const result = await engine.execute(transaction);
|
|
614
|
+
const engineMs = Date.now() - tEngine;
|
|
615
|
+
if (!result.success) {
|
|
616
|
+
log('execute:done trxId=%s engine=%dms success=false total=%dms', trxId, engineMs, Date.now() - t0);
|
|
617
|
+
return result;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
if (!result.actions || result.actions.length === 0) {
|
|
621
|
+
return { success: true }; // Nothing to do
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
// 1b. Stage the returned actions into the collection trackers.
|
|
625
|
+
//
|
|
626
|
+
// Reaching here means the engine RETURNED non-empty actions — i.e. the pure-
|
|
627
|
+
// translator model (see the ITransactionEngine contract): it translated the
|
|
628
|
+
// statements but did NOT apply them. So THIS path owns application — we stage the
|
|
629
|
+
// actions here via applyActions() (which also snapshots/tracks the stamp for
|
|
630
|
+
// rollback) BEFORE the loop below reads each tracker's transforms to materialise
|
|
631
|
+
// the log entry. (Previously ActionsEngine applied as a side effect and this
|
|
632
|
+
// method merely re-read the already-staged trackers; that side effect is gone, so
|
|
633
|
+
// the application must happen explicitly here. A side-effecting engine that
|
|
634
|
+
// applied internally would instead return EMPTY actions and short-circuit at the
|
|
635
|
+
// guard above.)
|
|
636
|
+
//
|
|
637
|
+
// applyActions() throws if a referenced collection is not registered — the same
|
|
638
|
+
// "Collection not found" the engine's side-effecting apply used to surface. Convert
|
|
639
|
+
// it back into a failure result so execute() keeps its return contract.
|
|
640
|
+
try {
|
|
641
|
+
await this.applyActions(result.actions, transaction.stamp.id);
|
|
642
|
+
} catch (error) {
|
|
643
|
+
const engineMs = Date.now() - tEngine;
|
|
644
|
+
log('execute:done trxId=%s engine=%dms apply-failed=true total=%dms', trxId, engineMs, Date.now() - t0);
|
|
645
|
+
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
// 2. Build a log entry per collection from the now-staged tracker transforms.
|
|
649
|
+
//
|
|
650
|
+
// NOTE: like commit(), this loop appends a log entry into each collection's
|
|
651
|
+
// tracker and these failure returns do NOT restore that state — so a partially
|
|
652
|
+
// applied engine transaction leaves appended-but-uncommitted entries in the
|
|
653
|
+
// trackers. This is deliberately NOT snapshot/restore-wrapped the way commit()
|
|
654
|
+
// is, because execute()'s asymmetry makes it lower risk: it is not the retryable
|
|
655
|
+
// session.commit() entry point (a failed execute() is not re-driven through the
|
|
656
|
+
// same loop), and its actions were tracked via applyActions() so rollback(stampId)
|
|
657
|
+
// CAN unwind them (unlike commit()'s directly-staged path). If execute() ever
|
|
658
|
+
// becomes retryable, mirror the commit() snapshot/restore fix here.
|
|
659
|
+
const tApply = Date.now();
|
|
660
|
+
const collectionTransforms = new Map<CollectionId, Transforms>();
|
|
661
|
+
const criticalBlocks = new Map<CollectionId, BlockId>();
|
|
662
|
+
const actionResults = new Map<CollectionId, any[]>();
|
|
663
|
+
const allCollectionIds = result.actions.map(ca => ca.collectionId);
|
|
664
|
+
// Same single-capture rev threading as commitOnce: stamped at the log append below,
|
|
665
|
+
// named again at pend, commit, and recordCommitted.
|
|
666
|
+
const pendedRevs = new Map<CollectionId, number>();
|
|
667
|
+
|
|
668
|
+
// Hold each participating collection's instance latch for the commit span, same
|
|
669
|
+
// discipline as commitOnce (sorted acquisition; see the comment there). Acquired only
|
|
670
|
+
// AFTER applyActions above: collection.act takes the same non-reentrant instance latch
|
|
671
|
+
// itself, so latching earlier would deadlock. Deduped before acquiring — taking one
|
|
672
|
+
// instance's latch twice would also deadlock. Released in the finally: execute has
|
|
673
|
+
// early failure returns.
|
|
674
|
+
const latchReleases: (() => void)[] = [];
|
|
675
|
+
try {
|
|
676
|
+
for (const collectionId of [...new Set(allCollectionIds)].sort()) {
|
|
677
|
+
const collection = this.collections.get(collectionId);
|
|
678
|
+
if (collection) {
|
|
679
|
+
latchReleases.push(await collection.acquireLatch());
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
for (const collectionActions of result.actions) {
|
|
684
|
+
const applyResult = await this.applyActionsToCollection(
|
|
685
|
+
collectionActions,
|
|
686
|
+
transaction,
|
|
687
|
+
allCollectionIds
|
|
688
|
+
);
|
|
689
|
+
|
|
690
|
+
if (!applyResult.success) {
|
|
691
|
+
return { success: false, error: applyResult.error };
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
collectionTransforms.set(collectionActions.collectionId, applyResult.transforms!);
|
|
695
|
+
criticalBlocks.set(collectionActions.collectionId, applyResult.logTailBlockId!);
|
|
696
|
+
actionResults.set(collectionActions.collectionId, applyResult.results!);
|
|
697
|
+
pendedRevs.set(collectionActions.collectionId, applyResult.rev!);
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// 3. Compute operations hash for validation (order-independent; see commit()).
|
|
701
|
+
const operationsHash = await hashOperations(collectOperations(collectionTransforms));
|
|
702
|
+
|
|
703
|
+
const applyMs = Date.now() - tApply;
|
|
704
|
+
|
|
705
|
+
// 4. Coordinate (GATHER if multi-collection)
|
|
706
|
+
const tCoord = Date.now();
|
|
707
|
+
const coordResult = await this.coordinateTransaction(
|
|
708
|
+
transaction,
|
|
709
|
+
operationsHash,
|
|
710
|
+
collectionTransforms,
|
|
711
|
+
criticalBlocks,
|
|
712
|
+
pendedRevs
|
|
713
|
+
);
|
|
714
|
+
|
|
715
|
+
const coordMs = Date.now() - tCoord;
|
|
716
|
+
if (!coordResult.success) {
|
|
717
|
+
log('execute:done trxId=%s engine=%dms apply=%dms coordinate=%dms success=false total=%dms', trxId, engineMs, applyMs, coordMs, Date.now() - t0);
|
|
718
|
+
// Stop lying to the caller about a partial commit: if some collections durably
|
|
719
|
+
// committed, surface that set. execute() is not snapshot/restore-wrapped (see the
|
|
720
|
+
// note above), but the committed subset must still get the success-path local
|
|
721
|
+
// treatment (recordCommitted + tracker.reset, as on the success path below) so its
|
|
722
|
+
// trackers aren't left mis-tracking already-durable state.
|
|
723
|
+
const committed = coordResult.committedCollections ?? new Set<CollectionId>();
|
|
724
|
+
if (committed.size > 0) {
|
|
725
|
+
for (const collectionActions of result.actions) {
|
|
726
|
+
const collection = this.collections.get(collectionActions.collectionId);
|
|
727
|
+
if (collection && committed.has(collectionActions.collectionId)) {
|
|
728
|
+
collection.recordCommitted(transaction.id, pendedRevs.get(collectionActions.collectionId)!);
|
|
729
|
+
collection.tracker.reset();
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
return {
|
|
734
|
+
success: false,
|
|
735
|
+
error: coordResult.error,
|
|
736
|
+
committedCollections: committed.size > 0 ? [...committed] : undefined,
|
|
737
|
+
failedCollections: coordResult.failedCollections ? [...coordResult.failedCollections] : undefined,
|
|
738
|
+
};
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// 5. Update actionContext and reset trackers after successful commit
|
|
742
|
+
for (const collectionActions of result.actions) {
|
|
743
|
+
const collection = this.collections.get(collectionActions.collectionId);
|
|
744
|
+
if (collection) {
|
|
745
|
+
collection.recordCommitted(transaction.id, pendedRevs.get(collectionActions.collectionId)!);
|
|
746
|
+
collection.tracker.reset();
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
// Clean up stamp tracking data
|
|
751
|
+
this.stampData.delete(transaction.stamp.id);
|
|
752
|
+
|
|
753
|
+
// 6. Return results from actions
|
|
754
|
+
log('execute:done trxId=%s engine=%dms apply=%dms coordinate=%dms total=%dms', trxId, engineMs, applyMs, coordMs, Date.now() - t0);
|
|
755
|
+
return {
|
|
756
|
+
success: true,
|
|
757
|
+
actions: result.actions,
|
|
758
|
+
results: actionResults
|
|
759
|
+
};
|
|
760
|
+
} finally {
|
|
761
|
+
for (const release of latchReleases.reverse()) {
|
|
762
|
+
release();
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
/**
|
|
768
|
+
* Apply actions to a collection.
|
|
769
|
+
*
|
|
770
|
+
* This runs the action handlers, writes to the log, and collects transforms.
|
|
771
|
+
*/
|
|
772
|
+
private async applyActionsToCollection(
|
|
773
|
+
collectionActions: CollectionActions,
|
|
774
|
+
transaction: Transaction,
|
|
775
|
+
allCollectionIds: CollectionId[]
|
|
776
|
+
): Promise<{
|
|
777
|
+
success: boolean;
|
|
778
|
+
transforms?: Transforms;
|
|
779
|
+
logTailBlockId?: BlockId;
|
|
780
|
+
/** The revision the log entry was stamped with — the ONE number the pend, the commit,
|
|
781
|
+
* and the local recordCommitted must all repeat (see the pendedRevs maps upstream). */
|
|
782
|
+
rev?: number;
|
|
783
|
+
results?: any[];
|
|
784
|
+
error?: string;
|
|
785
|
+
}> {
|
|
786
|
+
const collection = this.collections.get(collectionActions.collectionId);
|
|
787
|
+
if (!collection) {
|
|
788
|
+
return {
|
|
789
|
+
success: false,
|
|
790
|
+
error: `Collection not found: ${collectionActions.collectionId}`
|
|
791
|
+
};
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
// At this point, actions have already been executed through collection.act()
|
|
795
|
+
// (via the engine or the vtab's staging path). The collection's tracker
|
|
796
|
+
// already has the transforms, and the actions are in the pending buffer.
|
|
797
|
+
|
|
798
|
+
// Get transforms from the collection's tracker
|
|
799
|
+
const transforms = collection.tracker.transforms;
|
|
800
|
+
|
|
801
|
+
// Write actions to the collection's log to get the log tail block ID
|
|
802
|
+
const log = await Log.open(collection.tracker, collectionActions.collectionId);
|
|
803
|
+
if (!log) {
|
|
804
|
+
return {
|
|
805
|
+
success: false,
|
|
806
|
+
error: `Log not found for collection ${collectionActions.collectionId}`
|
|
807
|
+
};
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
// Generate action ID from transaction ID
|
|
811
|
+
const actionId = transaction.id;
|
|
812
|
+
const newRev = collection.getNextRev();
|
|
813
|
+
|
|
814
|
+
// Add actions to log (this updates the tracker with log block changes).
|
|
815
|
+
// Persist the transaction's read set on the entry so a later invalidation cascade can
|
|
816
|
+
// discover this action's read-dependents (see ActionEntry.reads). The whole transaction's
|
|
817
|
+
// reads are recorded on every collection's entry: a read may target a block in another
|
|
818
|
+
// collection, and the cascade matches read-dependents by (blockId, revision) regardless of
|
|
819
|
+
// which collection's log the dependent landed in.
|
|
820
|
+
// NOTE: `allCollectionIds` names the participants of THIS attempt, and a retry's participant
|
|
821
|
+
// set can be SMALLER than the first attempt's. After a torn commit, the participant whose
|
|
822
|
+
// entry landed durably consumes that entry on the inter-attempt refresh
|
|
823
|
+
// (Collection.inFlightActionId), empties its pending queue and resets its tracker, so
|
|
824
|
+
// commitOnce's non-empty-transforms filter drops it from the next attempt. The retry's
|
|
825
|
+
// entries therefore list only the REMAINING participants, while the torn participant's
|
|
826
|
+
// already-durable entry lists them all — one transaction id, two different
|
|
827
|
+
// `allCollectionIds` values across its entries. Nothing today keys off that list for
|
|
828
|
+
// correctness; a cross-collection invalidation cascade that treats it as "the definitive
|
|
829
|
+
// participant set of this transaction" must union it across the transaction's entries
|
|
830
|
+
// rather than trusting any single one.
|
|
831
|
+
const addResult = await log.addActions(
|
|
832
|
+
collectionActions.actions,
|
|
833
|
+
actionId,
|
|
834
|
+
newRev,
|
|
835
|
+
() => blockIdsForTransforms(transforms),
|
|
836
|
+
allCollectionIds,
|
|
837
|
+
transaction.reads
|
|
838
|
+
);
|
|
839
|
+
|
|
840
|
+
// Return the transforms and log tail block ID
|
|
841
|
+
return {
|
|
842
|
+
success: true,
|
|
843
|
+
transforms,
|
|
844
|
+
logTailBlockId: addResult.tailPath.block.header.id,
|
|
845
|
+
rev: newRev,
|
|
846
|
+
results: [] // TODO: Collect results from action handlers when we support read operations
|
|
847
|
+
};
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* Coordinate a transaction across multiple collections.
|
|
852
|
+
*
|
|
853
|
+
* @param transaction - The transaction to coordinate
|
|
854
|
+
* @param operationsHash - Hash of all operations for validation
|
|
855
|
+
* @param collectionTransforms - Map of collectionId to its transforms
|
|
856
|
+
* @param criticalBlocks - Map of collectionId to its log tail blockId
|
|
857
|
+
* @param pendedRevs - Per collection, the revision its log entry was stamped with (from
|
|
858
|
+
* applyActionsToCollection) — repeated verbatim on the pend and commit requests so storage
|
|
859
|
+
* and the local record name the same number.
|
|
860
|
+
*/
|
|
861
|
+
private async coordinateTransaction(
|
|
862
|
+
transaction: Transaction,
|
|
863
|
+
operationsHash: string,
|
|
864
|
+
collectionTransforms: Map<CollectionId, Transforms>,
|
|
865
|
+
criticalBlocks: Map<CollectionId, BlockId>,
|
|
866
|
+
pendedRevs: ReadonlyMap<CollectionId, number>
|
|
867
|
+
): Promise<{
|
|
868
|
+
success: boolean;
|
|
869
|
+
error?: string;
|
|
870
|
+
committedCollections?: Set<CollectionId>;
|
|
871
|
+
failedCollections?: Set<CollectionId>;
|
|
872
|
+
/** True when the failure was a clean optimistic-concurrency conflict (stale loss / pending
|
|
873
|
+
* contention) with nothing durable — i.e. safe to re-drive after a re-read. */
|
|
874
|
+
staleLoss?: boolean;
|
|
875
|
+
}> {
|
|
876
|
+
const trxId = transaction.id;
|
|
877
|
+
const t0 = Date.now();
|
|
878
|
+
|
|
879
|
+
// 1. GATHER phase: collect critical cluster nominees (skip if single collection)
|
|
880
|
+
const criticalBlockIds = Array.from(criticalBlocks.values());
|
|
881
|
+
const tGather = Date.now();
|
|
882
|
+
const superclusterNominees = await this.gatherPhase(criticalBlockIds);
|
|
883
|
+
const gatherMs = Date.now() - tGather;
|
|
884
|
+
|
|
885
|
+
// 2. PEND phase: distribute to all block clusters
|
|
886
|
+
const tPend = Date.now();
|
|
887
|
+
const pendResult = await this.pendPhase(
|
|
888
|
+
transaction,
|
|
889
|
+
operationsHash,
|
|
890
|
+
collectionTransforms,
|
|
891
|
+
pendedRevs,
|
|
892
|
+
superclusterNominees
|
|
893
|
+
);
|
|
894
|
+
const pendMs = Date.now() - tPend;
|
|
895
|
+
if (!pendResult.success) {
|
|
896
|
+
log('trx:phases trxId=%s gather=%dms pend=%dms (failed) total=%dms', trxId, gatherMs, pendMs, Date.now() - t0);
|
|
897
|
+
return pendResult;
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
// 3. COMMIT phase: commit to all critical blocks (with retry for forward recovery)
|
|
901
|
+
const tCommit = Date.now();
|
|
902
|
+
const commitResult = await this.commitPhase(
|
|
903
|
+
transaction.id as ActionId,
|
|
904
|
+
criticalBlockIds,
|
|
905
|
+
pendResult.pendedBlockIds!,
|
|
906
|
+
pendedRevs
|
|
907
|
+
);
|
|
908
|
+
const commitMs = Date.now() - tCommit;
|
|
909
|
+
if (!commitResult.success) {
|
|
910
|
+
// Targeted cancel: only cancel collections that are still pending (not already committed)
|
|
911
|
+
await this.cancelPhase(
|
|
912
|
+
transaction.id as ActionId,
|
|
913
|
+
pendResult.pendedBlockIds!,
|
|
914
|
+
commitResult.committedCollections
|
|
915
|
+
);
|
|
916
|
+
log('trx:phases trxId=%s gather=%dms pend=%dms commit=%dms (failed) total=%dms', trxId, gatherMs, pendMs, commitMs, Date.now() - t0);
|
|
917
|
+
// Surface the committed/failed partition so commit()/execute() can report which
|
|
918
|
+
// collections durably landed. A non-empty committedCollections is a PARTIAL commit:
|
|
919
|
+
// those collections cannot be rolled back and the caller must reconcile.
|
|
920
|
+
return {
|
|
921
|
+
success: false,
|
|
922
|
+
error: commitResult.error,
|
|
923
|
+
committedCollections: commitResult.committedCollections,
|
|
924
|
+
failedCollections: commitResult.failedCollections,
|
|
925
|
+
staleLoss: commitResult.staleLoss,
|
|
926
|
+
};
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
// 4. PROPAGATE and CHECKPOINT phases are handled by clusters automatically
|
|
930
|
+
// (as per user's note: "managed by each cluster, the client doesn't have to worry about them")
|
|
931
|
+
|
|
932
|
+
log('trx:phases trxId=%s gather=%dms pend=%dms commit=%dms total=%dms', trxId, gatherMs, pendMs, commitMs, Date.now() - t0);
|
|
933
|
+
return { success: true };
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
/**
|
|
937
|
+
* GATHER phase: Collect nominees from critical clusters.
|
|
938
|
+
*
|
|
939
|
+
* Skip if only one collection affected (single-collection consensus).
|
|
940
|
+
*
|
|
941
|
+
* @param criticalBlockIds - Block IDs of all log tails
|
|
942
|
+
* @returns Set of peer IDs to use for consensus, or null for single-collection
|
|
943
|
+
*/
|
|
944
|
+
private async gatherPhase(
|
|
945
|
+
criticalBlockIds: readonly BlockId[]
|
|
946
|
+
): Promise<ReadonlySet<PeerId> | null> {
|
|
947
|
+
// Skip GATHER if only one collection affected
|
|
948
|
+
if (criticalBlockIds.length === 1) {
|
|
949
|
+
return null; // Use normal single-collection consensus
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
// Check if transactor supports cluster queries (optional method)
|
|
953
|
+
if (!this.transactor.queryClusterNominees) {
|
|
954
|
+
// Transactor doesn't support cluster queries - proceed without supercluster
|
|
955
|
+
return null;
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
// Query each critical cluster for their nominees and merge into supercluster
|
|
959
|
+
const nomineePromises = criticalBlockIds.map(blockId =>
|
|
960
|
+
this.transactor.queryClusterNominees!(blockId)
|
|
961
|
+
);
|
|
962
|
+
const results = await Promise.all(nomineePromises);
|
|
963
|
+
|
|
964
|
+
// Merge all nominees into a single set, deduped by peer identity. Each
|
|
965
|
+
// queryClusterNominees builds a fresh PeerId object per call (peerIdFromString),
|
|
966
|
+
// so a Set keyed by object reference would keep the same physical peer twice when
|
|
967
|
+
// it nominates for two critical clusters. Key by toString() to collapse duplicates.
|
|
968
|
+
const byId = results.reduce(
|
|
969
|
+
(acc, result) => {
|
|
970
|
+
result.nominees.forEach(nominee => acc.set(nominee.toString(), nominee));
|
|
971
|
+
return acc;
|
|
972
|
+
},
|
|
973
|
+
new Map<string, PeerId>()
|
|
974
|
+
);
|
|
975
|
+
|
|
976
|
+
return new Set(byId.values());
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
/**
|
|
980
|
+
* PEND phase: Distribute transaction to all affected block clusters.
|
|
981
|
+
*
|
|
982
|
+
* @param transaction - The full transaction for replay/validation
|
|
983
|
+
* @param operationsHash - Hash of all operations for validation
|
|
984
|
+
* @param collectionTransforms - Map of collectionId to its transforms
|
|
985
|
+
* @param pendedRevs - Per collection, the revision its log entry was stamped with — the pend
|
|
986
|
+
* request repeats it verbatim rather than recomputing from the collection (a recompute after
|
|
987
|
+
* the append could name a different number if the collection refreshed in between).
|
|
988
|
+
* @param superclusterNominees - Nominees for multi-collection consensus (null for single-collection)
|
|
989
|
+
*/
|
|
990
|
+
private async pendPhase(
|
|
991
|
+
transaction: Transaction,
|
|
992
|
+
operationsHash: string,
|
|
993
|
+
collectionTransforms: ReadonlyMap<CollectionId, Transforms>,
|
|
994
|
+
pendedRevs: ReadonlyMap<CollectionId, number>,
|
|
995
|
+
superclusterNominees: ReadonlySet<PeerId> | null
|
|
996
|
+
): Promise<{ success: boolean; error?: string; pendedBlockIds?: Map<CollectionId, BlockId[]>; staleLoss?: boolean }> {
|
|
997
|
+
if (collectionTransforms.size === 0) {
|
|
998
|
+
return { success: false, error: 'No transforms to pend' };
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
const actionId = transaction.id as ActionId;
|
|
1002
|
+
const nominees = superclusterNominees ? Array.from(superclusterNominees) : undefined;
|
|
1003
|
+
|
|
1004
|
+
// Fan out the independent per-collection pends concurrently. Each settles to a
|
|
1005
|
+
// { collectionId, blockIds } on success, or rejects with the per-collection reason.
|
|
1006
|
+
// NOTE: unbounded fan-out — one concurrent coordinator round-trip per collection.
|
|
1007
|
+
// Transactions touch few collections today; if one ever spans very many, bound this
|
|
1008
|
+
// with a concurrency limiter so peak in-flight round-trips stays sane. Same for commitPhase.
|
|
1009
|
+
const outcomes = await Promise.allSettled(
|
|
1010
|
+
Array.from(collectionTransforms.entries()).map(([collectionId, transforms]) =>
|
|
1011
|
+
this.pendCollection(transaction, operationsHash, collectionId, transforms, pendedRevs.get(collectionId)!, actionId, nominees)
|
|
1012
|
+
)
|
|
1013
|
+
);
|
|
1014
|
+
|
|
1015
|
+
// Partition settled results: every collection that DID pend (keyed with its block
|
|
1016
|
+
// ids), plus the first failure reason if any collection failed.
|
|
1017
|
+
const pendedBlockIds = new Map<CollectionId, BlockId[]>();
|
|
1018
|
+
let failure: string | undefined;
|
|
1019
|
+
// Classify across ALL failures (mirroring commitPhase, and independent of iteration order):
|
|
1020
|
+
// the pend is a retryable clean stale loss only if at least one failure was a conflicting pend
|
|
1021
|
+
// (PendRejectedError.conflict) AND none was a hard failure. A single hard failure (storage/
|
|
1022
|
+
// policy rejection, or a thrown/unavailable transactor) will not clear on a re-read, so
|
|
1023
|
+
// re-driving it would just burn the retry budget — fail fast instead.
|
|
1024
|
+
let anyConflict = false;
|
|
1025
|
+
let anyHard = false;
|
|
1026
|
+
for (const outcome of outcomes) {
|
|
1027
|
+
if (outcome.status === 'fulfilled') {
|
|
1028
|
+
pendedBlockIds.set(outcome.value.collectionId, outcome.value.blockIds);
|
|
1029
|
+
} else {
|
|
1030
|
+
if (failure === undefined) {
|
|
1031
|
+
failure = outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason);
|
|
1032
|
+
}
|
|
1033
|
+
if (outcome.reason instanceof PendRejectedError && outcome.reason.conflict) anyConflict = true;
|
|
1034
|
+
else anyHard = true;
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
if (failure !== undefined) {
|
|
1039
|
+
// Any failure aborts the whole pend. With concurrency several collections may
|
|
1040
|
+
// have pended in parallel, so cancel EVERY successfully-pended collection — not
|
|
1041
|
+
// only those started before the failure. Cancels are best-effort (cancelPhase
|
|
1042
|
+
// swallows their errors) so they cannot mask the original pend failure.
|
|
1043
|
+
await this.cancelPhase(actionId, pendedBlockIds);
|
|
1044
|
+
return { success: false, error: failure, staleLoss: anyConflict && !anyHard };
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
return { success: true, pendedBlockIds };
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
/**
|
|
1051
|
+
* Pend a single collection's transforms. Resolves with the collection id and its
|
|
1052
|
+
* pended block ids on success; throws with a per-collection reason on failure so the
|
|
1053
|
+
* fan-out in {@link pendPhase} can settle it as a rejection.
|
|
1054
|
+
*/
|
|
1055
|
+
private async pendCollection(
|
|
1056
|
+
transaction: Transaction,
|
|
1057
|
+
operationsHash: string,
|
|
1058
|
+
collectionId: CollectionId,
|
|
1059
|
+
transforms: Transforms,
|
|
1060
|
+
/** The revision the log entry was stamped with (threaded from applyActionsToCollection),
|
|
1061
|
+
* NOT recomputed here: a `getNextRev()` after the append round trips could name a number
|
|
1062
|
+
* a concurrent refresh already moved past. */
|
|
1063
|
+
rev: number,
|
|
1064
|
+
actionId: ActionId,
|
|
1065
|
+
nominees: PeerId[] | undefined
|
|
1066
|
+
): Promise<{ collectionId: CollectionId; blockIds: BlockId[] }> {
|
|
1067
|
+
const collection = this.collections.get(collectionId);
|
|
1068
|
+
if (!collection) {
|
|
1069
|
+
throw new Error(`Collection not found: ${collectionId}`);
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
// Create pend request with the validation payload (transaction + operations hash) —
|
|
1073
|
+
// always BOTH, as one pair: this is the only producer of PendRequest.validation.
|
|
1074
|
+
const pendRequest: PendRequest = {
|
|
1075
|
+
actionId,
|
|
1076
|
+
rev,
|
|
1077
|
+
transforms,
|
|
1078
|
+
policy: 'r', // Return policy: fail but return pending actions
|
|
1079
|
+
validation: { transaction, operationsHash },
|
|
1080
|
+
superclusterNominees: nominees
|
|
1081
|
+
};
|
|
1082
|
+
|
|
1083
|
+
const pendResult = await this.transactor.pend(pendRequest);
|
|
1084
|
+
if (!pendResult.success) {
|
|
1085
|
+
// Retryability comes from the response itself: a producer that classified the failure sets
|
|
1086
|
+
// `conflict`, and only where no producer set it do we fall back to inferring from
|
|
1087
|
+
// `missing`/`pending`. Either way a conflict is an optimistic-concurrency loss, clearable
|
|
1088
|
+
// by a re-read; anything else is a hard rejection (storage/policy) that re-driving won't fix.
|
|
1089
|
+
throw new PendRejectedError(collectionId, isConflictFailure(pendResult), pendResult.reason, pendResult.staleAt);
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
return { collectionId, blockIds: pendResult.blockIds };
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
/**
|
|
1096
|
+
* COMMIT phase: Commit to all critical blocks with retry for transient failures.
|
|
1097
|
+
*
|
|
1098
|
+
* Once all collections are pended (Phase 1 passes), the coordinator has decided
|
|
1099
|
+
* to commit. Failed commits are retried (forward recovery) before giving up.
|
|
1100
|
+
* Returns which collections committed vs failed so the caller can do targeted cancel.
|
|
1101
|
+
*/
|
|
1102
|
+
private async commitPhase(
|
|
1103
|
+
actionId: ActionId,
|
|
1104
|
+
criticalBlockIds: BlockId[],
|
|
1105
|
+
pendedBlockIds: Map<CollectionId, BlockId[]>,
|
|
1106
|
+
pendedRevs: ReadonlyMap<CollectionId, number>
|
|
1107
|
+
): Promise<{
|
|
1108
|
+
success: boolean;
|
|
1109
|
+
error?: string;
|
|
1110
|
+
committedCollections: Set<CollectionId>;
|
|
1111
|
+
failedCollections: Set<CollectionId>;
|
|
1112
|
+
staleLoss?: boolean;
|
|
1113
|
+
}> {
|
|
1114
|
+
// Fan out the independent per-collection commit-with-retry concurrently, then
|
|
1115
|
+
// aggregate the committed/failed partition from the settled results.
|
|
1116
|
+
const outcomes = await Promise.allSettled(
|
|
1117
|
+
Array.from(pendedBlockIds.entries()).map(([collectionId, blockIds]) =>
|
|
1118
|
+
this.commitCollection(actionId, criticalBlockIds, collectionId, blockIds, pendedRevs.get(collectionId)!)
|
|
1119
|
+
)
|
|
1120
|
+
);
|
|
1121
|
+
|
|
1122
|
+
const committedCollections = new Set<CollectionId>();
|
|
1123
|
+
const failedCollections = new Set<CollectionId>();
|
|
1124
|
+
const errors: string[] = [];
|
|
1125
|
+
// Classify failures: a returned stale loss (someone committed a newer rev) is retryable after
|
|
1126
|
+
// a re-read; a thrown/transient-exhausted or structural failure is not. staleLoss holds only
|
|
1127
|
+
// if EVERY failure was a stale loss — a single hard failure makes the whole attempt not worth
|
|
1128
|
+
// re-driving.
|
|
1129
|
+
let anyStale = false;
|
|
1130
|
+
let anyHard = false;
|
|
1131
|
+
for (const outcome of outcomes) {
|
|
1132
|
+
if (outcome.status === 'fulfilled') {
|
|
1133
|
+
const { collectionId, committed, error, stale } = outcome.value;
|
|
1134
|
+
if (committed) {
|
|
1135
|
+
committedCollections.add(collectionId);
|
|
1136
|
+
} else {
|
|
1137
|
+
failedCollections.add(collectionId);
|
|
1138
|
+
if (error) errors.push(error);
|
|
1139
|
+
if (stale) anyStale = true; else anyHard = true;
|
|
1140
|
+
}
|
|
1141
|
+
} else {
|
|
1142
|
+
// commitCollection resolves rather than rejects, but treat any unexpected
|
|
1143
|
+
// rejection as a (hard) failure so the partitioned sets stay honest.
|
|
1144
|
+
errors.push(outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason));
|
|
1145
|
+
anyHard = true;
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
if (failedCollections.size > 0 || errors.length > 0) {
|
|
1150
|
+
return {
|
|
1151
|
+
success: false,
|
|
1152
|
+
error: errors.join('; ') || 'Commit failed',
|
|
1153
|
+
committedCollections,
|
|
1154
|
+
failedCollections,
|
|
1155
|
+
staleLoss: anyStale && !anyHard,
|
|
1156
|
+
};
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
return { success: true, committedCollections, failedCollections };
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
/**
|
|
1163
|
+
* Commit a single collection's pended blocks, retrying transient failures up to three
|
|
1164
|
+
* times (forward recovery). Always resolves — success is carried in the returned
|
|
1165
|
+
* `committed` flag — so the fan-out in {@link commitPhase} can aggregate every result.
|
|
1166
|
+
*/
|
|
1167
|
+
private async commitCollection(
|
|
1168
|
+
actionId: ActionId,
|
|
1169
|
+
criticalBlockIds: BlockId[],
|
|
1170
|
+
collectionId: CollectionId,
|
|
1171
|
+
blockIds: BlockId[],
|
|
1172
|
+
/** The revision this collection PENDED at, threaded from the log append — the same bug
|
|
1173
|
+
* family as pendCollection's: recomputing `getNextRev()` here, after the pend round
|
|
1174
|
+
* trips, could stamp the CommitRequest with a different number than the pend named. */
|
|
1175
|
+
rev: number
|
|
1176
|
+
): Promise<{ collectionId: CollectionId; committed: boolean; error?: string; stale?: boolean }> {
|
|
1177
|
+
const collection = this.collections.get(collectionId);
|
|
1178
|
+
if (!collection) {
|
|
1179
|
+
return { collectionId, committed: false, error: `Collection not found: ${collectionId}` };
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
// Find the critical block (log tail) for this collection
|
|
1183
|
+
const logTailBlockId = criticalBlockIds.find(blockId => blockIds.includes(blockId));
|
|
1184
|
+
if (!logTailBlockId) {
|
|
1185
|
+
return { collectionId, committed: false, error: `Log tail block not found for collection ${collectionId}` };
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
// Declare what each pended block will contain once committed. The collection's tracker still
|
|
1189
|
+
// holds this transaction's staged transforms (it is reset only after commit succeeds) and
|
|
1190
|
+
// layers over the collection's CacheSource, so this is a purely local computation; an id whose
|
|
1191
|
+
// base is not cached is omitted and falls back to corroboration on the member side. Only the
|
|
1192
|
+
// client can declare this — CoordinatorRepo.commit forwards without materializing.
|
|
1193
|
+
const blockDigests = await computeBlockContentDigests(collection.tracker, blockIds);
|
|
1194
|
+
|
|
1195
|
+
// Create commit request
|
|
1196
|
+
const commitRequest: CommitRequest = {
|
|
1197
|
+
actionId,
|
|
1198
|
+
blockIds,
|
|
1199
|
+
tailId: logTailBlockId,
|
|
1200
|
+
rev,
|
|
1201
|
+
...blockDigestsField(blockDigests)
|
|
1202
|
+
};
|
|
1203
|
+
|
|
1204
|
+
// Retry ONLY transient/thrown failures (unreachable peers, timeout) — forward recovery.
|
|
1205
|
+
// A returned { success:false } is a permanent stale loss (someone committed a newer rev);
|
|
1206
|
+
// the identical request can never win, so return immediately without retrying. Either way
|
|
1207
|
+
// cancelPhase (run by coordinateTransaction on commitPhase failure) releases the pend
|
|
1208
|
+
// exactly once — commit itself no longer self-cancels.
|
|
1209
|
+
let lastTransientError: string | undefined;
|
|
1210
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
1211
|
+
try {
|
|
1212
|
+
const commitResult = await this.transactor.commit(commitRequest);
|
|
1213
|
+
if (commitResult.success) {
|
|
1214
|
+
return { collectionId, committed: true };
|
|
1215
|
+
}
|
|
1216
|
+
// Permanent stale failure: do not retry here. It IS a clean stale loss, though, so
|
|
1217
|
+
// mark it retryable at the coordinator level (after a re-read advances the rev).
|
|
1218
|
+
// NOTE: deliberately does NOT consult `isConflictFailure` / `StaleFailure.conflict`
|
|
1219
|
+
// like the pend path does. Once the pend succeeded, a returned commit failure means
|
|
1220
|
+
// the revision slot moved. One commit producer DOES set `conflict` now —
|
|
1221
|
+
// db-p2p's CoordinatorRepo.commit returns lost commit-consensus races and classified
|
|
1222
|
+
// stale-commit rejections as `{ success:false, conflict:true }` (returning, not
|
|
1223
|
+
// throwing, is what keeps them out of the verbatim retry above) — but every returned
|
|
1224
|
+
// failure still maps to `stale: true` here, and `isConflictFailure` covers that new
|
|
1225
|
+
// shape, so no behavior change is needed. If a commit producer ever starts returning
|
|
1226
|
+
// hard commit rejections (validator policy, storage fault) as results too, gate
|
|
1227
|
+
// `stale` on isConflictFailure here.
|
|
1228
|
+
return {
|
|
1229
|
+
collectionId,
|
|
1230
|
+
committed: false,
|
|
1231
|
+
stale: true,
|
|
1232
|
+
error: commitResult.reason ?? `Stale commit for collection ${collectionId}`
|
|
1233
|
+
};
|
|
1234
|
+
} catch (e) {
|
|
1235
|
+
lastTransientError = e instanceof Error ? e.message : String(e);
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
return { collectionId, committed: false, error: `Commit failed for collection ${collectionId} after 3 attempts: ${lastTransientError}` };
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
/**
|
|
1242
|
+
* CANCEL phase: Cancel pending actions on affected blocks.
|
|
1243
|
+
*
|
|
1244
|
+
* Uses the authoritative pended block IDs from pendPhase rather than
|
|
1245
|
+
* recomputing from transforms. Optionally skips already-committed collections.
|
|
1246
|
+
*/
|
|
1247
|
+
private async cancelPhase(
|
|
1248
|
+
actionId: ActionId,
|
|
1249
|
+
pendedBlockIds: Map<CollectionId, BlockId[]>,
|
|
1250
|
+
excludeCollections?: Set<CollectionId>
|
|
1251
|
+
): Promise<void> {
|
|
1252
|
+
// Fan out the per-collection cancels concurrently. Each is best-effort: a cancel
|
|
1253
|
+
// fault is logged and swallowed so it cannot mask the pend/commit failure that
|
|
1254
|
+
// triggered this sweep, and so one failed cancel does not abort the others.
|
|
1255
|
+
const cancels = Array.from(pendedBlockIds.entries())
|
|
1256
|
+
.filter(([collectionId]) => !excludeCollections?.has(collectionId))
|
|
1257
|
+
.map(([collectionId, blockIds]) =>
|
|
1258
|
+
this.transactor.cancel({ actionId, blockIds }).catch(err => {
|
|
1259
|
+
log('cancelPhase: best-effort cancel failed collection=%s: %o', collectionId, err);
|
|
1260
|
+
})
|
|
1261
|
+
);
|
|
1262
|
+
await Promise.all(cancels);
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
}
|
|
1266
|
+
|