@optimystic/db-core 1.26.0 → 1.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/dist/src/chain/chain.d.ts +4 -0
  2. package/dist/src/chain/chain.d.ts.map +1 -1
  3. package/dist/src/chain/chain.js +15 -3
  4. package/dist/src/chain/chain.js.map +1 -1
  5. package/dist/src/cluster/membership.d.ts +9 -2
  6. package/dist/src/cluster/membership.d.ts.map +1 -1
  7. package/dist/src/cluster/membership.js +51 -11
  8. package/dist/src/cluster/membership.js.map +1 -1
  9. package/dist/src/cluster/structs.d.ts +21 -0
  10. package/dist/src/cluster/structs.d.ts.map +1 -1
  11. package/dist/src/cohort-topic/willingness.d.ts +12 -11
  12. package/dist/src/cohort-topic/willingness.d.ts.map +1 -1
  13. package/dist/src/cohort-topic/willingness.js +21 -11
  14. package/dist/src/cohort-topic/willingness.js.map +1 -1
  15. package/dist/src/collection/collection.d.ts +46 -2
  16. package/dist/src/collection/collection.d.ts.map +1 -1
  17. package/dist/src/collection/collection.js +65 -15
  18. package/dist/src/collection/collection.js.map +1 -1
  19. package/dist/src/collection/segments.d.ts +16 -3
  20. package/dist/src/collection/segments.d.ts.map +1 -1
  21. package/dist/src/collection/segments.js +24 -7
  22. package/dist/src/collection/segments.js.map +1 -1
  23. package/dist/src/collection/write-attempts.d.ts +5 -3
  24. package/dist/src/collection/write-attempts.d.ts.map +1 -1
  25. package/dist/src/collection/write-attempts.js +7 -5
  26. package/dist/src/collection/write-attempts.js.map +1 -1
  27. package/dist/src/collections/tree/tree.d.ts +5 -4
  28. package/dist/src/collections/tree/tree.d.ts.map +1 -1
  29. package/dist/src/collections/tree/tree.js +1 -1
  30. package/dist/src/collections/tree/tree.js.map +1 -1
  31. package/dist/src/network/block-size.d.ts +30 -5
  32. package/dist/src/network/block-size.d.ts.map +1 -1
  33. package/dist/src/network/block-size.js +34 -6
  34. package/dist/src/network/block-size.js.map +1 -1
  35. package/dist/src/network/i-key-network.d.ts +28 -0
  36. package/dist/src/network/i-key-network.d.ts.map +1 -1
  37. package/dist/src/network/i-peer-network.d.ts +5 -0
  38. package/dist/src/network/i-peer-network.d.ts.map +1 -1
  39. package/dist/src/network/i-repo.d.ts +4 -0
  40. package/dist/src/network/i-repo.d.ts.map +1 -1
  41. package/dist/src/network/promise-shortfall-error.d.ts +5 -1
  42. package/dist/src/network/promise-shortfall-error.d.ts.map +1 -1
  43. package/dist/src/network/promise-shortfall-error.js +5 -1
  44. package/dist/src/network/promise-shortfall-error.js.map +1 -1
  45. package/dist/src/network/struct.d.ts +5 -1
  46. package/dist/src/network/struct.d.ts.map +1 -1
  47. package/dist/src/network/struct.js.map +1 -1
  48. package/dist/src/network/transaction-size.d.ts +37 -8
  49. package/dist/src/network/transaction-size.d.ts.map +1 -1
  50. package/dist/src/network/transaction-size.js +36 -7
  51. package/dist/src/network/transaction-size.js.map +1 -1
  52. package/dist/src/transaction/coordinator.d.ts +5 -3
  53. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  54. package/dist/src/transaction/coordinator.js +9 -7
  55. package/dist/src/transaction/coordinator.js.map +1 -1
  56. package/dist/src/transform/helpers.d.ts +12 -2
  57. package/dist/src/transform/helpers.d.ts.map +1 -1
  58. package/dist/src/transform/helpers.js +18 -2
  59. package/dist/src/transform/helpers.js.map +1 -1
  60. package/package.json +1 -1
  61. package/src/chain/chain.ts +15 -3
  62. package/src/cluster/membership.ts +129 -87
  63. package/src/cluster/structs.ts +21 -0
  64. package/src/cohort-topic/willingness.ts +39 -22
  65. package/src/collection/collection.ts +78 -14
  66. package/src/collection/segments.ts +30 -7
  67. package/src/collection/write-attempts.ts +7 -5
  68. package/src/collections/tree/tree.ts +5 -4
  69. package/src/network/block-size.ts +40 -6
  70. package/src/network/i-key-network.ts +30 -0
  71. package/src/network/i-peer-network.ts +6 -0
  72. package/src/network/i-repo.ts +4 -0
  73. package/src/network/promise-shortfall-error.ts +5 -1
  74. package/src/network/struct.ts +5 -1
  75. package/src/network/transaction-size.ts +35 -7
  76. package/src/transaction/coordinator.ts +9 -6
  77. package/src/transform/helpers.ts +19 -2
@@ -13,9 +13,10 @@ import { TransactorSource, answeredBlock, servedRevision } from "../transactor/t
13
13
  import { BlockFloors } from "../transactor/block-floors.js";
14
14
  import type { WriteDurability } from "../network/struct.js";
15
15
  import { withBlockRevs } from "../network/durability.js";
16
+ import { assertStagedActionsFit, jsonEncodedBytes } from "../network/transaction-size.js";
16
17
  import type { CollectionHeaderBlock, CollectionId, CommittedTail, CommittedTailAnswer, CommittedTailQuery, CommittedTailReadOptions, HeadCommitOptions, ICollection, InFlightAttempt, InFlightCycle, OpenHeadCommitOptions, SyncOptions, TornActionReason } from "./index.js";
17
18
  import { ActionRefusedError, CollectionHeaderVanishedError, SealedSegmentsError, TornActionError } from "./struct.js";
18
- import { SegmentList, unitsOf } from "./segments.js";
19
+ import { SegmentList, unitOf } from "./segments.js";
19
20
  import type { EntryId, SealedSegment, StagedUnit } from "./segments.js";
20
21
  import type { ActionContext, ActionRev, AlreadyAppliedTally } from "./action.js";
21
22
  import { actionIdAt, addTallies, ZeroTally } from "./action.js";
@@ -187,6 +188,15 @@ export type CollectionInitOptions<TAction> = {
187
188
  * and inside await-free steps on all of them, so it must be synchronous and must not call into
188
189
  * this collection; a throw out of it is caught and logged (`collection:view-changed-callback-threw`). */
189
190
  onViewChanged?: () => void;
191
+ /** Called, synchronously, once per refresh that adopted changes this instance did not write: log
192
+ * entries other writers committed, a durable invalidation, or another lineage (see
193
+ * docs/internals.md §Reactive Watch Bridge). `rev` is the revision adopted. This instance's own
194
+ * commits never count: a successful one is folded in and the held revision moves past it, a torn
195
+ * one is finished and consumed as the write's own, and one still on the network defers the
196
+ * refresh. Runs inside the refresh's await-free adopt step, under the collection latch on most
197
+ * paths, so it must be synchronous and must not call into this collection; a throw out of it is
198
+ * caught and logged (`collection:external-commits-callback-threw`). */
199
+ onExternalCommits?: (rev: number) => void;
190
200
  }
191
201
 
192
202
  /** Options for building a committed read view (see {@link Collection.createReadTracker}
@@ -392,6 +402,8 @@ export class Collection<TAction> implements ICollection<TAction> {
392
402
  private readonly onSealedRefused?: (entryId: EntryId, error: ActionRefusedError) => void,
393
403
  /** See {@link CollectionInitOptions.onViewChanged}. */
394
404
  private readonly onViewChanged?: () => void,
405
+ /** See {@link CollectionInitOptions.onExternalCommits}. */
406
+ private readonly onExternalCommits?: (rev: number) => void,
395
407
  /** Short random tag naming THIS instance (see {@link newInstanceTag}). Open paths generate
396
408
  * it BEFORE construction (so pre-construction diagnostics such as attachToLog can carry it);
397
409
  * the default covers direct construction in tests. */
@@ -442,7 +454,7 @@ export class Collection<TAction> implements ICollection<TAction> {
442
454
  return undefined;
443
455
  }
444
456
  await Collection.attachToLog<TAction>(source, sourceCache, tracker, floors, id, instanceTag, ends, clockOf(transactor));
445
- const collection = new Collection(id, transactor, init.modules, source, sourceCache, tracker, floors, init.merge, init.onSealedRefused, init.onViewChanged, instanceTag);
457
+ const collection = new Collection(id, transactor, init.modules, source, sourceCache, tracker, floors, init.merge, init.onSealedRefused, init.onViewChanged, init.onExternalCommits, instanceTag);
446
458
  collection.logTailId = ends.header.tailId;
447
459
  return collection;
448
460
  }
@@ -473,7 +485,7 @@ export class Collection<TAction> implements ICollection<TAction> {
473
485
  inventedBaseline = copyTransforms(tracker.transforms);
474
486
  }
475
487
 
476
- const collection = new Collection(id, transactor, init.modules, source, sourceCache, tracker, floors, init.merge, init.onSealedRefused, init.onViewChanged, instanceTag);
488
+ const collection = new Collection(id, transactor, init.modules, source, sourceCache, tracker, floors, init.merge, init.onSealedRefused, init.onViewChanged, init.onExternalCommits, instanceTag);
477
489
  collection.logTailId = ends?.header.tailId;
478
490
  collection.inventedBaseline = inventedBaseline;
479
491
  return collection;
@@ -881,12 +893,20 @@ export class Collection<TAction> implements ICollection<TAction> {
881
893
  }
882
894
  }
883
895
 
896
+ /** Stage `actions` into the OPEN segment as one unit — a sealed segment is closed to further
897
+ * staging — recording what the staging produced with them, so the segment they are sealed into
898
+ * knows what it staged without staging it again (SealedSegment.delta).
899
+ *
900
+ * Refused before anything is staged when the open segment's actions and these would encode past
901
+ * the per-transaction limit together (`assertStagedActionsFit`): every one of them rides verbatim
902
+ * in the next commit's log entry, so that commit could never be sent, and staging more of it would
903
+ * only cost time and memory. Each action is measured once, here, and the size kept on its unit.
904
+ * Replays do not come through here: they re-stage what was already admitted. */
884
905
  private async actInternal(...actions: Action<TAction>[]) {
906
+ const sizes = actions.map(action => jsonEncodedBytes(action));
907
+ assertStagedActionsFit(sizes.reduce((total, size) => total + size, this.segments.openBytes()));
885
908
  const { transforms, applied } = await this.internalTransact(...actions);
886
- // Staging appends to the OPEN segment: a sealed segment is closed to further staging. What
887
- // the staging produced is recorded with the actions, so the segment they are sealed into
888
- // knows what it staged without staging it again (SealedSegment.delta).
889
- this.segments.pushOpen(actions, transforms, applied);
909
+ this.segments.pushOpen(actions, sizes, transforms, applied);
890
910
  }
891
911
 
892
912
  private internalTransact(...actions: Action<TAction>[]): Promise<Staging> {
@@ -937,6 +957,18 @@ export class Collection<TAction> implements ICollection<TAction> {
937
957
  }
938
958
  }
939
959
 
960
+ /** Tell {@link CollectionInitOptions.onExternalCommits} that a refresh adopted revision `rev` and,
961
+ * with it, changes this instance did not write. Called only from {@link forgetAndAdopt}, after the
962
+ * adoption and in the same synchronous step. */
963
+ private externalCommitsAdopted(rev: number): void {
964
+ try {
965
+ this.onExternalCommits?.(rev);
966
+ } catch (err) {
967
+ log('collection:external-commits-callback-threw id=%s tag=%s rev=%d error=%s',
968
+ this.id, this.instanceTag, rev, err instanceof Error ? err.message : String(err));
969
+ }
970
+ }
971
+
940
972
  /** Load external changes and update our context to the latest log revision - resolve any conflicts with our pending actions. */
941
973
  async update() {
942
974
  const release = await Latches.acquire(this.latchId);
@@ -1336,7 +1368,7 @@ export class Collection<TAction> implements ICollection<TAction> {
1336
1368
  } else {
1337
1369
  // A replacement's staging is not on record; the replay this forces records it.
1338
1370
  changed = true;
1339
- if (effective !== undefined) after.push({ action: effective });
1371
+ if (effective !== undefined) after.push(unitOf(effective));
1340
1372
  }
1341
1373
  }
1342
1374
  this.segments.setOpen(after);
@@ -1490,7 +1522,8 @@ export class Collection<TAction> implements ICollection<TAction> {
1490
1522
  anyConflicts = true;
1491
1523
  }
1492
1524
 
1493
- const adoption = this.forgetAndAdopt(latest, entryRevs, revertedBlockIds, ends);
1525
+ const othersWrote = Collection.walkedOthersWrites(latest?.entries ?? [], ownEntry, invalidations.length);
1526
+ const adoption = this.forgetAndAdopt(latest, entryRevs, revertedBlockIds, ends, othersWrote);
1494
1527
 
1495
1528
  Collection.reportShortfall(this.id, this.instanceTag, tailRev, actionContext?.rev, this.source.actionContext?.rev);
1496
1529
 
@@ -1550,12 +1583,22 @@ export class Collection<TAction> implements ICollection<TAction> {
1550
1583
  * write through it never mints below an entry it has seen — whatever the verdict, since a clock
1551
1584
  * only rises and the entries were read either way.
1552
1585
  *
1586
+ * **Last, it tells the owner when what it adopted includes other writers' changes**
1587
+ * ({@link CollectionInitOptions.onExternalCommits}): the walk found them (`othersWrote`), or the
1588
+ * adoption is another lineage. In this step rather than after it, so the owner hears of it from
1589
+ * every path that refreshes — a reader's, a write's between attempts, a head commit's — and
1590
+ * before anything the refresh does next (the replay) can throw. A refused read adopted nothing
1591
+ * and reports nothing.
1592
+ *
1593
+ * @param othersWrote whether the walk found anything another writer committed
1594
+ * ({@link walkedOthersWrites}).
1553
1595
  * @returns what the adoption did, as {@link judgeAdoption} answered it before anything changed. */
1554
1596
  private forgetAndAdopt(
1555
1597
  latest: GetFromResult<Action<TAction>> | undefined,
1556
1598
  entryRevs: ReadonlyMap<ActionId, number>,
1557
1599
  revertedBlockIds: BlockId[],
1558
1600
  ends: LogEnds,
1601
+ othersWrote: boolean,
1559
1602
  ): Adoption {
1560
1603
  const adoption = Collection.judgeAdoption(this.source.actionContext, latest?.context);
1561
1604
  if (latest?.hlcHighWater !== undefined) clockOf(this.transactor).observe(latest.hlcHighWater);
@@ -1571,9 +1614,29 @@ export class Collection<TAction> implements ICollection<TAction> {
1571
1614
  this.sourceCache.clear(revertedBlockIds);
1572
1615
  Collection.advanceContext(this.source, this.id, this.instanceTag, 'refresh', latest?.context, adoption);
1573
1616
  Collection.keepLogEnds(this.sourceCache, this.floors, this.source.actionContext, ends);
1617
+ const adopted = this.source.actionContext;
1618
+ if (adopted !== undefined && (adoption === 'adopt-divergent' || (adoption === 'adopt' && othersWrote))) {
1619
+ this.externalCommitsAdopted(adopted.rev);
1620
+ }
1574
1621
  return adoption;
1575
1622
  }
1576
1623
 
1624
+ /** Whether a refresh's walk found changes this instance did not write: an action entry other than
1625
+ * `ownEntry` — the in-flight write's own, which the refresh finishes and consumes — or a durable
1626
+ * invalidation, which reverted committed content. Every other action entry above the held
1627
+ * revision is another writer's, because this instance's own commits are folded in and the held
1628
+ * revision moves past them ({@link foldOwnCommit}, {@link recordCommitted}), and a refresh that
1629
+ * meets its head commit still on the network returns before adopting anything
1630
+ * ({@link headInFlight}). `entries` holds action entries only; a checkpoint, which changes no
1631
+ * block, is never among them. */
1632
+ private static walkedOthersWrites<T>(
1633
+ entries: readonly ActionEntry<T>[],
1634
+ ownEntry: ActionEntry<T> | undefined,
1635
+ invalidationCount: number,
1636
+ ): boolean {
1637
+ return invalidationCount > 0 || entries.some(entry => entry !== ownEntry);
1638
+ }
1639
+
1577
1640
  /** Put the header and log tail block {@link readLogEnds} already fetched into a handle's own
1578
1641
  * cache, so the next read of either is a hit rather than a second fetch of a block just
1579
1642
  * received. Two sites, one rule:
@@ -1582,8 +1645,8 @@ export class Collection<TAction> implements ICollection<TAction> {
1582
1645
  * - **A refresh that walked** ({@link forgetAndAdopt}). Without it, a write that follows ANOTHER
1583
1646
  * handle's commit fetches the log tail twice: the refresh reads it, the walked entry names it
1584
1647
  * (a commit's blocks include the log block its entry was appended to) so the clear drops it,
1585
- * and `Chain.add` then reads it again to append this write's own entry. Last in that one
1586
- * synchronous step, after BOTH clears — offering before either would simply be undone by it.
1648
+ * and `Chain.add` then reads it again to append this write's own entry. Last of that one
1649
+ * synchronous step's cache moves, after BOTH clears — offering before either would simply be undone by it.
1587
1650
  * An id that an invalidation reverted is therefore re-offered rather than left forgotten, which
1588
1651
  * is sound because the revert is itself a commit and {@link readLogEnds} read unpinned: what it
1589
1652
  * read already includes the revert.
@@ -3187,12 +3250,13 @@ export class Collection<TAction> implements ICollection<TAction> {
3187
3250
  }
3188
3251
  }
3189
3252
  // The list holds this array from here on, so each unit is recorded in place as its action
3190
- // is re-staged, and a throw leaves the units behind it with no staging on record.
3191
- const units = unitsOf(this.segments.openActions());
3253
+ // is re-staged, and a throw leaves the units behind it with no staging on record. Each keeps
3254
+ // the size measured when it was first staged.
3255
+ const units: StagedUnit<TAction>[] = this.segments.openUnits().map(({ action, bytes }) => ({ action, bytes }));
3192
3256
  this.segments.setOpen(units);
3193
3257
  for (const [index, unit] of units.entries()) {
3194
3258
  const { transforms, applied } = await this.internalTransact(unit.action);
3195
- units[index] = { action: unit.action, delta: transforms, applied };
3259
+ units[index] = { action: unit.action, bytes: unit.bytes, delta: transforms, applied };
3196
3260
  }
3197
3261
  }
3198
3262
 
@@ -3,6 +3,7 @@ import type { AlreadyAppliedTally } from "./action.js";
3
3
  import { addTallies, ZeroTally } from "./action.js";
4
4
  import type { ActionRefusedError } from "./struct.js";
5
5
  import { emptyTransforms, foldTransforms } from "../transform/helpers.js";
6
+ import { jsonEncodedBytes } from "../network/transaction-size.js";
6
7
 
7
8
  /** A queue entry's id, opaque to the collection: the offline-sync queue (docs/offline-sync.md)
8
9
  * mints one per locally committed transaction and seals that transaction's actions under it. */
@@ -19,13 +20,22 @@ export type EntryId = string;
19
20
  * inside a group. Immutable once made, so a snapshot may hold the same unit. */
20
21
  export type StagedUnit<TAction> = {
21
22
  readonly action: Action<TAction>;
23
+ /** What the action encodes to by `jsonEncodedBytes`, measured once, when the unit is made, and
24
+ * carried by every unit rebuilt from it, so the open segment's total ({@link SegmentList.openBytes})
25
+ * never serializes an action twice. */
26
+ readonly bytes: number;
22
27
  readonly delta?: Transforms;
23
28
  readonly applied?: AlreadyAppliedTally;
24
29
  };
25
30
 
31
+ /** The unit for `action` with its staging not on record. */
32
+ export function unitOf<TAction>(action: Action<TAction>): StagedUnit<TAction> {
33
+ return { action, bytes: jsonEncodedBytes(action) };
34
+ }
35
+
26
36
  /** Units for `actions` whose staging is not on record. */
27
37
  export function unitsOf<TAction>(actions: readonly Action<TAction>[]): StagedUnit<TAction>[] {
28
- return actions.map(action => ({ action }));
38
+ return actions.map(unitOf);
29
39
  }
30
40
 
31
41
  /** The actions one queue entry staged into a collection, closed off from later staging. */
@@ -97,6 +107,9 @@ export class SegmentList<TAction> {
97
107
  * ({@link StagedUnit}). Private so that it is replaced through {@link setOpen} alone and grown
98
108
  * through {@link pushOpen} alone, each unit carrying what its staging produced. */
99
109
  private units: StagedUnit<TAction>[] = [];
110
+ /** The units' `bytes` summed: set by {@link setOpen}, grown by {@link pushOpen}, so every way the
111
+ * open segment changes keeps it right. A replay's in-place recording keeps each unit's `bytes`. */
112
+ private unitBytes = 0;
100
113
  /** Bumped on every change to the sealed list — a seal, an unseal, an insert, a drop, a refusal
101
114
  * recorded. Captured by a snapshot so a restore can tell whether the sealed segments it was taken
102
115
  * over are still the ones in place (ticket `offline-sync-open-segment-snapshot`). Read and written
@@ -119,18 +132,28 @@ export class SegmentList<TAction> {
119
132
  return this.units.length;
120
133
  }
121
134
 
135
+ /** What the open segment's actions encode to together, by `jsonEncodedBytes`: the lower bound of
136
+ * the next commit's pend that `Collection.act` holds to the per-transaction limit. */
137
+ openBytes(): number {
138
+ return this.unitBytes;
139
+ }
140
+
122
141
  /** Replace the open segment: the one way it is set. A caller putting actions back whose staging
123
142
  * it does not have records them through {@link unitsOf}, and the next replay records it. */
124
143
  setOpen(units: StagedUnit<TAction>[]): void {
125
144
  this.units = units;
145
+ this.unitBytes = units.reduce((total, unit) => total + unit.bytes, 0);
126
146
  }
127
147
 
128
- /** Append the actions one `act` call staged as one unit, with the delta and tally that staging
129
- * produced, recorded as {@link StagedUnit} describes for a group. */
130
- pushOpen(actions: readonly Action<TAction>[], delta: Transforms, applied: AlreadyAppliedTally): void {
148
+ /** Append the actions one `act` call staged as one unit, with what each encodes to (`sizes`, in
149
+ * the same order) and the delta and tally that staging produced, recorded as {@link StagedUnit}
150
+ * describes for a group. */
151
+ pushOpen(actions: readonly Action<TAction>[], sizes: readonly number[], delta: Transforms, applied: AlreadyAppliedTally): void {
131
152
  const last = actions.length - 1;
132
153
  actions.forEach((action, index) => {
133
- this.units.push(index === last ? { action, delta, applied } : { action, delta: emptyTransforms(), applied: ZeroTally });
154
+ const bytes = sizes[index]!;
155
+ this.units.push(index === last ? { action, bytes, delta, applied } : { action, bytes, delta: emptyTransforms(), applied: ZeroTally });
156
+ this.unitBytes += bytes;
134
157
  });
135
158
  }
136
159
 
@@ -149,7 +172,7 @@ export class SegmentList<TAction> {
149
172
  const segment: SealedSegment<TAction> = { entryId, actions: this.units.map(unit => unit.action), ...foldUnits(this.units) };
150
173
  this.sealed.push(segment);
151
174
  this.byEntry.set(entryId, segment);
152
- this.units = [];
175
+ this.setOpen([]);
153
176
  this.version++;
154
177
  return true;
155
178
  }
@@ -173,7 +196,7 @@ export class SegmentList<TAction> {
173
196
  }
174
197
  this.sealed.pop();
175
198
  this.byEntry.delete(entryId);
176
- this.units = unitsOf(newest.actions);
199
+ this.setOpen(unitsOf(newest.actions));
177
200
  this.version++;
178
201
  }
179
202
 
@@ -8,7 +8,7 @@ import type { BasePins } from "../transform/base-pins.js";
8
8
  import type { CacheSource } from "../transform/cache-source.js";
9
9
  import { copyTransforms } from "../transform/helpers.js";
10
10
  import { describeTouchedBlocks, baseRevsField } from "../transform/digest.js";
11
- import { oversizedBlock } from "../network/block-size.js";
11
+ import { oversizedWriteRefusal } from "../network/block-size.js";
12
12
  import { mergeDurability } from "../network/durability.js";
13
13
  import { highestStaleAt, isExpiryFailure, isRejectedFailure, isShortfallFailure } from "../network/stale-failure.js";
14
14
  import { TransactionExpiredError } from "../network/transaction-expired-error.js";
@@ -296,9 +296,11 @@ export type StagedAttempt = {
296
296
  * cheap relative to the round trips it is retrying; if a high-contention write ever shows digest
297
297
  * hashing in a profile, memoize per (id, staged ops).
298
298
  *
299
- * @throws BlockTooLargeError when a touched block would be left larger than one message can carry —
300
- * before anything is sent, judged apart from the declaration (which never throws), and unretried by
301
- * either write path, since a later attempt would leave the same block. */
299
+ * @throws TransactionTooLargeError when a touched block would be left over the per-block limit and the
300
+ * attempt's transforms are over the per-transaction one, so the pend could not be sent either; and
301
+ * BlockTooLargeError when only the block is — both judged by `oversizedWriteRefusal` before anything is
302
+ * sent, apart from the declaration (which never throws), and unretried by either write path, since a
303
+ * later attempt would be refused the same way. */
302
304
  export async function stageAttempt<TAction>(
303
305
  collectionId: CollectionId,
304
306
  cache: CacheSource<IBlock>,
@@ -315,7 +317,7 @@ export async function stageAttempt<TAction>(
315
317
  const added = await collectionLog.addActions([...target.actions], identity.actionId, rev,
316
318
  () => tracker.transformedBlockIds(), { timestamp: identity.timestamp, hlc: identity.hlc, lane: identity.lane });
317
319
  const touched = await describeTouchedBlocks(tracker, tracker.transformedBlockIds());
318
- const tooLarge = oversizedBlock(touched.bytes);
320
+ const tooLarge = oversizedWriteRefusal(touched.bytes, tracker.transforms, identity.actionId);
319
321
  if (tooLarge) throw tooLarge;
320
322
  return {
321
323
  tracker,
@@ -187,13 +187,14 @@ interface BTreeHolder<TKey, TEntry> {
187
187
 
188
188
  /** The collection options a tree's opener may set — the ones that are not the tree's own to
189
189
  * decide: the sealed-segment refusal callback the offline-sync queue installs (see
190
- * {@link CollectionInitOptions.onSealedRefused}), and the convergent-lane merge declaration
191
- * ({@link CollectionInitOptions.merge}), which a raw tree user may make. `Tree` itself declares none:
190
+ * {@link CollectionInitOptions.onSealedRefused}), the callback that hears of other writers' commits a
191
+ * refresh adopted ({@link CollectionInitOptions.onExternalCommits}), and the convergent-lane merge
192
+ * declaration ({@link CollectionInitOptions.merge}), which a raw tree user may make. `Tree` itself declares none:
192
193
  * a tree is strong unless its opener says otherwise, and the SQL table over a tree never declares
193
194
  * one here, because it classifies per statement and passes the lane on each write
194
195
  * (`SyncOptions.lane`). */
195
196
  export type TreeCollectionOptions<TKey = unknown, TEntry = unknown> =
196
- Pick<CollectionInitOptions<TreeReplaceAction<TKey, TEntry>>, 'onSealedRefused' | 'merge'>;
197
+ Pick<CollectionInitOptions<TreeReplaceAction<TKey, TEntry>>, 'onSealedRefused' | 'onExternalCommits' | 'merge'>;
197
198
 
198
199
  export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
199
200
 
@@ -244,7 +245,7 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
244
245
  * the header and read it back on reopen rather than trusting the caller to re-supply it. */
245
246
  nodeCapacity?: number,
246
247
  /** Collection options the opener sets rather than the tree: the sealed-segment refusal
247
- * callback and the merge declaration (see {@link TreeCollectionOptions}). Omit for a strong
248
+ * callback, the external-commits callback and the merge declaration (see {@link TreeCollectionOptions}). Omit for a strong
248
249
  * tree nothing seals. */
249
250
  options?: TreeCollectionOptions<TKey, TEntry>,
250
251
  ): Promise<Tree<TKey, TEntry>> {
@@ -1,5 +1,7 @@
1
1
  import type { BlockId } from "../blocks/structs.js";
2
- import { MAX_TRANSACTION_BYTES } from "./transaction-size.js";
2
+ import type { ActionId } from "../collection/action.js";
3
+ import type { Transforms } from "../transform/struct.js";
4
+ import { MAX_TRANSACTION_BYTES, TransactionTooLargeError, jsonEncodedBytes } from "./transaction-size.js";
3
5
 
4
6
  /**
5
7
  * The most bytes one block may encode to, measured by `jsonEncodedBytes` (the UTF-8 length of its
@@ -16,7 +18,9 @@ import { MAX_TRANSACTION_BYTES } from "./transaction-size.js";
16
18
  * - **No single transaction can exceed it on its own.** A block holding one entry or child is never
17
19
  * larger than the transaction that wrote it, since that entry is inside the transaction's pend (as the
18
20
  * new block's insert, or as the update operation that appends it), and the pend is held to
19
- * {@link MAX_TRANSACTION_BYTES}.
21
+ * {@link MAX_TRANSACTION_BYTES}. The writer judges the transaction first when both are over
22
+ * ({@link oversizedWriteRefusal}), so a transaction too large to send is refused as one, whatever
23
+ * block it would have grown.
20
24
  *
21
25
  * Structures that hold many entries per block (the log, tree nodes) keep each block that holds more than
22
26
  * one entry near {@link BLOCK_TARGET_BYTES}, far under this limit; this is the bound itself.
@@ -43,10 +47,12 @@ export const BLOCK_TARGET_BYTES = 1024 * 1024;
43
47
  /**
44
48
  * A write that would leave a block larger than {@link MAX_BLOCK_BYTES}: once committed, the block could
45
49
  * never be copied to another machine in one message. Raised by the writer before anything is sent
46
- * ({@link oversizedBlock}), so nothing was reserved and nothing needs cancelling. Not a `StaleFailure`
47
- * and never retried — the same write leaves the same block on every attempt. The structures that hold
48
- * many entries per block split by size (the log rolls over, a tree node divides), so this names a block
49
- * that outgrew the limit some other way; the change has to be written in a form that does not.
50
+ * ({@link oversizedWriteRefusal}), so nothing was reserved and nothing needs cancelling. Not a
51
+ * `StaleFailure` and never retried — the same write leaves the same block on every attempt. The
52
+ * structures that hold many entries per block split by size (the log rolls over, a tree node divides),
53
+ * so this names a block that outgrew the limit some other way — over several writes, each small enough
54
+ * to send, since a write too large to send is refused as a {@link TransactionTooLargeError} instead —
55
+ * and the change has to be written in a form that does not.
50
56
  */
51
57
  export class BlockTooLargeError extends Error {
52
58
  constructor(
@@ -79,3 +85,31 @@ export function oversizedBlock(bytes: Readonly<Record<BlockId, number>>): BlockT
79
85
  }
80
86
  return undefined;
81
87
  }
88
+
89
+ /**
90
+ * The refusal both write paths make before anything is sent, judged on what one collection's attempt
91
+ * would send: `touchedBytes`, per block, what the write would leave its content encoding to, and
92
+ * `transforms`, the attempt's staged transforms with its log append. `undefined` when every block fits.
93
+ *
94
+ * When a block is over {@link MAX_BLOCK_BYTES}, the transaction is judged first: transforms over
95
+ * {@link MAX_TRANSACTION_BYTES} are refused as a {@link TransactionTooLargeError}, since a collection's
96
+ * pend carries its transforms plus an envelope, and any batch it travels in carries that pend, so the
97
+ * transaction could not be sent whatever became of the block. Only a block over the limit inside
98
+ * transforms under it — a block grown over several writes — is a {@link BlockTooLargeError}. The
99
+ * transforms are measured only on that refusal path, so a write that fits pays nothing for it.
100
+ *
101
+ * NOTE: transforms just under the limit can still make a pend over it once the envelope (a validation
102
+ * payload, say) is added, while the write also leaves a block grown by earlier writes over the block
103
+ * limit. That write is refused as the block, not the transaction: both are true of it, and only the
104
+ * block's refusal says the change has to be written in another form, since splitting it would not help.
105
+ */
106
+ export function oversizedWriteRefusal(
107
+ touchedBytes: Readonly<Record<BlockId, number>>,
108
+ transforms: Transforms,
109
+ actionId: ActionId,
110
+ ): TransactionTooLargeError | BlockTooLargeError | undefined {
111
+ const tooLarge = oversizedBlock(touchedBytes);
112
+ if (tooLarge === undefined) return undefined;
113
+ const bytes = jsonEncodedBytes(transforms);
114
+ return bytes > MAX_TRANSACTION_BYTES ? new TransactionTooLargeError(actionId, bytes) : tooLarge;
115
+ }
@@ -31,6 +31,22 @@ export type FindCoordinatorOptions = {
31
31
  };
32
32
 
33
33
 
34
+ /**
35
+ * What {@link IKeyNetwork.connectedCohort} reports for a key: a cohort that is the whole serving
36
+ * network this node knows of, this node among it and every other member connected to it.
37
+ *
38
+ * A **connection run** is the span during which a remote peer has at least one open connection to
39
+ * this node; it ends when the peer's last connection closes, and the next connection starts a new
40
+ * run with a new id. So two reports with equal `members` and equal `runs` prove that no member's
41
+ * connection to this node broke between them.
42
+ */
43
+ export type ConnectedCohort = {
44
+ /** The cohort `findCluster` assembles for the key now, as peer-id strings, this node included, sorted ascending. */
45
+ readonly members: readonly string[];
46
+ /** Per member other than this node: the id of its current connection run. Ids are never reused within a process. */
47
+ readonly runs: Readonly<Record<string, number>>;
48
+ };
49
+
34
50
  /**
35
51
  * Maps a block to the peers responsible for it.
36
52
  *
@@ -81,4 +97,18 @@ export type IKeyNetwork = {
81
97
  * such peers omit this, which reads as "none known".
82
98
  */
83
99
  absentCohortMembers?(key: RoutingKey): Promise<string[]>;
100
+
101
+ /**
102
+ * Optionally report, for `key`, that the cohort `findCluster` would assemble now is every machine
103
+ * that could coordinate a commit to the key's block as far as this node can know — the whole
104
+ * serving network it knows of, this node included — and that every other member is connected to
105
+ * this node, with the connection run each is on (see {@link ConnectedCohort}). Answers from what
106
+ * the node already holds, without dialing or opening a stream.
107
+ *
108
+ * `undefined` means no such guarantee: the cohort is a selection from a wider network, a peer is
109
+ * not yet identified or is remembered but absent, a member is not connected, or the
110
+ * implementation cannot judge. Implementations that track no connections omit this, which reads
111
+ * the same, so a caller then behaves as if the method did not exist.
112
+ */
113
+ connectedCohort?(key: RoutingKey): Promise<ConnectedCohort | undefined>;
84
114
  }
@@ -1,6 +1,12 @@
1
1
  import type { AbortOptions, PeerId, Stream } from "./types.js";
2
2
 
3
3
  export type IPeerNetwork = {
4
+ /**
5
+ * This node's own peer id, so a protocol client never dials or follows a redirect to itself.
6
+ * Implementations without an identity omit it.
7
+ */
8
+ readonly localPeerId?: PeerId;
9
+
4
10
  /**
5
11
  * Dial a peer and establish a protocol stream
6
12
  */
@@ -50,6 +50,10 @@ export type MessageOptions = {
50
50
  * {@link asker}: a path that forgets it leaves the record standing, as every record stood before
51
51
  * the mark existed, whereas a forgotten negative mark would have a live remote writer's record
52
52
  * judged dead.
53
+ *
54
+ * It also says the writer measured the pend against `MAX_TRANSACTION_BYTES` before sending it, so
55
+ * `CoordinatorRepo.pendMany` does not measure it again. An in-process writer that sets the mark
56
+ * must measure first, or an oversized pend to a cohort of one commits unchecked.
53
57
  */
54
58
  localWriter?: true;
55
59
  }
@@ -28,7 +28,11 @@ export class PromiseShortfallError extends Error {
28
28
  readonly peerCount: number;
29
29
  /** The approvals the round needed. */
30
30
  readonly needed: number;
31
- /** Peer-id strings of the cohort members that cast no promise vote at all. */
31
+ /**
32
+ * Peer-id strings of the cohort members whose promise vote the round did not count: those that never
33
+ * answered, and those still unanswered when the answers in hand put the bar out of reach — the round
34
+ * stops waiting then, so a member merely slower than the rest is named here too.
35
+ */
32
36
  readonly silentPeers: readonly string[];
33
37
  /** Why a lone coordinator refused to write alone, when that is what this shortfall is
34
38
  * ({@link PromiseShortfall.soloRefused}); absent when a consensus round fell short. */
@@ -334,7 +334,11 @@ export type PromiseShortfall = {
334
334
  peerCount: number;
335
335
  /** The approvals the round needed: `ceil(peerCount × superMajorityThreshold)`. */
336
336
  needed: number;
337
- /** Peer-id strings of the cohort members that cast no promise vote at all. */
337
+ /**
338
+ * Peer-id strings of the cohort members whose promise vote the round did not count: those that never
339
+ * answered, and those still unanswered when the answers in hand put the bar out of reach (the round
340
+ * stops waiting then, so a member merely slower than the rest is named here too).
341
+ */
338
342
  silentPeers: string[];
339
343
  /**
340
344
  * Set only when no consensus round ran: a coordinator whose cohort for the write was only itself
@@ -33,16 +33,26 @@ export function jsonEncodedBytes(value: unknown): number {
33
33
  * Raised before anything is sent, so nothing was reserved and nothing needs cancelling. Not a
34
34
  * `StaleFailure` and never retried — the same transaction is the same size on every attempt. The
35
35
  * writer has to split the change into smaller transactions.
36
+ *
37
+ * Three places raise it, each as early as it can prove the pend is over the limit: staging, once the
38
+ * actions one collection has staged are over it on their own ({@link assertStagedActionsFit}); building
39
+ * an attempt, once a block the write leaves is over the per-block limit and the attempt's transforms are
40
+ * over this one (`oversizedWriteRefusal`); and the pend itself ({@link assertTransactionFits}).
36
41
  */
37
42
  export class TransactionTooLargeError extends Error {
38
43
  constructor(
39
- readonly actionId: ActionId,
40
- /** What the transaction's pend encodes to, by {@link jsonEncodedBytes}. */
44
+ /** The transaction's action id, or `undefined` when it was refused while still staging, before
45
+ * the commit minted one. */
46
+ readonly actionId: ActionId | undefined,
47
+ /** The transaction's pend encodes to at least this many bytes, by {@link jsonEncodedBytes}:
48
+ * exactly this many when the pend itself was measured, more when the writer measured a part of it
49
+ * before building it — the attempt's transforms, or the actions staged so far. */
41
50
  readonly bytes: number,
42
51
  /** The limit it exceeded: {@link MAX_TRANSACTION_BYTES}. */
43
52
  readonly limit: number = MAX_TRANSACTION_BYTES,
44
53
  ) {
45
- super(`transaction ${actionId} encodes to ${bytes} bytes, over the per-transaction limit MAX_TRANSACTION_BYTES (${limit} bytes); split it into smaller transactions`);
54
+ const subject = actionId === undefined ? 'the transaction staged so far' : `transaction ${actionId}`;
55
+ super(`${subject} encodes to at least ${bytes} bytes, over the per-transaction limit MAX_TRANSACTION_BYTES (${limit} bytes); split it into smaller transactions`);
46
56
  this.name = 'TransactionTooLargeError';
47
57
  }
48
58
  }
@@ -51,9 +61,11 @@ export class TransactionTooLargeError extends Error {
51
61
  * What `requests` — one transaction's pends that ride in one consensus record — encode to together,
52
62
  * the measure {@link MAX_TRANSACTION_BYTES} is judged by.
53
63
  *
54
- * NOTE: encodes every request once more than sending it does, on the writer and again on the
55
- * coordinator. Negligible for an ordinary write; if pend encoding ever shows in a profile of large
56
- * writes, measure once on the writer and hand the size along instead.
64
+ * NOTE: encodes every request once more than sending it does, once per machine: on the writer, and on
65
+ * a coordinator that received the pend from another machine, which cannot take a remote writer's
66
+ * measure on trust. A coordinator does not measure again a pend from its own node's transactor
67
+ * (`MessageOptions.localWriter`), which measured it before sending. If the writer's measure ever shows
68
+ * in a profile of large writes, take it from the encoding the pend is sent in rather than a separate one.
57
69
  */
58
70
  export function transactionBytes(requests: readonly PendRequest[]): number {
59
71
  let bytes = 0;
@@ -64,10 +76,26 @@ export function transactionBytes(requests: readonly PendRequest[]): number {
64
76
  /**
65
77
  * Throws {@link TransactionTooLargeError} when `requests` together encode to more than
66
78
  * {@link MAX_TRANSACTION_BYTES} ({@link transactionBytes}). The one rule both the writer (before
67
- * sending) and the coordinator (on receipt) apply.
79
+ * sending) and the coordinator (on receipt of a pend from another machine) apply.
68
80
  */
69
81
  export function assertTransactionFits(requests: readonly PendRequest[]): void {
70
82
  if (requests.length === 0) return;
71
83
  const bytes = transactionBytes(requests);
72
84
  if (bytes > MAX_TRANSACTION_BYTES) throw new TransactionTooLargeError(requests[0]!.actionId, bytes);
73
85
  }
86
+
87
+ /**
88
+ * Throws {@link TransactionTooLargeError}, with no action id, when the actions one collection has staged
89
+ * for its next commit — its open segment — encode to `actionBytes` together and that is over
90
+ * {@link MAX_TRANSACTION_BYTES}: the check `Collection.act` makes before staging more.
91
+ *
92
+ * Sound because the log entry a commit appends repeats every action it records verbatim, so the
93
+ * actions' sizes summed are a lower bound of the pend that carries the entry, and a pend over the limit
94
+ * can never be sent. Early rather than precise: the pend also carries the blocks the actions changed,
95
+ * which for a tree hold a row's data a second time, so this fires only once about twice the row data
96
+ * {@link assertTransactionFits} admits is staged, and a transaction between the two is still refused at
97
+ * commit. What it buys is not staging the rest of a transaction that can never commit.
98
+ */
99
+ export function assertStagedActionsFit(actionBytes: number): void {
100
+ if (actionBytes > MAX_TRANSACTION_BYTES) throw new TransactionTooLargeError(undefined, actionBytes);
101
+ }
@@ -11,7 +11,7 @@ import { ValidityRejectedError } from "../network/validity-rejected-error.js";
11
11
  import { refuseSoloField } from "../network/refuse-solo.js";
12
12
  import type { Lane } from "../lane/lane.js";
13
13
  import { TransactionTooLargeError } from "../network/transaction-size.js";
14
- import { BlockTooLargeError, oversizedBlock } from "../network/block-size.js";
14
+ import { BlockTooLargeError, oversizedWriteRefusal } from "../network/block-size.js";
15
15
  import type { Collection, CollectionSnapshot, RefreshReport } from "../collection/collection.js";
16
16
  import type { InFlightCycle, SyncOptions } from "../collection/index.js";
17
17
  import { DefaultMaxStalledAttempts, RevisionStallWatch } from "../collection/write-attempts.js";
@@ -2091,7 +2091,7 @@ export class TransactionCoordinator {
2091
2091
  ): Promise<CoordinationResult> {
2092
2092
  // Before anything is sent: one materialization of every block the transaction touches, measured
2093
2093
  // against MAX_BLOCK_BYTES here and declared on the commit below.
2094
- const described = await this.declareTouchedBlocks(collectionTransforms, criticalBlocks, pendedRevs);
2094
+ const described = await this.declareTouchedBlocks(context.actionId, collectionTransforms, criticalBlocks, pendedRevs);
2095
2095
  if (!described.success) {
2096
2096
  log('trx:phases trxId=%s refused before sending: %s', context.actionId, described.error);
2097
2097
  return described;
@@ -2208,9 +2208,11 @@ export class TransactionCoordinator {
2208
2208
  * What each collection's staged transforms send, described from its live tracker after the log
2209
2209
  * append: one local materialization of every block the transforms touch
2210
2210
  * ({@link describeTouchedBlocks}), taken before anything is sent. A block the write would leave
2211
- * over `MAX_BLOCK_BYTES` refuses the transaction here, as a pend-phase failure carrying the
2212
- * {@link BlockTooLargeError} — the same shape as a pend too large to send, so `commit` throws it by
2213
- * name and `execute` reports it as a failure. Otherwise each collection's digests are kept as its
2211
+ * over `MAX_BLOCK_BYTES` refuses the transaction here, as a pend-phase failure carrying the refusal
2212
+ * {@link oversizedWriteRefusal} judges — a `TransactionTooLargeError` when the collection's
2213
+ * transforms are over the per-transaction limit too, a {@link BlockTooLargeError} otherwise — the
2214
+ * same shape as a pend too large to send, so `commit` throws it by name and `execute` reports it as
2215
+ * a failure. Otherwise each collection's digests are kept as its
2214
2216
  * commit's declaration ({@link commitRequestFor}), so the commit does not materialize again, and
2215
2217
  * the base each update-only block's operations were computed against is read from the same
2216
2218
  * tracker's pins for its pend (`PendRequest.baseRevs`; storage keeps it with the pending record so
@@ -2221,6 +2223,7 @@ export class TransactionCoordinator {
2221
2223
  * is a defect here: {@link applyActionsToCollection} has already refused it.
2222
2224
  */
2223
2225
  private async declareTouchedBlocks(
2226
+ actionId: ActionId,
2224
2227
  collectionTransforms: ReadonlyMap<CollectionId, Transforms>,
2225
2228
  criticalBlocks: ReadonlyMap<CollectionId, BlockId>,
2226
2229
  pendedRevs: ReadonlyMap<CollectionId, number>
@@ -2233,7 +2236,7 @@ export class TransactionCoordinator {
2233
2236
  }
2234
2237
  const blockIds = blockIdsForTransforms(transforms);
2235
2238
  const touched = await describeTouchedBlocks(collection.tracker, blockIds);
2236
- const tooLarge = oversizedBlock(touched.bytes);
2239
+ const tooLarge = oversizedWriteRefusal(touched.bytes, transforms, actionId);
2237
2240
  if (tooLarge) return pendFailure([tooLarge]);
2238
2241
  attempts.set(collectionId, {
2239
2242
  transforms,