@optimystic/db-core 1.25.0 → 1.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/src/cluster/structs.d.ts +10 -0
  2. package/dist/src/cluster/structs.d.ts.map +1 -1
  3. package/dist/src/cohort-topic/addressing.d.ts +26 -2
  4. package/dist/src/cohort-topic/addressing.d.ts.map +1 -1
  5. package/dist/src/cohort-topic/addressing.js +35 -4
  6. package/dist/src/cohort-topic/addressing.js.map +1 -1
  7. package/dist/src/cohort-topic/member-engine.d.ts +17 -10
  8. package/dist/src/cohort-topic/member-engine.d.ts.map +1 -1
  9. package/dist/src/cohort-topic/member-engine.js +20 -18
  10. package/dist/src/cohort-topic/member-engine.js.map +1 -1
  11. package/dist/src/cohort-topic/registration/renewal.d.ts +21 -14
  12. package/dist/src/cohort-topic/registration/renewal.d.ts.map +1 -1
  13. package/dist/src/cohort-topic/registration/renewal.js +38 -28
  14. package/dist/src/cohort-topic/registration/renewal.js.map +1 -1
  15. package/dist/src/collection/collection.d.ts +123 -34
  16. package/dist/src/collection/collection.d.ts.map +1 -1
  17. package/dist/src/collection/collection.js +231 -74
  18. package/dist/src/collection/collection.js.map +1 -1
  19. package/dist/src/collection/segments.d.ts +71 -14
  20. package/dist/src/collection/segments.d.ts.map +1 -1
  21. package/dist/src/collection/segments.js +94 -14
  22. package/dist/src/collection/segments.js.map +1 -1
  23. package/dist/src/network/i-key-network.d.ts +28 -0
  24. package/dist/src/network/i-key-network.d.ts.map +1 -1
  25. package/dist/src/network/i-repo.d.ts +16 -0
  26. package/dist/src/network/i-repo.d.ts.map +1 -1
  27. package/dist/src/offline/queue.d.ts +13 -0
  28. package/dist/src/offline/queue.d.ts.map +1 -1
  29. package/dist/src/offline/queue.js +22 -3
  30. package/dist/src/offline/queue.js.map +1 -1
  31. package/dist/src/reactivity/topic-anchor.d.ts.map +1 -1
  32. package/dist/src/reactivity/topic-anchor.js +20 -2
  33. package/dist/src/reactivity/topic-anchor.js.map +1 -1
  34. package/dist/src/transaction/coordinator.d.ts +1 -1
  35. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  36. package/dist/src/transaction/coordinator.js +15 -16
  37. package/dist/src/transaction/coordinator.js.map +1 -1
  38. package/dist/src/transactor/network-transactor.d.ts +16 -3
  39. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  40. package/dist/src/transactor/network-transactor.js +22 -5
  41. package/dist/src/transactor/network-transactor.js.map +1 -1
  42. package/dist/src/transform/atomic.d.ts +8 -2
  43. package/dist/src/transform/atomic.d.ts.map +1 -1
  44. package/dist/src/transform/atomic.js +10 -4
  45. package/dist/src/transform/atomic.js.map +1 -1
  46. package/dist/src/transform/helpers.d.ts +29 -3
  47. package/dist/src/transform/helpers.d.ts.map +1 -1
  48. package/dist/src/transform/helpers.js +62 -3
  49. package/dist/src/transform/helpers.js.map +1 -1
  50. package/dist/src/transform/tracker.d.ts +57 -5
  51. package/dist/src/transform/tracker.d.ts.map +1 -1
  52. package/dist/src/transform/tracker.js +111 -11
  53. package/dist/src/transform/tracker.js.map +1 -1
  54. package/dist/src/utility/expiring-lru-map.d.ts +16 -0
  55. package/dist/src/utility/expiring-lru-map.d.ts.map +1 -0
  56. package/dist/src/utility/expiring-lru-map.js +39 -0
  57. package/dist/src/utility/expiring-lru-map.js.map +1 -0
  58. package/package.json +1 -1
  59. package/src/cluster/structs.ts +10 -0
  60. package/src/cohort-topic/addressing.ts +42 -4
  61. package/src/cohort-topic/member-engine.ts +35 -28
  62. package/src/cohort-topic/registration/renewal.ts +62 -42
  63. package/src/collection/collection.ts +3462 -3272
  64. package/src/collection/segments.ts +140 -22
  65. package/src/network/i-key-network.ts +30 -0
  66. package/src/network/i-repo.ts +16 -0
  67. package/src/offline/queue.ts +23 -3
  68. package/src/reactivity/topic-anchor.ts +23 -3
  69. package/src/transaction/coordinator.ts +15 -16
  70. package/src/transactor/network-transactor.ts +24 -6
  71. package/src/transform/atomic.ts +12 -5
  72. package/src/transform/helpers.ts +61 -3
  73. package/src/transform/tracker.ts +110 -12
  74. package/src/utility/expiring-lru-map.ts +42 -0
@@ -3,12 +3,12 @@ import { hlcOf } from "../log/struct.js";
3
3
  import { Atomic } from "../transform/atomic.js";
4
4
  import { Tracker } from "../transform/tracker.js";
5
5
  import { CacheSource } from "../transform/cache-source.js";
6
- import { copyTransforms, emptyTransforms, isTransformsEmpty } from "../transform/helpers.js";
6
+ import { blockIdsForTransforms, copyTransforms, emptyTransforms, isTransformsEmpty, shallowCopyTransforms } from "../transform/helpers.js";
7
7
  import { TransactorSource, answeredBlock, servedRevision } from "../transactor/transactor-source.js";
8
8
  import { BlockFloors } from "../transactor/block-floors.js";
9
9
  import { withBlockRevs } from "../network/durability.js";
10
10
  import { ActionRefusedError, CollectionHeaderVanishedError, SealedSegmentsError, TornActionError } from "./struct.js";
11
- import { SegmentList } from "./segments.js";
11
+ import { SegmentList, unitsOf } from "./segments.js";
12
12
  import { actionIdAt, addTallies, ZeroTally } from "./action.js";
13
13
  import { AttemptCycle, resumedAttempt, retainedAttempt, stageAttempt } from "./write-attempts.js";
14
14
  import { driveHeadCommit } from "./head-commit.js";
@@ -38,6 +38,14 @@ export class Collection {
38
38
  * exactly the one pending list it replaced, and every path behaves as it did then. */
39
39
  segments = new SegmentList();
40
40
  latchId;
41
+ /** Bumped at the start of every replay ({@link replayActions}): a replay stages every segment
42
+ * again, so the blocks the staged actions insert take fresh ids, and the deltas recorded on the
43
+ * segments name those. A head attempt built from the head's delta carries the epoch it was built
44
+ * in, and the committed head is taken out of the live tracker only while the epoch is unchanged —
45
+ * otherwise the attempt's ids are not the ones the live tracker holds, and the head is replayed
46
+ * out as before ({@link subtractCommittedHead}). A snapshot captures it for the same reason
47
+ * ({@link CollectionSnapshot.replayEpoch}). */
48
+ replayEpoch = 0;
41
49
  /** The transforms an INVENTED collection starts from — its header, its empty log block and (for
42
50
  * a tree) its empty root, as {@link createOrOpen}'s create branch staged them. A replay resets
43
51
  * the tracker to a copy of this, rather than to empty, for as long as no committed revision has
@@ -611,9 +619,11 @@ export class Collection {
611
619
  }
612
620
  }
613
621
  async actInternal(...actions) {
614
- await this.internalTransact(...actions);
615
- // Staging appends to the OPEN segment: a sealed segment is closed to further staging.
616
- this.segments.open.push(...actions);
622
+ const { transforms, applied } = await this.internalTransact(...actions);
623
+ // Staging appends to the OPEN segment: a sealed segment is closed to further staging. What
624
+ // the staging produced is recorded with the actions, so the segment they are sealed into
625
+ // knows what it staged without staging it again (SealedSegment.delta).
626
+ this.segments.pushOpen(actions, transforms, applied);
617
627
  }
618
628
  internalTransact(...actions) {
619
629
  return this.stageInto(this.tracker, ...actions);
@@ -624,8 +634,10 @@ export class Collection {
624
634
  * commit changes what this collection's reads see ({@link viewChanged}); a head commit's tracker
625
635
  * is private to the commit.
626
636
  *
627
- * @returns the sum of what the handlers accepted as already applied ({@link AlreadyAppliedTally});
628
- * a handler that answers nothing adds the zero tally. */
637
+ * @returns what the staging produced ({@link Staging}): the transforms the handlers staged — the
638
+ * atomic's own, which `tracker` copied what it kept of, so the caller may keep them as the record
639
+ * of this staging — and the sum of what the handlers accepted as already applied
640
+ * ({@link AlreadyAppliedTally}); a handler that answers nothing adds the zero tally. */
629
641
  async stageInto(tracker, ...actions) {
630
642
  const atomic = new Atomic(tracker);
631
643
  let applied = ZeroTally;
@@ -642,8 +654,7 @@ export class Collection {
642
654
  if (tracker === this.tracker) {
643
655
  this.viewChanged();
644
656
  }
645
- atomic.commit();
646
- return applied;
657
+ return { transforms: atomic.commit(), applied };
647
658
  }
648
659
  /** Tell {@link CollectionInitOptions.onViewChanged} that what this collection's reads see is
649
660
  * about to change. Every caller calls it in the same synchronous step as the change itself — no
@@ -935,7 +946,7 @@ export class Collection {
935
946
  this.consumeHead(entry, this.headCommit.entryId);
936
947
  }
937
948
  else {
938
- this.segments.open = this.consumedFromOpen(entry);
949
+ this.segments.setOpen(this.consumedFromOpen(entry));
939
950
  }
940
951
  return true;
941
952
  }
@@ -984,7 +995,7 @@ export class Collection {
984
995
  }
985
996
  /** The open segment with the leading actions `entry` made durable dropped off it. */
986
997
  consumedFromOpen(entry) {
987
- const open = this.segments.open;
998
+ const open = this.segments.openUnits();
988
999
  if (entry.actions.length > open.length) {
989
1000
  throw new Error(`Collection ${this.id}: own committed entry for action ${entry.actionId} holds `
990
1001
  + `${entry.actions.length} actions but only ${open.length} are pending; `
@@ -1011,12 +1022,23 @@ export class Collection {
1011
1022
  *
1012
1023
  * @returns whether the open segment changed. */
1013
1024
  filterAgainstEntry(entry) {
1014
- const before = this.segments.open;
1015
- const after = before
1016
- .map(p => this.doFilterConflict(p, entry.actions))
1017
- .filter((a) => a !== undefined);
1018
- this.segments.open = after;
1019
- return after.length !== before.length || after.some((a, i) => a !== before[i]);
1025
+ const before = this.segments.openUnits();
1026
+ let changed = false;
1027
+ const after = [];
1028
+ for (const unit of before) {
1029
+ const effective = this.doFilterConflict(unit.action, entry.actions);
1030
+ if (effective === unit.action) {
1031
+ after.push(unit);
1032
+ }
1033
+ else {
1034
+ // A replacement's staging is not on record; the replay this forces records it.
1035
+ changed = true;
1036
+ if (effective !== undefined)
1037
+ after.push({ action: effective });
1038
+ }
1039
+ }
1040
+ this.segments.setOpen(after);
1041
+ return changed;
1020
1042
  }
1021
1043
  /** Refresh this instance against the stored log: adopt the latest committed revision, resolve
1022
1044
  * pending actions against everything that landed since, and replay them if anything conflicts.
@@ -1078,10 +1100,12 @@ export class Collection {
1078
1100
  // tail, or a tail with no `latest` — both legitimate "nothing committed yet" states.
1079
1101
  const tailRev = source.actionContext?.rev;
1080
1102
  const actionContext = this.source.actionContext;
1081
- // A write's retry refresh always walks. Not needed for soundness — the write's own entry would
1082
- // sit above the held revision, which the tail test already refuses — but losing that entry
1083
- // loses the write, and a retry refresh is rare enough that the walk costs nothing that matters.
1084
- if (this.inFlightActionId === undefined && Collection.tailShowsNothingNewer(actionContext, ends.tail)) {
1103
+ // Stops here under the in-flight mark too, a retried write's refresh included: an entry of this
1104
+ // write that the tail does not show is one the walk could not find either, because the walk below
1105
+ // starts from this same tail block (its cache is seeded with it). What reaches past a lagging
1106
+ // answer is the tail floor a retry's read carries, which has already been applied to this tail.
1107
+ // This is the opening refresh of every head commit, where a walk would read the whole log.
1108
+ if (Collection.tailShowsNothingNewer(actionContext, ends.tail)) {
1085
1109
  return;
1086
1110
  }
1087
1111
  // One block cache for the whole refresh, seeded with the header and tail just read: Log.open,
@@ -1520,15 +1544,20 @@ export class Collection {
1520
1544
  * until the first sync. Resetting such a collection to empty would leave it
1521
1545
  * unreadable; restoring the snapshot returns it to its prior (readable) state.
1522
1546
  *
1523
- * The returned snapshot is deep-cloned and independent of subsequent mutations.
1547
+ * The returned snapshot stays as captured under subsequent staging, without copying what is
1548
+ * staged: it shares the staged values with the tracker, which copies one before changing it
1549
+ * ({@link Tracker.shareTransforms}). A successful commit restores none of its snapshots, and the
1550
+ * Quereus bridge takes one per statement per written collection, so a deep copy here cost every
1551
+ * statement in proportion to the whole offline queue.
1524
1552
  * Synchronous and latch-free: intended to bracket transaction-scoped staging,
1525
1553
  * when no concurrent act/sync is in flight. */
1526
1554
  snapshotPending() {
1527
1555
  return {
1528
- transforms: copyTransforms(this.tracker.transforms),
1556
+ transforms: this.tracker.shareTransforms(),
1529
1557
  pins: this.tracker.pins.copy(),
1530
- pending: [...this.segments.open],
1558
+ pending: [...this.segments.openUnits()],
1531
1559
  sealedVersion: this.segments.version,
1560
+ replayEpoch: this.replayEpoch,
1532
1561
  context: structuredClone(this.source.actionContext),
1533
1562
  };
1534
1563
  }
@@ -1545,19 +1574,23 @@ export class Collection {
1545
1574
  * revision), and whether any segment is sealed now:
1546
1575
  *
1547
1576
  * - **Verbatim**, when the sealed list is unchanged and the boundary has not moved: the tracker
1548
- * is reset to a copy of the captured transforms, the captured pins are restored, and the open
1577
+ * is reset to new maps over the captured transforms' values, so one snapshot can be restored
1578
+ * again (a repeated `rollback to`), the captured pins are restored, and the open
1549
1579
  * segment is set to the captured actions. This keeps an INVENTED collection's header, log
1550
1580
  * block and root, which live in the tracker until the first sync and would be lost by a
1551
1581
  * reset. It is the arm every restore took before segments existed, so while nothing is ever
1552
1582
  * sealed nothing behaves differently.
1553
1583
  * - **Replay**, when the sealed list has changed — a segment sealed, dropped or refused while
1554
- * the snapshot was outstanding — or when the boundary has moved while a segment is sealed:
1555
- * the open segment is set to the captured actions and the tracker is rebuilt by
1556
- * {@link replayActions}, the sealed segments as they are NOW (refused ones skipped, a newly
1557
- * refused one recorded and reported) and then the restored open actions. Reinstating the
1558
- * captured transforms would put a dropped segment's effects back — the drain removes the
1559
- * head segment after committing it, and the next head commit would commit it a second time
1560
- * — and a reset would drop the sealed segments' effects.
1584
+ * the snapshot was outstanding — or when the boundary has moved, or a replay has run
1585
+ * ({@link CollectionSnapshot.replayEpoch}), while a segment is sealed: the open segment is
1586
+ * set to the captured actions and the tracker is rebuilt by {@link replayActions}, the sealed
1587
+ * segments as they are NOW (refused ones skipped, a newly refused one recorded and reported)
1588
+ * and then the restored open actions. Reinstating the captured transforms would put a dropped
1589
+ * segment's effects back — the drain removes the head segment after committing it, and the
1590
+ * next head commit would commit it a second time — and a reset would drop the sealed
1591
+ * segments' effects. After a replay, reinstating them would put back block ids the sealed
1592
+ * segments' recorded deltas no longer name, and the next head commit would take the wrong
1593
+ * blocks out of the tracker.
1561
1594
  * - A moved boundary with NOTHING sealed keeps the two arms it had before segments. An empty
1562
1595
  * captured open segment (the transaction-rollback shape: the capture predates the
1563
1596
  * transaction's first stage) resets the tracker empty, because the captured transforms
@@ -1594,27 +1627,30 @@ export class Collection {
1594
1627
  * pre-pend re-validation (restageIfBasesMoved) finds those bases moved and re-stages. */
1595
1628
  async restorePending(snapshot) {
1596
1629
  const sealedChanged = snapshot.sealedVersion !== undefined && snapshot.sealedVersion !== this.segments.version;
1630
+ const replayed = snapshot.replayEpoch !== undefined && snapshot.replayEpoch !== this.replayEpoch;
1597
1631
  const capturedRev = snapshot.context?.rev;
1598
1632
  const currentRev = this.source.actionContext?.rev;
1599
1633
  const boundaryMoved = currentRev !== undefined && (capturedRev === undefined || currentRev > capturedRev);
1600
- if (sealedChanged || (boundaryMoved && this.segments.hasSealed())) {
1601
- this.segments.open = [...snapshot.pending];
1634
+ if (sealedChanged || ((boundaryMoved || replayed) && this.segments.hasSealed())) {
1635
+ this.segments.setOpen([...snapshot.pending]);
1602
1636
  await this.replayActions();
1603
1637
  return;
1604
1638
  }
1605
1639
  this.viewChanged();
1606
1640
  if (boundaryMoved && snapshot.pending.length === 0) {
1607
1641
  this.tracker.reset();
1608
- this.segments.open = [];
1642
+ this.segments.setOpen([]);
1609
1643
  return;
1610
1644
  }
1611
- this.tracker.reset(copyTransforms(snapshot.transforms));
1645
+ this.tracker.reset(shallowCopyTransforms(snapshot.transforms));
1612
1646
  if (snapshot.pins)
1613
1647
  this.tracker.pins.replaceWith(snapshot.pins);
1614
- this.segments.open = [...snapshot.pending];
1648
+ this.segments.setOpen([...snapshot.pending]);
1615
1649
  }
1616
1650
  /** A read-only {@link Tracker} pinned to this collection's committed state AS OF the
1617
- * moment of this call, seeded with a (deep-copied) set of pre-transaction transforms.
1651
+ * moment of this call, seeded with a snapshot's transforms ({@link CollectionSnapshot.transforms}).
1652
+ * It shares their values, so pass a snapshot's and not a live tracker's, which would go on changing
1653
+ * the values it owns; a view does not write, and if one did it would copy first.
1618
1654
  * Reads through it observe exactly that revision plus exactly those transforms — NOT
1619
1655
  * the mutations staged into this collection's live tracker afterward, and NOT commits
1620
1656
  * that fold into (or clear) the live read cache while the view is being walked. Used
@@ -1670,7 +1706,7 @@ export class Collection {
1670
1706
  // too old by the view either. A view pinned below a floor is untouched by it.
1671
1707
  const pinnedSource = new TransactorSource(this.id, this.transactor, structuredClone(pinContext), collector, this.floors);
1672
1708
  const pinnedCache = new CacheSource(pinnedSource, undefined, collector, seed);
1673
- return new Tracker(pinnedCache, copyTransforms(transforms));
1709
+ return new Tracker(pinnedCache, shallowCopyTransforms(transforms));
1674
1710
  }
1675
1711
  /** The OPEN segment: the staged (not-yet-synced) actions queued by {@link act} that no
1676
1712
  * {@link sealPending} has closed off.
@@ -1686,14 +1722,14 @@ export class Collection {
1686
1722
  * existing catch, which restores every participant. */
1687
1723
  getPendingActions() {
1688
1724
  this.assertNoSealedSegments();
1689
- return this.segments.open;
1725
+ return this.segments.openActions();
1690
1726
  }
1691
1727
  /** Drop the OPEN segment after it has been committed through a coordinator. Counterpart to
1692
1728
  * {@link getPendingActions}, which cannot have returned anything while a segment was sealed,
1693
1729
  * so only the open segment is ever cleared here; {@link sync} clears its own inline, so this
1694
1730
  * is only needed when commit was orchestrated externally. */
1695
1731
  clearPendingActions() {
1696
- this.segments.open = [];
1732
+ this.segments.setOpen([]);
1697
1733
  }
1698
1734
  /** Whether {@link sync} has anything to push: staged actions in the open segment, or tracker
1699
1735
  * transforms that were never committed — an INVENTED collection's header/root blocks, which
@@ -1716,7 +1752,7 @@ export class Collection {
1716
1752
  * holds), so over an otherwise empty tracker this reads `false` while {@link sealedEntries} is
1717
1753
  * non-empty. The queue drives off {@link sealedEntries}, never off this predicate. */
1718
1754
  hasUnsyncedChanges() {
1719
- return this.segments.open.length > 0 || !isTransformsEmpty(this.tracker.transforms);
1755
+ return this.segments.openCount() > 0 || !isTransformsEmpty(this.tracker.transforms);
1720
1756
  }
1721
1757
  /** Close the open segment into a sealed segment owned by `entryId` (see {@link SegmentList}).
1722
1758
  * Returns `false` and changes nothing when the open segment is empty: an invented collection's
@@ -1732,6 +1768,16 @@ export class Collection {
1732
1768
  sealedEntries() {
1733
1769
  return this.segments.entryIds();
1734
1770
  }
1771
+ /** Whether `entryId` owns a sealed segment here, refused or not: asked of one entry at a time, at
1772
+ * any queue depth, so it builds no list. Synchronous and latch-free, like {@link sealedEntries}. */
1773
+ holdsSealed(entryId) {
1774
+ return this.segments.find(entryId) !== undefined;
1775
+ }
1776
+ /** Whether any segment is sealed here, refused ones included. Synchronous and latch-free, like
1777
+ * {@link sealedEntries}. */
1778
+ hasSealed() {
1779
+ return this.segments.hasSealed();
1780
+ }
1735
1781
  /** The log tail block id the most recent header read named ({@link logTailId}), for a caller that
1736
1782
  * resolves this collection's cohorts from routing keys without a read — the offline queue's
1737
1783
  * reachability gate (docs/offline-sync.md §When to submit). `undefined` before the first header
@@ -1752,12 +1798,11 @@ export class Collection {
1752
1798
  return this.segments.find(entryId)?.actions;
1753
1799
  }
1754
1800
  /** The OPEN segment's actions — what the current transaction has staged since the last seal — as
1755
- * the array the segment holds (never mutated by a caller). Unlike {@link getPendingActions} it
1756
- * answers while segments are sealed, for a reader that judges the open transaction beside the
1757
- * queued ones (the Quereus plugin's `CommitOutlook()`). Synchronous and latch-free, like
1758
- * {@link sealedActions}. */
1801
+ * a new array. Unlike {@link getPendingActions} it answers while segments are sealed, for a
1802
+ * reader that judges the open transaction beside the queued ones (the Quereus plugin's
1803
+ * `CommitOutlook()`). Synchronous and latch-free, like {@link sealedActions}. */
1759
1804
  openActions() {
1760
- return this.segments.open;
1805
+ return this.segments.openActions();
1761
1806
  }
1762
1807
  /** The exact inverse of {@link sealPending}, list-only and latch-free by the same contract: move
1763
1808
  * the segment's actions back into the open segment, for an enqueue whose journal write failed
@@ -1817,9 +1862,9 @@ export class Collection {
1817
1862
  async restoreSealed(entryId, actions, position) {
1818
1863
  const release = await Latches.acquire(this.latchId);
1819
1864
  try {
1820
- if (this.segments.open.length > 0) {
1865
+ if (this.segments.openCount() > 0) {
1821
1866
  throw new Error(`Collection ${this.id}: cannot restore sealed segment ${entryId} while a transaction has `
1822
- + `${this.segments.open.length} action(s) staged in the open segment`);
1867
+ + `${this.segments.openCount()} action(s) staged in the open segment`);
1823
1868
  }
1824
1869
  if (this.headInFlight !== undefined) {
1825
1870
  throw new SealedSegmentsError(this.id, this.segments.entryIds(), 'in-flight', this.headInFlight.entryId);
@@ -2153,7 +2198,7 @@ export class Collection {
2153
2198
  // inter-attempt refresh would leave the entry claiming the lane of the action it replaced. No
2154
2199
  // hook in this repository replaces an action; if one ever does, the member-side lane check is
2155
2200
  // what catches the mismatch.
2156
- lane: options?.lane ?? this.laneOf(this.segments.open),
2201
+ lane: options?.lane ?? this.laneOf(this.segments.openActions()),
2157
2202
  };
2158
2203
  const endInFlight = this.beginInFlightAction(identity.actionId, { lane: identity.lane, refuseSolo: options?.refuseSoloCommit });
2159
2204
  try {
@@ -2187,7 +2232,7 @@ export class Collection {
2187
2232
  await this.restageIfBasesMoved();
2188
2233
  // Snapshot the open segment so that any new actions aren't assumed to be part of this action
2189
2234
  // (no segment is sealed here — syncInternal refused if one was).
2190
- const pending = [...this.segments.open];
2235
+ const pending = this.segments.openActions();
2191
2236
  const staged = await this.buildAttempt({ actions: pending, transforms: this.tracker.transforms, pins: this.tracker.pins }, identity);
2192
2237
  const attempt = await this.send(staged, identity, cycle.priority());
2193
2238
  if (!attempt.success) {
@@ -2198,7 +2243,7 @@ export class Collection {
2198
2243
  else {
2199
2244
  cycle.recordCommit(attempt.durability);
2200
2245
  // Clear the open-segment actions that were part of this action
2201
- this.segments.open = this.segments.open.slice(pending.length);
2246
+ this.segments.setOpen(this.segments.openUnits().slice(pending.length));
2202
2247
  // Fold the committed transforms into the read cache and adopt the revision BEFORE
2203
2248
  // replaying whatever is still staged — the order `updateInternal` and
2204
2249
  // `TransactionCoordinator.commitOnceLatched` require, where the newly committed state
@@ -2366,6 +2411,10 @@ export class Collection {
2366
2411
  let held = true;
2367
2412
  let closed = false;
2368
2413
  let head;
2414
+ // How the attempt `stage()` last built relates to the live tracker ({@link HeadSubtraction}),
2415
+ // read by `replay()`: set when it was built from the head's recorded delta, cleared when it
2416
+ // was a recorded attempt re-sent verbatim.
2417
+ let subtraction;
2369
2418
  const assertUsable = (step, needsLatch) => {
2370
2419
  if (closed) {
2371
2420
  throw new Error(`Collection ${this.id}: head commit of ${entryId} is closed; ${step}() is a defect`);
@@ -2390,9 +2439,12 @@ export class Collection {
2390
2439
  const segment = this.committableHead(entryId);
2391
2440
  const resumed = resumedAttempt(this.sourceCache, resume, this.getNextRev());
2392
2441
  if (resumed !== undefined) {
2442
+ // A recorded attempt's ids are an earlier cycle's, not the live tracker's.
2443
+ subtraction = undefined;
2393
2444
  return resumed;
2394
2445
  }
2395
2446
  head = await this.headTrackerFor(segment, head);
2447
+ subtraction = head.subtract;
2396
2448
  return this.buildAttempt({ actions: segment.actions, transforms: head.tracker.transforms, pins: head.tracker.pins }, identity);
2397
2449
  },
2398
2450
  send: (staged, priority) => {
@@ -2425,14 +2477,18 @@ export class Collection {
2425
2477
  this.foldOwnCommit(staged, identity.actionId);
2426
2478
  this.reportCommittedHead(dropped);
2427
2479
  },
2428
- replay: () => {
2480
+ replay: async () => {
2429
2481
  assertUsable('replay', true);
2430
- // NOTE: the remaining sealed segments and the open segment are replayed after EVERY head
2431
- // commit, so draining a queue of n entries is quadratic in n (docs/offline-sync.md
2432
- // §Failure modes and accepted tradeoffs). Negligible for tens of entries; if it shows up
2433
- // for a device that queued thousands, keep per-segment trackers chained over the cache,
2434
- // so a committed head is popped and folded instead of replayed.
2435
- return this.replayActions();
2482
+ if (subtraction !== undefined && subtraction.epoch === this.replayEpoch
2483
+ && await this.subtractCommittedHead(entryId, subtraction.touched)) {
2484
+ return;
2485
+ }
2486
+ // The attempt's ids are not the live tracker's — it was built by running the handlers
2487
+ // (an invented collection's first head, a delta not on record, a moved base), re-sent
2488
+ // from a recorded attempt, or a replay has run since it was built — or a later
2489
+ // segment's staging is not on record: the remaining segments are staged again.
2490
+ log('collection:committed-head-replayed id=%s tag=%s entry=%s reason=%s', this.id, this.instanceTag, entryId, subtraction === undefined ? 'attempt-not-from-delta' : subtraction.epoch !== this.replayEpoch ? 'replayed-since' : 'deltas-missing');
2491
+ await this.replayActions();
2436
2492
  },
2437
2493
  refused: (staged) => {
2438
2494
  assertUsable('refused', true);
@@ -2491,13 +2547,14 @@ export class Collection {
2491
2547
  }
2492
2548
  /** The tracker a head attempt is built from (see {@link commitHead}): `current`, while it was
2493
2549
  * built for the revision the next attempt requests and no base under it has moved; otherwise a
2494
- * fresh one ({@link buildHeadTracker}). A revision that moved is a write that is genuinely
2495
- * different, and so is one whose held lineage moved at the same revision: a refresh between
2496
- * attempts that adopted another lineage ({@link judgeAdoption}) dropped the read cache the
2497
- * tracker was staged over, and an action that inserted a block computed from that content pinned
2498
- * no base the re-judge below would see as moved. A base that moved at the same revision is
2499
- * re-judged exactly as {@link restageIfBasesMoved} re-judges the live queue before an attempt,
2500
- * and rebuilds.
2550
+ * fresh one — from the head's recorded delta where that is possible
2551
+ * ({@link headTrackerFromDelta}), else by running its handlers ({@link buildHeadTracker}). A
2552
+ * revision that moved is a write that is genuinely different, and so is one whose held lineage
2553
+ * moved at the same revision: a refresh between attempts that adopted another lineage
2554
+ * ({@link judgeAdoption}) dropped the read cache the tracker was staged over, and an action that
2555
+ * inserted a block computed from that content pinned no base the re-judge below would see as
2556
+ * moved. A base that moved at the same revision is re-judged exactly as
2557
+ * {@link restageIfBasesMoved} re-judges the live queue before an attempt, and rebuilds.
2501
2558
  * NOTE: that rebuild is the same exception to the byte-identical retry that {@link syncAttempts}
2502
2559
  * records at its `restageIfBasesMoved` call — fresh ids for the blocks the actions insert under
2503
2560
  * an unchanged `(action id, revision)` — reached the same narrow way and refused by storage's
@@ -2512,18 +2569,99 @@ export class Collection {
2512
2569
  return current;
2513
2570
  }
2514
2571
  }
2572
+ const fromDelta = held === undefined ? undefined : await this.headTrackerFromDelta(segment);
2573
+ if (fromDelta !== undefined) {
2574
+ return { rev, lineage, ...fromDelta };
2575
+ }
2515
2576
  return { rev, lineage, tracker: await this.buildHeadTracker(segment) };
2516
2577
  }
2578
+ /** A head tracker built from what `segment`'s actions staged into the live tracker
2579
+ * ({@link SealedSegment.delta}) rather than by running them again: a tracker over the read cache
2580
+ * holding a copy of the delta, with the live tracker's pins for the blocks the delta updates — the
2581
+ * bases those operations were computed against, which the live tracker has judged exactly as this
2582
+ * tracker's will be (the operations are the same). Correct because the segment's actions were
2583
+ * staged over the committed state plus the segments ahead of it, and once those have committed
2584
+ * with nothing from any other writer in between, the committed state IS that base, so running the
2585
+ * handlers again would make every decision the same way; their only different output would be
2586
+ * fresh ids for the blocks they insert, which the live tracker does not hold, and which is why a
2587
+ * rebuilt head has to be replayed out of the tracker afterwards. A refresh that adopts another
2588
+ * writer's revision replays, which records every delta again. `undefined` — build by running the
2589
+ * handlers — when the delta is not on record, or when a pin under it has moved: the delta was
2590
+ * computed on content the cache no longer describes. The caller keeps this for a collection that
2591
+ * holds a committed revision only: an invented collection's first commit has to carry the baseline,
2592
+ * which the delta does not. */
2593
+ async headTrackerFromDelta(segment) {
2594
+ const delta = segment.delta;
2595
+ if (delta === undefined) {
2596
+ return undefined;
2597
+ }
2598
+ const tracker = new Tracker(this.sourceCache, copyTransforms(delta));
2599
+ for (const id of Object.keys(delta.updates ?? {})) {
2600
+ const pin = this.tracker.pins.get(id);
2601
+ if (pin !== undefined)
2602
+ tracker.pins.set(id, pin);
2603
+ }
2604
+ await this.rereadUnretainedBases(tracker);
2605
+ if (this.logMovedBases(tracker.movedBases()) > 0) {
2606
+ return undefined;
2607
+ }
2608
+ return { tracker, subtract: { epoch: this.replayEpoch, touched: blockIdsForTransforms(delta) } };
2609
+ }
2610
+ /** Take a committed head segment out of the live tracker without running a handler, the
2611
+ * counterpart of {@link headTrackerFromDelta}: the attempt's transforms are in the read cache and
2612
+ * its revision adopted (`fold`), so the live tracker's state for the blocks the head touched
2613
+ * (`touched`) is rebuilt from the remaining segments' and the open units' recorded deltas alone
2614
+ * (`Tracker.refold`), and nothing else is visited. Two steps. First, each touched block that a
2615
+ * remaining delta updates and the read cache no longer retains is read through the cache now:
2616
+ * the fold applies the head's operations into a cached block only, so an evicted one would be
2617
+ * pinned at the revision last served, which the remaining operations were not computed on. Then,
2618
+ * in one synchronous step, {@link viewChanged} and the re-fold — told even though no id changes,
2619
+ * because a read that fetched its base before the fold and applied the remaining operations
2620
+ * after it would otherwise see content missing the head; the signal readers already handle
2621
+ * (docs/internals.md, "The drain changes a tree beneath its readers"), once per commit instead of
2622
+ * once per unit re-staged.
2623
+ *
2624
+ * NOTE: `touched` is what the head's delta touched, not what its attempt touched: a block the
2625
+ * attempt's log append changed and the delta did not — the collection header, when the log tail
2626
+ * rolled over in this commit — is not visited, so a later segment's pin on it is judged moved
2627
+ * and the next refresh or head commit replays what is staged. At most once per roll-over, which
2628
+ * is every 32 entries; if it ever shows in a drain profile, visit the attempt's touched ids too,
2629
+ * on the ground that the append changes only the header's tail id, which no handler reads.
2630
+ *
2631
+ * @returns `false`, with nothing changed, when a remaining segment's or open unit's delta is not
2632
+ * on record: the caller replays instead. */
2633
+ async subtractCommittedHead(entryId, touched) {
2634
+ const deltas = this.segments.stagedDeltas();
2635
+ if (deltas === undefined) {
2636
+ return false;
2637
+ }
2638
+ // NOTE: each read here awaits with the head already folded into the cache and still staged in
2639
+ // the live tracker, so an unlatched read of a touched block in that window applies the head's
2640
+ // operations twice. Only when the cache evicted such a block (128 entries, LRU) under a later
2641
+ // segment's update; the coordinator's per-entry commit opens the same window for its second
2642
+ // participant onwards. Backlog `bug-a-folded-head-stays-in-the-live-tracker-across-an-await`.
2643
+ for (const id of touched) {
2644
+ if (!this.sourceCache.retains(id) && deltas.some(delta => (delta.updates?.[id]?.length ?? 0) > 0)) {
2645
+ await this.sourceCache.tryGet(id, 'navigation');
2646
+ }
2647
+ }
2648
+ this.viewChanged();
2649
+ this.tracker.refold(touched, deltas);
2650
+ log('collection:committed-head-subtracted id=%s tag=%s entry=%s touched=%d remaining=%d', this.id, this.instanceTag, entryId, touched.length, deltas.length);
2651
+ return true;
2652
+ }
2517
2653
  /** A tracker over this instance's read cache, with pins of its own, holding `segment`'s actions
2518
2654
  * staged atomically over the committed state — from the invented baseline while this collection
2519
2655
  * has no committed revision ({@link replayBaseline}): its header, log block and root then exist
2520
2656
  * nowhere else, and the head's commit has to carry them. A refusal is the segment's own,
2521
2657
  * recorded and reported ({@link recordSealedRefusal}) and thrown; a live replay that found it
2522
- * first has already done the recording, and the caller never gets here. */
2658
+ * first has already done the recording, and the caller never gets here. Records the tally this
2659
+ * staging produced on the segment, never the transforms: their ids are this tracker's, not the
2660
+ * live tracker's, which `SealedSegment.delta` names. */
2523
2661
  async buildHeadTracker(segment) {
2524
2662
  const tracker = new Tracker(this.sourceCache, this.replayBaseline());
2525
2663
  try {
2526
- segment.applied = await this.stageInto(tracker, ...segment.actions);
2664
+ segment.applied = (await this.stageInto(tracker, ...segment.actions)).applied;
2527
2665
  }
2528
2666
  catch (err) {
2529
2667
  if (err instanceof ActionRefusedError) {
@@ -2594,7 +2732,7 @@ export class Collection {
2594
2732
  // declares no merge and `Diary` declares no hook); if one ever does, carry a per-call token on
2595
2733
  // the staged actions and match on that instead.
2596
2734
  // The actions were staged into the OPEN segment (actAndSync refuses over sealed segments).
2597
- this.segments.open = this.segments.open.filter(staged => !actions.includes(staged));
2735
+ this.segments.setOpen(this.segments.openUnits().filter(unit => !actions.includes(unit.action)));
2598
2736
  if (this.source.actionContext?.rev === revBefore) {
2599
2737
  this.viewChanged();
2600
2738
  this.tracker.reset(stagedBefore);
@@ -2631,23 +2769,37 @@ export class Collection {
2631
2769
  *
2632
2770
  * Each re-staged update pins its base through `Atomic.commit` → `Tracker.absorb`, the path
2633
2771
  * {@link act} uses, so the pend declares the bases the re-staged operations were computed on. Each
2634
- * sealed segment that staged records what its actions accepted as already applied
2635
- * (`SealedSegment.applied`), the tally the head commit reports when the segment is committed.
2772
+ * sealed segment that staged records what its actions staged (`SealedSegment.delta`, under the ids
2773
+ * this replay minted — the ones the live tracker now holds, which is what lets the next head
2774
+ * commit take the head out of the tracker without a replay) and what they accepted as already
2775
+ * applied (`SealedSegment.applied`), the tally the head commit reports when the segment is
2776
+ * committed; each open unit records the same for its action. The replay epoch moves first
2777
+ * ({@link replayEpoch}), so a head attempt built before this replay is known to hold other ids.
2636
2778
  *
2637
2779
  * A replay changes the tree beneath unlatched readers twice over: the reset drops every block a
2638
2780
  * staged action inserted, and the re-staging inserts them again under fresh ids. Both are told
2639
2781
  * through {@link viewChanged} — the reset here, each re-staged unit in {@link stageInto} — so a
2640
2782
  * reader part-way through a descent knows to read again rather than follow an id that is gone.
2641
2783
  *
2784
+ * NOTE: a throw out of the open segment's staging leaves the units behind it with no staging on
2785
+ * record (every record is cleared first, since the ids it named are gone), on top of the
2786
+ * half-restaged tracker the callers' note on this method already accepts; the next replay
2787
+ * records them again, and a seal over them meanwhile carries no delta, which the head commit
2788
+ * takes as "stage it again".
2789
+ *
2642
2790
  * @param unreported - a segment whose refusal is recorded but NOT reported through
2643
2791
  * `onSealedRefused`: the one {@link restoreSealed} is inserting, whose caller reads the recorded
2644
2792
  * refusal off the segment itself. */
2645
2793
  async replayActions(unreported) {
2794
+ this.replayEpoch++;
2646
2795
  this.viewChanged();
2647
2796
  this.tracker.reset(this.replayBaseline());
2648
2797
  for (const segment of this.segments.replayable()) {
2798
+ segment.delta = undefined;
2649
2799
  try {
2650
- segment.applied = await this.internalTransact(...segment.actions);
2800
+ const { transforms, applied } = await this.internalTransact(...segment.actions);
2801
+ segment.delta = transforms;
2802
+ segment.applied = applied;
2651
2803
  }
2652
2804
  catch (err) {
2653
2805
  if (!(err instanceof ActionRefusedError)) {
@@ -2656,8 +2808,13 @@ export class Collection {
2656
2808
  this.recordSealedRefusal(segment, err, segment.entryId !== unreported);
2657
2809
  }
2658
2810
  }
2659
- for (const action of this.segments.open) {
2660
- await this.internalTransact(action);
2811
+ // The list holds this array from here on, so each unit is recorded in place as its action
2812
+ // is re-staged, and a throw leaves the units behind it with no staging on record.
2813
+ const units = unitsOf(this.segments.openActions());
2814
+ this.segments.setOpen(units);
2815
+ for (const [index, unit] of units.entries()) {
2816
+ const { transforms, applied } = await this.internalTransact(unit.action);
2817
+ units[index] = { action: unit.action, delta: transforms, applied };
2661
2818
  }
2662
2819
  }
2663
2820
  /** What a replay resets the tracker to: a copy of {@link inventedBaseline} while this collection