@fluidframework/merge-tree 2.20.0 → 2.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/.eslintrc.cjs +0 -1
  2. package/CHANGELOG.md +8 -0
  3. package/README.md +1 -0
  4. package/dist/attributionCollection.js +5 -7
  5. package/dist/attributionCollection.js.map +1 -1
  6. package/dist/localReference.js +6 -8
  7. package/dist/localReference.js.map +1 -1
  8. package/dist/mergeTree.d.ts.map +1 -1
  9. package/dist/mergeTree.js +69 -34
  10. package/dist/mergeTree.js.map +1 -1
  11. package/dist/mergeTreeNodes.d.ts +15 -4
  12. package/dist/mergeTreeNodes.d.ts.map +1 -1
  13. package/dist/mergeTreeNodes.js +1 -1
  14. package/dist/mergeTreeNodes.js.map +1 -1
  15. package/dist/partialLengths.d.ts +114 -144
  16. package/dist/partialLengths.d.ts.map +1 -1
  17. package/dist/partialLengths.js +431 -525
  18. package/dist/partialLengths.js.map +1 -1
  19. package/dist/perspective.d.ts +10 -1
  20. package/dist/perspective.d.ts.map +1 -1
  21. package/dist/perspective.js +10 -1
  22. package/dist/perspective.js.map +1 -1
  23. package/dist/properties.d.ts.map +1 -1
  24. package/dist/properties.js +2 -3
  25. package/dist/properties.js.map +1 -1
  26. package/dist/revertibles.js +3 -3
  27. package/dist/revertibles.js.map +1 -1
  28. package/dist/segmentInfos.d.ts +3 -0
  29. package/dist/segmentInfos.d.ts.map +1 -1
  30. package/dist/segmentInfos.js.map +1 -1
  31. package/dist/segmentPropertiesManager.js +3 -3
  32. package/dist/segmentPropertiesManager.js.map +1 -1
  33. package/dist/snapshotLoader.js +2 -2
  34. package/dist/snapshotLoader.js.map +1 -1
  35. package/dist/sortedSegmentSet.d.ts +5 -3
  36. package/dist/sortedSegmentSet.d.ts.map +1 -1
  37. package/dist/sortedSegmentSet.js +33 -41
  38. package/dist/sortedSegmentSet.js.map +1 -1
  39. package/dist/sortedSet.d.ts +20 -3
  40. package/dist/sortedSet.d.ts.map +1 -1
  41. package/dist/sortedSet.js +23 -14
  42. package/dist/sortedSet.js.map +1 -1
  43. package/dist/test/Snapshot.perf.spec.js +1 -1
  44. package/dist/test/Snapshot.perf.spec.js.map +1 -1
  45. package/dist/test/client.applyMsg.spec.js +20 -0
  46. package/dist/test/client.applyMsg.spec.js.map +1 -1
  47. package/dist/test/client.applyStashedOpFarm.spec.js +1 -1
  48. package/dist/test/client.applyStashedOpFarm.spec.js.map +1 -1
  49. package/dist/test/client.attributionFarm.spec.js +1 -1
  50. package/dist/test/client.attributionFarm.spec.js.map +1 -1
  51. package/dist/test/client.localReference.spec.js +48 -0
  52. package/dist/test/client.localReference.spec.js.map +1 -1
  53. package/dist/test/client.obliterateFarm.spec.d.ts +12 -0
  54. package/dist/test/client.obliterateFarm.spec.d.ts.map +1 -0
  55. package/dist/test/client.obliterateFarm.spec.js +89 -0
  56. package/dist/test/client.obliterateFarm.spec.js.map +1 -0
  57. package/dist/test/client.reconnectFarm.spec.js +1 -1
  58. package/dist/test/client.reconnectFarm.spec.js.map +1 -1
  59. package/dist/test/client.searchForMarker.spec.js +2 -2
  60. package/dist/test/client.searchForMarker.spec.js.map +1 -1
  61. package/dist/test/mergeTreeOperationRunner.d.ts +7 -2
  62. package/dist/test/mergeTreeOperationRunner.d.ts.map +1 -1
  63. package/dist/test/mergeTreeOperationRunner.js +31 -14
  64. package/dist/test/mergeTreeOperationRunner.js.map +1 -1
  65. package/dist/test/obliterate.concurrent.spec.js +45 -1
  66. package/dist/test/obliterate.concurrent.spec.js.map +1 -1
  67. package/dist/test/obliterate.rangeExpansion.spec.js +81 -5
  68. package/dist/test/obliterate.rangeExpansion.spec.js.map +1 -1
  69. package/dist/test/obliterate.spec.js +3 -3
  70. package/dist/test/obliterate.spec.js.map +1 -1
  71. package/dist/test/obliterateOperations.d.ts +15 -0
  72. package/dist/test/obliterateOperations.d.ts.map +1 -0
  73. package/dist/test/obliterateOperations.js +132 -0
  74. package/dist/test/obliterateOperations.js.map +1 -0
  75. package/dist/test/partialSyncHelper.d.ts +42 -0
  76. package/dist/test/partialSyncHelper.d.ts.map +1 -0
  77. package/dist/test/partialSyncHelper.js +96 -0
  78. package/dist/test/partialSyncHelper.js.map +1 -0
  79. package/dist/test/revertibles.spec.js +3 -3
  80. package/dist/test/revertibles.spec.js.map +1 -1
  81. package/dist/test/sortedSegmentSet.spec.js +21 -0
  82. package/dist/test/sortedSegmentSet.spec.js.map +1 -1
  83. package/dist/test/testClient.d.ts +1 -1
  84. package/dist/test/testClient.d.ts.map +1 -1
  85. package/dist/test/testClient.js +1 -0
  86. package/dist/test/testClient.js.map +1 -1
  87. package/dist/test/testUtils.js +2 -2
  88. package/dist/test/testUtils.js.map +1 -1
  89. package/lib/attributionCollection.js +5 -7
  90. package/lib/attributionCollection.js.map +1 -1
  91. package/lib/localReference.js +6 -8
  92. package/lib/localReference.js.map +1 -1
  93. package/lib/mergeTree.d.ts.map +1 -1
  94. package/lib/mergeTree.js +69 -34
  95. package/lib/mergeTree.js.map +1 -1
  96. package/lib/mergeTreeNodes.d.ts +15 -4
  97. package/lib/mergeTreeNodes.d.ts.map +1 -1
  98. package/lib/mergeTreeNodes.js +1 -1
  99. package/lib/mergeTreeNodes.js.map +1 -1
  100. package/lib/partialLengths.d.ts +114 -144
  101. package/lib/partialLengths.d.ts.map +1 -1
  102. package/lib/partialLengths.js +432 -525
  103. package/lib/partialLengths.js.map +1 -1
  104. package/lib/perspective.d.ts +10 -1
  105. package/lib/perspective.d.ts.map +1 -1
  106. package/lib/perspective.js +10 -1
  107. package/lib/perspective.js.map +1 -1
  108. package/lib/properties.d.ts.map +1 -1
  109. package/lib/properties.js +2 -3
  110. package/lib/properties.js.map +1 -1
  111. package/lib/revertibles.js +3 -3
  112. package/lib/revertibles.js.map +1 -1
  113. package/lib/segmentInfos.d.ts +3 -0
  114. package/lib/segmentInfos.d.ts.map +1 -1
  115. package/lib/segmentInfos.js.map +1 -1
  116. package/lib/segmentPropertiesManager.js +3 -3
  117. package/lib/segmentPropertiesManager.js.map +1 -1
  118. package/lib/snapshotLoader.js +2 -2
  119. package/lib/snapshotLoader.js.map +1 -1
  120. package/lib/sortedSegmentSet.d.ts +5 -3
  121. package/lib/sortedSegmentSet.d.ts.map +1 -1
  122. package/lib/sortedSegmentSet.js +33 -41
  123. package/lib/sortedSegmentSet.js.map +1 -1
  124. package/lib/sortedSet.d.ts +20 -3
  125. package/lib/sortedSet.d.ts.map +1 -1
  126. package/lib/sortedSet.js +23 -14
  127. package/lib/sortedSet.js.map +1 -1
  128. package/lib/test/Snapshot.perf.spec.js +1 -1
  129. package/lib/test/Snapshot.perf.spec.js.map +1 -1
  130. package/lib/test/client.applyMsg.spec.js +20 -0
  131. package/lib/test/client.applyMsg.spec.js.map +1 -1
  132. package/lib/test/client.applyStashedOpFarm.spec.js +1 -1
  133. package/lib/test/client.applyStashedOpFarm.spec.js.map +1 -1
  134. package/lib/test/client.attributionFarm.spec.js +1 -1
  135. package/lib/test/client.attributionFarm.spec.js.map +1 -1
  136. package/lib/test/client.localReference.spec.js +48 -0
  137. package/lib/test/client.localReference.spec.js.map +1 -1
  138. package/lib/test/client.obliterateFarm.spec.d.ts +12 -0
  139. package/lib/test/client.obliterateFarm.spec.d.ts.map +1 -0
  140. package/lib/test/client.obliterateFarm.spec.js +88 -0
  141. package/lib/test/client.obliterateFarm.spec.js.map +1 -0
  142. package/lib/test/client.reconnectFarm.spec.js +1 -1
  143. package/lib/test/client.reconnectFarm.spec.js.map +1 -1
  144. package/lib/test/client.searchForMarker.spec.js +2 -2
  145. package/lib/test/client.searchForMarker.spec.js.map +1 -1
  146. package/lib/test/mergeTreeOperationRunner.d.ts +7 -2
  147. package/lib/test/mergeTreeOperationRunner.d.ts.map +1 -1
  148. package/lib/test/mergeTreeOperationRunner.js +31 -14
  149. package/lib/test/mergeTreeOperationRunner.js.map +1 -1
  150. package/lib/test/obliterate.concurrent.spec.js +45 -1
  151. package/lib/test/obliterate.concurrent.spec.js.map +1 -1
  152. package/lib/test/obliterate.rangeExpansion.spec.js +81 -5
  153. package/lib/test/obliterate.rangeExpansion.spec.js.map +1 -1
  154. package/lib/test/obliterate.spec.js +3 -3
  155. package/lib/test/obliterate.spec.js.map +1 -1
  156. package/lib/test/obliterateOperations.d.ts +15 -0
  157. package/lib/test/obliterateOperations.d.ts.map +1 -0
  158. package/lib/test/obliterateOperations.js +123 -0
  159. package/lib/test/obliterateOperations.js.map +1 -0
  160. package/lib/test/partialSyncHelper.d.ts +42 -0
  161. package/lib/test/partialSyncHelper.d.ts.map +1 -0
  162. package/lib/test/partialSyncHelper.js +92 -0
  163. package/lib/test/partialSyncHelper.js.map +1 -0
  164. package/lib/test/revertibles.spec.js +3 -3
  165. package/lib/test/revertibles.spec.js.map +1 -1
  166. package/lib/test/sortedSegmentSet.spec.js +21 -0
  167. package/lib/test/sortedSegmentSet.spec.js.map +1 -1
  168. package/lib/test/testClient.d.ts +1 -1
  169. package/lib/test/testClient.d.ts.map +1 -1
  170. package/lib/test/testClient.js +1 -0
  171. package/lib/test/testClient.js.map +1 -1
  172. package/lib/test/testUtils.js +2 -2
  173. package/lib/test/testUtils.js.map +1 -1
  174. package/package.json +21 -79
  175. package/src/mergeTree.ts +80 -28
  176. package/src/mergeTreeNodes.ts +15 -4
  177. package/src/partialLengths.ts +559 -776
  178. package/src/perspective.ts +10 -1
  179. package/src/properties.ts +2 -3
  180. package/src/segmentInfos.ts +3 -0
  181. package/src/snapshotLoader.ts +1 -1
  182. package/src/sortedSegmentSet.ts +41 -50
  183. package/src/sortedSet.ts +32 -16
@@ -5,36 +5,31 @@
5
5
 
6
6
  import { assert } from "@fluidframework/core-utils/internal";
7
7
 
8
- import { Property, RedBlackTree } from "./collections/index.js";
9
8
  import { UnassignedSequenceNumber } from "./constants.js";
10
9
  import { MergeTree } from "./mergeTree.js";
11
10
  import {
12
11
  CollaborationWindow,
13
12
  IMergeNode,
14
13
  ISegmentPrivate,
15
- compareNumbers,
16
14
  seqLTE,
17
15
  type MergeBlock,
18
16
  } from "./mergeTreeNodes.js";
19
- import {
20
- toRemovalInfo,
21
- toMoveInfo,
22
- IRemovalInfo,
23
- IMoveInfo,
24
- assertInserted,
25
- isRemoved,
26
- } from "./segmentInfos.js";
17
+ import { toRemovalInfo, toMoveInfo, assertInserted } from "./segmentInfos.js";
27
18
  import { SortedSet } from "./sortedSet.js";
28
19
 
29
- class PartialSequenceLengthsSet extends SortedSet<PartialSequenceLength, number> {
30
- protected getKey(item: PartialSequenceLength): number {
31
- return item.seq;
20
+ class PartialSequenceLengthsSet extends SortedSet<PartialSequenceLength> {
21
+ protected compare(a: PartialSequenceLength, b: PartialSequenceLength): number {
22
+ return a.seq - b.seq;
32
23
  }
33
24
 
34
25
  public addOrUpdate(
35
26
  newItem: PartialSequenceLength,
36
27
  update?: (existingItem: PartialSequenceLength, newItem: PartialSequenceLength) => void,
37
28
  ): void {
29
+ if (newItem.seglen === 0) {
30
+ // Don't bother doing any updates for deltas of 0.
31
+ return;
32
+ }
38
33
  const prev = this.latestLeq(newItem.seq);
39
34
 
40
35
  if (prev?.seq !== newItem.seq) {
@@ -43,8 +38,8 @@ class PartialSequenceLengthsSet extends SortedSet<PartialSequenceLength, number>
43
38
  }
44
39
 
45
40
  // update the len of all following elements
46
- for (let i = this.keySortedItems.length - 1; i >= 0; i--) {
47
- const element = this.keySortedItems[i];
41
+ for (let i = this.sortedItems.length - 1; i >= 0; i--) {
42
+ const element = this.sortedItems[i];
48
43
  if (!element || element.seq <= newItem.seq) {
49
44
  break;
50
45
  }
@@ -53,15 +48,12 @@ class PartialSequenceLengthsSet extends SortedSet<PartialSequenceLength, number>
53
48
  }
54
49
 
55
50
  super.addOrUpdate(newItem, (currentPartial, partialLength) => {
51
+ assert(
52
+ partialLength.clientId === currentPartial.clientId,
53
+ 0xab6 /* clientId mismatch */,
54
+ );
56
55
  currentPartial.seglen += partialLength.seglen;
57
-
58
- if (partialLength.remoteObliteratedLen) {
59
- currentPartial.remoteObliteratedLen ??= 0;
60
- currentPartial.remoteObliteratedLen += partialLength.remoteObliteratedLen;
61
- }
62
-
63
56
  currentPartial.len += partialLength.seglen;
64
- combineOverlapClients(currentPartial, partialLength);
65
57
  });
66
58
  }
67
59
 
@@ -71,7 +63,7 @@ class PartialSequenceLengthsSet extends SortedSet<PartialSequenceLength, number>
71
63
  * @param key - sequence number
72
64
  */
73
65
  latestLeq(key: number): PartialSequenceLength | undefined {
74
- return this.keySortedItems[this.latestLeqIndex(key)];
66
+ return this.sortedItems[this.latestLeqIndex(key)];
75
67
  }
76
68
 
77
69
  /**
@@ -81,7 +73,7 @@ class PartialSequenceLengthsSet extends SortedSet<PartialSequenceLength, number>
81
73
  */
82
74
  firstGte(key: number): PartialSequenceLength | undefined {
83
75
  const { index } = this.findItemPosition({ seq: key, len: 0, seglen: 0 });
84
- return this.keySortedItems[index];
76
+ return this.sortedItems[index];
85
77
  }
86
78
 
87
79
  private latestLeqIndex(key: number): number {
@@ -93,28 +85,23 @@ class PartialSequenceLengthsSet extends SortedSet<PartialSequenceLength, number>
93
85
  const mindex = this.latestLeqIndex(minSeq);
94
86
  let minLength = 0;
95
87
  if (mindex >= 0) {
96
- minLength = this.keySortedItems[mindex].len;
88
+ minLength = this.sortedItems[mindex].len;
97
89
  const seqCount = this.size;
98
90
  if (mindex <= seqCount - 1) {
99
91
  // Still some entries remaining
100
92
  const remainingCount = seqCount - mindex - 1;
101
93
  // Copy down
102
94
  for (let i = 0; i < remainingCount; i++) {
103
- this.keySortedItems[i] = this.keySortedItems[i + mindex + 1];
104
- this.keySortedItems[i].len -= minLength;
95
+ this.sortedItems[i] = this.sortedItems[i + mindex + 1];
96
+ this.sortedItems[i].len -= minLength;
105
97
  }
106
- this.keySortedItems.length = remainingCount;
98
+ this.sortedItems.length = remainingCount;
107
99
  }
108
100
  }
109
101
  return minLength;
110
102
  }
111
103
  }
112
104
 
113
- interface IOverlapClient {
114
- clientId: number;
115
- seglen: number;
116
- }
117
-
118
105
  /**
119
106
  * Tracks length information for a part of a MergeTree (block) at a given time (seq).
120
107
  * These objects are associated with internal nodes (i.e. blocks).
@@ -136,80 +123,40 @@ export interface PartialSequenceLength {
136
123
  * clientId for the client that submitted the op with sequence number `seq`.
137
124
  */
138
125
  clientId?: number;
139
- /**
140
- * If this partial length obliterated remote segments, this is the length of
141
- * those segments
142
- */
143
- remoteObliteratedLen?: number;
144
- /**
145
- * This field maps each client to the size of the intersection between segments deleted at this seq
146
- * and segments concurrently deleted by that client.
147
- *
148
- * For example, this PartialSequenceLength:
149
- * ```typescript
150
- * {
151
- * seq: 5,
152
- * len: 100,
153
- * seglen: -10,
154
- * clientId: 0,
155
- * overlapRemoveClients: <RedBlack tree with key-values expressed by>{
156
- * 1: { clientId: 1, seglen: -5 },
157
- * 3: { clientId: 3, seglen: -10 }
158
- * }
159
- * }
160
- * ```
161
- *
162
- * corresponds to an op submitted by client 0 which:
163
- * - reduces the length of this block by 10 (it may have deleted a single segment of length 10,
164
- * several segments totalling length 10, or even delete and add content for a total reduction of 10 length)
165
- * - was concurrent to one or more ops submitted by client 1 that also removed some of the same segments,
166
- * whose length totalled 5
167
- * - was concurrent to one or more ops submitted by client 3 that removed some of the same segments,
168
- * whose length totalled 10
169
- */
170
- overlapRemoveClients?: RedBlackTree<number, IOverlapClient>;
171
- /**
172
- * This field is the same as `overlapRemoveClients`, except that it tracks
173
- * overlapping obliterates rather than removes.
174
- */
175
- overlapObliterateClients?: RedBlackTree<number, IOverlapClient>;
176
126
  }
177
127
 
178
128
  interface UnsequencedPartialLengthInfo {
179
129
  /**
180
130
  * Contains entries for all local operations.
181
131
  * The "seq" field of each entry actually corresponds to the delta at that localSeq on the local client.
132
+ *
133
+ * The length entries in this set are analogous to `PartialSequenceLengths.partialLengths` in that they represent the delta over the min seq
134
+ * that an observer client would see if they were to observe the local client's edits performed from the minSeq.
182
135
  */
183
136
  partialLengths: PartialSequenceLengthsSet;
184
137
 
185
138
  /**
186
- * Only contains entries for segments (or aggregates thereof) which were concurrently deleted
187
- * by another client. Ordered by `seq` of the removing client.
139
+ * Like PerClientAdjustments, except we store one set of PartialSequenceLengthsSet for each refSeq. The "seq" keys in these sets
140
+ * are all local seqs.
188
141
  *
189
- * The "length" field of these entries is not populated. This is because pre-computing the lengths
190
- * of segments doesn't help given the usage pattern.
142
+ * These entries are aggregated by {@link PartialSequenceLengths.computeOverallRefSeqAdjustment} when a local perspective for a
143
+ * given refSeq is requested.
191
144
  *
192
- * These entries need both `seq` and `localSeq`, because a given segment remove is double-counted iff
193
- * the refSeq exceeds the seq of the remote remove AND the localSeq exceeds the localSeq of the local remove.
145
+ * In general, adjustments in this map are added to avoid double-counting an operation performed by both the local client and some
146
+ * remote client, and an adjustment at (refSeq = A, clientSeq = B) takes effect for all perspectives (refSeq = C, clientSeq = D) where
147
+ * A \<= C and B \<= D.
194
148
  */
195
- overlappingRemoves: LocalPartialSequenceLength[];
149
+ perRefSeqAdjustments: Map<number, PartialSequenceLengthsSet>;
196
150
 
197
151
  /**
198
- * Cached keyed on refSeq which stores length information for the total overlap of removed segments at
152
+ * Cache keyed on refSeq which stores length information for the total overlap of removed segments at
199
153
  * that refSeq.
200
- * This information is derivable from the entries of `overlappingRemoves`.
154
+ * This information is derivable from the entries of `perRefSeqAdjustments`.
201
155
  *
202
156
  * Like the `partialLengths` field, `seq` on each entry is actually the local seq.
203
- * See `computeOverlappingLocalRemoves` for more information.
204
- */
205
- cachedOverlappingByRefSeq: Map<number, PartialSequenceLengthsSet>;
206
- }
207
-
208
- interface LocalPartialSequenceLength extends PartialSequenceLength {
209
- /**
210
- * Local sequence number
157
+ * See `computeOverallRefSeqAdjustment` for more information.
211
158
  */
212
- localSeq: number;
159
+ cachedAdjustmentByRefSeq: Map<number, PartialSequenceLengthsSet>;
213
160
  }
214
161
 
215
162
  export interface PartialSequenceLengthsOptions {
@@ -232,13 +179,13 @@ export interface PartialSequenceLengthsOptions {
232
179
  * "What is the length of `block` from the perspective of some particular seq and clientId?".
233
180
  *
234
181
  * It also supports incremental updating of state for newly-sequenced ops that don't affect the structure of the
235
- * MergeTree.
182
+ * MergeTree (in most cases--see AB#31003 or comments on {@link PartialSequenceLengths.update}).
236
183
  *
237
184
  * To answer these queries, it pre-builds several lists which track the length of the block at a per-sequence-number
238
185
  * level. These lists are:
239
186
  *
240
187
  * 1. (`partialLengths`): Stores the total length of the block.
241
- * 2. (`clientSeqNumbers[clientId]`): Stores only the total lengths of segments submitted by `clientId`. [see footnote]
188
+ * 2. (`perClientAdjustments[clientId]`): Stores adjustments to the base length which account for all changes submitted by `clientId`. [see footnote]
242
189
  *
243
190
  * The reason both lists are necessary is that resolving the length of the block from the perspective of
244
191
  * (clientId, refSeq) requires including both of the following types of segments:
@@ -261,19 +208,119 @@ export interface PartialSequenceLengthsOptions {
261
208
  * (length of the block at the minimum sequence number)
262
209
  * + (partialLengths total length at refSeq)
263
210
  * + (unsequenced edits' total length submitted before localSeq)
264
- * - (overlapping remove of the unsequenced edits' total length at refSeq)
211
+ * + (adjustments for changes double-counted by happening at or before both localSeq and refSeq)
265
212
  *
266
213
  * This algorithm scales roughly linearly with number of editing clients and the size of the collab window.
267
214
  * (certain unlikely sequences of operations may introduce log factors on those variables)
268
215
  *
269
- * Note: there is some slight complication with clientSeqNumbers resulting from the possibility of different clients
270
- * concurrently removing the same segment. See the field's documentation for more details.
216
+ * @privateRemarks
217
+ * If you are looking to understand this class in more detail, a suggested order of internalization is:
218
+ *
219
+ * 1. The above description and how it relates to the implementation of `getPartialLength` (which implements the above high-level description
220
+ * 2. `PartialSequenceLengthsSet`, which allows binary searching for overall length deltas at a given sequence number and handles updates.
221
+ * 3. The `fromLeaves` method, which is the base case for the [potential] recursion in `combine`
222
+ * 4. The logic in `combine` to aggregate smaller block entries into larger ones
223
+ * 5. The incremental code path of `update`
271
224
  */
272
225
  export class PartialSequenceLengths {
273
226
  public static options: PartialSequenceLengthsOptions = {
274
227
  zamboni: true,
275
228
  };
276
229
 
230
+ /**
231
+ * Length of the block this PartialSequenceLength corresponds to when viewed at `minSeq`.
232
+ */
233
+ private minLength = 0;
234
+
235
+ /**
236
+ * Total number of segments in the subtree rooted at the block this PartialSequenceLength corresponds to.
237
+ */
238
+ private segmentCount = 0;
239
+
240
+ /**
241
+ * List of PartialSequenceLength objects--ordered by increasing seq--giving length information about
242
+ * the block associated with this PartialSequenceLengths object.
243
+ *
244
+ * `minLength + partialLengths[i].len` gives the length of this block when considering the perspective of an observer
245
+ * client who has received edits up to (and including) sequence number `i`.
246
+ */
247
+ private readonly partialLengths: PartialSequenceLengthsSet = new PartialSequenceLengthsSet();
248
+
249
+ /**
250
+ * perClientAdjustments[clientId] contains a PartialSequenceLengthsSet of adjustments to the observer client's
251
+ * perspective (see {@link PartialSequenceLengths.partialLengths}) necessary to account for changes made by
252
+ * that client.
253
+ *
254
+ * As per doc comment on {@link PartialSequenceLengths}, the overall adjustment performed for the perspective of
255
+ * (clientId, refSeq) is given by the sum of length deltas in `perClientAdjustments[clientId]`
256
+ * for all sequence numbers S such that S \>= refSeq.
257
+ *
258
+ * (since these are ordered by sequence number and we cache cumulative sums, this is implemented using two lookups and a subtraction).
259
+ *
260
+ * The specific adjustments are roughly categorized as follows:
261
+ *
262
+ * - Ops submitted by a given client generally receive a partial lengths entry corresponding to their sequence number.
263
+ * e.g. insert of "ABC" at seq 5 will have a per-client adjustment entry of \{ seq: 5, seglen: 3 \}.
264
+ *
265
+ * - When client A deletes a segment concurrently with client B and loses the race (B's delete is sequenced first),
266
+ * A's per-client adjustments will contain an entry with a negative `seglen` corresponding to the length of the segment
267
+ * and a sequence number corresponding to that of B's delete. It will *not* receive a per-client adjustment for its own delete.
268
+ * This ensures that for perspectives (A, refSeq), the deleted segment will show up as a negative delta for all values of refSeq, since:
269
+ * 1. For refSeq \< B's delete, the per-client adjustment will apply and be added to the total length
270
+ * 2. For refSeq \>= B's delete, B's partial length entry in the overall set will apply, and the per-client adjustment will not apply
271
+ *
272
+ * - When client A attempts to insert a segment into a location that is concurrently obliterated by client B immediately upon insertion,
273
+ * A's per-client adjustments will again not include an entry for its own insert.
274
+ * Instead, the entry which would normally contain `seq` equal to that of A's insert would instead have `seq` equal to that of B's obliterate.
275
+ * This gives the overall correct behavior: for any perspective which isn't client A, there is no adjustment necessary anywhere (it's as if
276
+ * the segment never existed). For client A's perspective, the segment should be considered visible until A has acked B's obliterate.
277
+ * This is accomplished as for the perspective (A, refSeq):
278
+ * 1. For refSeq \< B's obliterate, the segment length will be included as part of the per-client adjustment for A
279
+ * 2. For refSeq \>= B's obliterate, the segment will be omitted from the per-client adjustment for A
280
+ *
281
+ * Note that the special-casing for inserting segments that are immediately obliterated is only necessary for segments that never were visible
282
+ * in the tree. If an insert and obliterate are concurrent but the insert is sequenced first, the normal per-client adjustment is fine.
283
+ *
284
+ * The second case (overlapping removal) applies to any combination of remove / obliterate operations.
285
+ */
286
+ private readonly perClientAdjustments: PartialSequenceLengthsSet[] = [];
287
+
288
+ /**
289
+ * Contains information required to answer queries for the length of this segment from the perspective of
290
+ * the local client but not including all local segments (i.e., `localSeq !== collabWindow.localSeq`).
291
+ * This field is only computed if requested in the constructor (i.e. `computeLocalPartials === true`).
292
+ *
293
+ * Note that the usage pattern for this list is a bit different from perClientAdjustments: when dealing with perspectives of remote clients,
294
+ * we generally want to know what their view of the block was accounting for all changes made by that client as well as all \<= some refSeq.
295
+ *
296
+ * However, when dealing with perspectives relevant to the local client, we are still interested in changes made \<= some refSeq, but instead
297
+ * of caring about all changes made by the local client, we additionally want the subset of them that were made \<= some localSeq.
298
+ *
299
+ * The PartialSequenceLengthsSets stored in this field therefore track localSeqs rather than seqs (it's still named seq for ease of implementation).
300
+ * Furthermore, when computing the length of the block at a given refSeq/localSeq perspective,
301
+ * rather than add something like `perClientAdjustments[clientId].latestLeq(latestSeq) - perClientAdjustments[clientId].latestLeq(refSeq)` [to
302
+ * get the tail end of adjustments necessary for a remote client client], we instead add `unsequencedRecords.partialLengths.latestLeq(localSeq)`
303
+ * [to get the head end of adjustments necessary for the local client].
304
+ */
305
+ private unsequencedRecords: UnsequencedPartialLengthInfo | undefined;
306
+
307
+ constructor(
308
+ /**
309
+ * The minimumSequenceNumber as defined by the collab window used in the last call to `update`,
310
+ * or if no such calls have been made, the one used on construction.
311
+ */
312
+ public minSeq: number,
313
+ computeLocalPartials: boolean,
314
+ ) {
315
+ if (computeLocalPartials) {
316
+ this.unsequencedRecords = {
317
+ partialLengths: new PartialSequenceLengthsSet(),
318
+ perRefSeqAdjustments: new Map(),
319
+ cachedAdjustmentByRefSeq: new Map(),
320
+ };
321
+ }
322
+ }
323
+
277
324
  /**
278
325
  * Combine the partial lengths of block's children
279
326
  * @param block - an interior node. If `recur` is false, it is assumed that each interior node child of this block
@@ -288,7 +335,6 @@ export class PartialSequenceLengths {
288
335
  */
289
336
  public static combine(
290
337
  block: MergeBlock,
291
-
292
338
  collabWindow: CollaborationWindow,
293
339
  recur = false,
294
340
  computeLocalPartials = false,
@@ -333,7 +379,7 @@ export class PartialSequenceLengths {
333
379
 
334
380
  const childPartialLengths: PartialSequenceLength[][] = [];
335
381
  const childUnsequencedPartialLengths: PartialSequenceLength[][] = [];
336
- const childOverlapRemoves: LocalPartialSequenceLength[][] = [];
382
+ const childPerRefSeqAdjustments: Map<number, PartialSequenceLengthsSet>[] = [];
337
383
  for (let i = 0; i < childPartialsLen; i++) {
338
384
  const { segmentCount, minLength, partialLengths, unsequencedRecords } =
339
385
  childPartials[i];
@@ -344,7 +390,7 @@ export class PartialSequenceLengths {
344
390
  childUnsequencedPartialLengths.push(
345
391
  unsequencedRecords.partialLengths.items as PartialSequenceLength[],
346
392
  );
347
- childOverlapRemoves.push(unsequencedRecords.overlappingRemoves);
393
+ childPerRefSeqAdjustments.push(unsequencedRecords.perRefSeqAdjustments);
348
394
  }
349
395
  }
350
396
 
@@ -353,13 +399,47 @@ export class PartialSequenceLengths {
353
399
  if (computeLocalPartials) {
354
400
  combinedPartialLengths.unsequencedRecords = {
355
401
  partialLengths: mergePartialLengths(childUnsequencedPartialLengths),
356
- overlappingRemoves: [...mergeSortedListsBySeq(childOverlapRemoves)],
357
- cachedOverlappingByRefSeq: new Map(),
402
+ cachedAdjustmentByRefSeq: new Map(),
403
+ perRefSeqAdjustments: new Map(),
358
404
  };
405
+
406
+ for (const perRefSeq of childPerRefSeqAdjustments) {
407
+ for (const [refSeq, partials] of perRefSeq) {
408
+ let combinedPartials =
409
+ combinedPartialLengths.unsequencedRecords.perRefSeqAdjustments.get(refSeq);
410
+ if (combinedPartials === undefined) {
411
+ combinedPartials = new PartialSequenceLengthsSet();
412
+ combinedPartialLengths.unsequencedRecords.perRefSeqAdjustments.set(
413
+ refSeq,
414
+ combinedPartials,
415
+ );
416
+ }
417
+ for (const item of partials.items) {
418
+ combinedPartials.addOrUpdate({ ...item });
419
+ }
420
+ }
421
+ }
359
422
  }
360
423
 
361
- for (const partial of combinedPartialLengths.partialLengths.items) {
362
- combinedPartialLengths.addClientSeqNumberFromPartial(partial);
424
+ // could merge these like we do above rather than do out of order like this
425
+ for (let i = 0; i < childPartialsLen; i++) {
426
+ const { perClientAdjustments } = childPartials[i];
427
+ if (perClientAdjustments.length > 0) {
428
+ for (let clientId = 0; clientId < perClientAdjustments.length; clientId++) {
429
+ const clientAdjustment = perClientAdjustments[clientId];
430
+ if (clientAdjustment === undefined) {
431
+ continue;
432
+ }
433
+
434
+ for (const partial of perClientAdjustments[clientId].items) {
435
+ combinedPartialLengths.addClientAdjustment(
436
+ clientId,
437
+ partial.seq,
438
+ partial.seglen,
439
+ );
440
+ }
441
+ }
442
+ }
363
443
  }
364
444
  }
365
445
  // TODO: incremental zamboni during build
@@ -372,14 +452,14 @@ export class PartialSequenceLengths {
372
452
  }
373
453
 
374
454
  /**
375
- * Creates and returns a PartialSequenceLengths structure that tracks the lengths of only the
376
- * leaf children of the provided MergeBlock.
455
+ * Create a `PartialSequenceLengths` which tracks only changes incurred by direct child leaves of `block`.
377
456
  */
378
457
  private static fromLeaves(
379
458
  block: MergeBlock,
380
459
 
381
460
  collabWindow: CollaborationWindow,
382
461
  computeLocalPartials: boolean,
462
+ retry = true,
383
463
  ): PartialSequenceLengths {
384
464
  const combinedPartialLengths = new PartialSequenceLengths(
385
465
  collabWindow.minSeq,
@@ -392,46 +472,26 @@ export class PartialSequenceLengths {
392
472
  if (child.isLeaf()) {
393
473
  // Leaf segment
394
474
  const segment = child;
395
- if (segment.seq !== undefined && seqLTE(segment.seq, collabWindow.minSeq)) {
396
- combinedPartialLengths.minLength += segment.cachedLength;
397
- } else {
398
- PartialSequenceLengths.insertSegment(combinedPartialLengths, segment);
399
- }
400
- const removalInfo = toRemovalInfo(segment);
401
475
  const moveInfo = toMoveInfo(segment);
402
- if (
403
- (removalInfo?.removedSeq !== undefined &&
404
- seqLTE(removalInfo.removedSeq, collabWindow.minSeq)) ||
405
- (moveInfo?.movedSeq !== undefined && seqLTE(moveInfo.movedSeq, collabWindow.minSeq))
406
- ) {
407
- combinedPartialLengths.minLength -= segment.cachedLength;
408
- } else if (removalInfo !== undefined || moveInfo !== undefined) {
409
- PartialSequenceLengths.insertSegment(
476
+ if (moveInfo?.wasMovedOnInsert) {
477
+ PartialSequenceLengths.accountForMoveOnInsert(
410
478
  combinedPartialLengths,
411
479
  segment,
412
- removalInfo,
413
- moveInfo,
480
+ collabWindow,
481
+ );
482
+ } else {
483
+ PartialSequenceLengths.accountForInsertion(
484
+ combinedPartialLengths,
485
+ segment,
486
+ collabWindow,
414
487
  );
415
- }
416
- }
417
- }
418
- // Post-process correctly-ordered partials computing sums and creating
419
- // lists for each present client id
420
- const seqPartials = combinedPartialLengths.partialLengths;
421
-
422
- let prevLen = 0;
423
- for (const partial of seqPartials.items) {
424
- partial.len = prevLen + partial.seglen;
425
- prevLen = partial.len;
426
- combinedPartialLengths.addClientSeqNumberFromPartial(partial);
427
- }
428
- prevLen = 0;
429
488
 
430
- if (combinedPartialLengths.unsequencedRecords !== undefined) {
431
- const localPartials = combinedPartialLengths.unsequencedRecords.partialLengths;
432
- for (const partial of localPartials.items) {
433
- partial.len = prevLen + partial.seglen;
434
- prevLen = partial.len;
489
+ PartialSequenceLengths.accountForRemoval(
490
+ combinedPartialLengths,
491
+ segment,
492
+ collabWindow,
493
+ );
494
+ }
435
495
  }
436
496
  }
437
497
 
@@ -439,230 +499,169 @@ export class PartialSequenceLengths {
439
499
  return combinedPartialLengths;
440
500
  }
441
501
 
442
- private static getOverlapClients(
443
- overlapClientIds: number[],
444
- seglen: number,
445
- ): RedBlackTree<number, IOverlapClient> {
446
- const bst = new RedBlackTree<number, IOverlapClient>(compareNumbers);
447
- for (const clientId of overlapClientIds) {
448
- bst.put(clientId, { clientId, seglen });
449
- }
450
- return bst;
451
- }
452
-
453
- private static accumulateRemoveClientOverlap(
454
- partialLength: PartialSequenceLength,
455
- overlapRemoveClientIds: number[],
456
- seglen: number,
457
- ): void {
458
- if (partialLength.overlapRemoveClients) {
459
- for (const clientId of overlapRemoveClientIds) {
460
- const overlapClientNode = partialLength.overlapRemoveClients.get(clientId);
461
- if (overlapClientNode) {
462
- overlapClientNode.data.seglen += seglen;
463
- } else {
464
- partialLength.overlapRemoveClients.put(clientId, { clientId, seglen });
465
- }
466
- }
467
- } else {
468
- partialLength.overlapRemoveClients = PartialSequenceLengths.getOverlapClients(
469
- overlapRemoveClientIds,
470
- seglen,
471
- );
472
- }
473
- }
474
-
475
- private static accumulateMoveClientOverlap(
476
- partialLength: PartialSequenceLength,
477
- overlapMoveClientIds: number[],
478
- seglen: number,
479
- ): void {
480
- if (partialLength.overlapObliterateClients) {
481
- for (const clientId of overlapMoveClientIds) {
482
- const overlapClientNode = partialLength.overlapObliterateClients.get(clientId);
483
- if (overlapClientNode) {
484
- overlapClientNode.data.seglen += seglen;
485
- } else {
486
- partialLength.overlapObliterateClients.put(clientId, { clientId, seglen });
487
- }
488
- }
489
- } else {
490
- partialLength.overlapObliterateClients = PartialSequenceLengths.getOverlapClients(
491
- overlapMoveClientIds,
492
- seglen,
493
- );
494
- }
495
- }
496
-
497
502
  /**
498
- * Coalesce overlapping move lengths for a partial length entry that already
499
- * exists
500
- *
501
- * @param segmentLen - Length of segment with overlapping moves
502
- * @param segment - Segment with overlapping moves
503
- * @param firstGte - Existing partial length entry
504
- * @param clientIds - Ids of clients that concurrently obliterated this segment
503
+ * Assuming this segment was moved on insertion, inserts length information about that operation
504
+ * into the appropriate per-client adjustments (the overall view needs no such adjustment since
505
+ * from an observing client's perspective, the segment never exists).
505
506
  */
506
- static accumulateMoveOverlapForExisting(
507
- segmentLen: number,
507
+ private static accountForMoveOnInsert(
508
+ combinedPartialLengths: PartialSequenceLengths,
508
509
  segment: ISegmentPrivate,
509
- firstGte: PartialSequenceLength,
510
- clientIds: number[],
510
+ collabWindow: CollaborationWindow,
511
511
  ): void {
512
512
  assertInserted(segment);
513
- const nonInsertingClientIds = clientIds.filter((id) => id !== segment.clientId);
513
+ const moveInfo = toMoveInfo(segment);
514
+ assert(moveInfo?.wasMovedOnInsert === true, 0xab7 /* Segment was not moved on insert */);
515
+ if (moveInfo.movedSeq <= collabWindow.minSeq) {
516
+ // This segment was obliterated as soon as it was inserted, and everyone was aware of the obliterate.
517
+ // Thus every single client treats this segment as length 0 from every perspective, and no adjustments
518
+ // are necessary.
519
+ return;
520
+ }
514
521
 
515
- PartialSequenceLengths.accumulateMoveClientOverlap(
516
- firstGte,
517
- nonInsertingClientIds,
518
- segmentLen,
519
- );
522
+ const isLocal = segment.seq === UnassignedSequenceNumber;
523
+ const clientId = segment.clientId;
520
524
 
521
- // if this segment was obliterated by the client that inserted it,
522
- // and if it overlaps with the obliterate of another client, we need to
523
- // take into account whether it was obliterated on insert by the other
524
- // client
525
- if (clientIds.length !== nonInsertingClientIds.length) {
526
- PartialSequenceLengths.accumulateMoveClientOverlap(
527
- firstGte,
528
- [segment.clientId],
529
- toMoveInfo(segment)?.wasMovedOnInsert ? -segment.cachedLength : segmentLen,
530
- );
525
+ const partials = isLocal
526
+ ? combinedPartialLengths.unsequencedRecords?.partialLengths
527
+ : combinedPartialLengths.partialLengths;
528
+ if (partials === undefined) {
529
+ // Local partial but its computation isn't required
530
+ return;
531
+ }
532
+
533
+ if (isLocal) {
534
+ // Implication -> this is a local segment which will be obliterated as soon as it is acked.
535
+ // For refSeqs preceding that movedSeq and localSeqs following the localSeq, it will be visible.
536
+ // For the rest, it will not be visible.
537
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
538
+ const localSeq = segment.localSeq!;
539
+ partials.addOrUpdate({
540
+ seq: localSeq,
541
+ len: 0,
542
+ seglen: segment.cachedLength,
543
+ clientId,
544
+ });
545
+
546
+ combinedPartialLengths.addLocalAdjustment({
547
+ refSeq: moveInfo.movedSeq,
548
+ localSeq,
549
+ seglen: -segment.cachedLength,
550
+ });
551
+ } else {
552
+ // Segment was obliterated on insert. Generally this means it should be visible only to the
553
+ // inserting client (in which case we add an adjustment to only that client's perspective),
554
+ // but if that client has also removed it, we don't need to add anything.
555
+ const removeInfo = toRemovalInfo(segment);
556
+
557
+ const wasRemovedByInsertingClient =
558
+ removeInfo !== undefined && removeInfo.removedClientIds.includes(clientId);
559
+ const wasMovedByInsertingClient =
560
+ moveInfo !== undefined && moveInfo.movedClientIds.includes(clientId);
561
+
562
+ if (!wasRemovedByInsertingClient && !wasMovedByInsertingClient) {
563
+ const moveSeq = moveInfo?.movedSeq;
564
+ assert(
565
+ moveSeq !== undefined,
566
+ 0xab8 /* ObliterateOnInsertion implies moveSeq is defined */,
567
+ );
568
+ combinedPartialLengths.addClientAdjustment(clientId, moveSeq, segment.cachedLength);
569
+ }
531
570
  }
532
571
  }
533
572
 
534
573
  /**
535
- * Tracks which clients have made concurrent obliterates.
536
- *
537
- * @param obliterateOverlapLen - Length of segment with overlap
538
- * @param clientIds - Ids of clients that have concurrently obliterated this
539
- * segment
574
+ * Inserts length information about the insertion of `segment` into
575
+ * `combinedPartialLengths.partialLengths` and the appropriate per-client adjustments.
540
576
  */
541
- private static getMoveOverlapForExisting(
577
+ private static accountForInsertion(
578
+ combinedPartialLengths: PartialSequenceLengths,
542
579
  segment: ISegmentPrivate,
543
- obliterateOverlapLen: number,
544
- clientIds: number[],
545
- ): RedBlackTree<number, IOverlapClient> {
580
+ collabWindow: CollaborationWindow,
581
+ ): void {
546
582
  assertInserted(segment);
547
- const nonInsertingClientIds = clientIds.filter((id) => id !== segment.clientId);
548
- const overlapObliterateClients = PartialSequenceLengths.getOverlapClients(
549
- nonInsertingClientIds,
550
- obliterateOverlapLen,
551
- );
552
-
553
- if (clientIds.length !== nonInsertingClientIds.length) {
554
- overlapObliterateClients.put(segment.clientId, {
555
- clientId: segment.clientId,
556
- seglen: toMoveInfo(segment)?.wasMovedOnInsert
557
- ? -segment.cachedLength
558
- : obliterateOverlapLen,
559
- });
583
+ if (segment.seq !== undefined && seqLTE(segment.seq, collabWindow.minSeq)) {
584
+ combinedPartialLengths.minLength += segment.cachedLength;
585
+ return;
560
586
  }
561
587
 
562
- return overlapObliterateClients;
563
- }
588
+ const isLocal = segment.seq === UnassignedSequenceNumber;
589
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
590
+ const seqOrLocalSeq = isLocal ? segment.localSeq! : segment.seq;
591
+ const segmentLen = segment.cachedLength;
592
+ const clientId = segment.clientId;
564
593
 
565
- private static updatePartialsAfterInsertion(
566
- segment: ISegmentPrivate,
567
- segmentLen: number,
568
- remoteObliteratedLen: number | undefined,
569
- obliterateOverlapLen: number = segmentLen,
570
- partials: PartialSequenceLengthsSet,
571
- seq: number,
572
- clientId: number,
573
- removeClientOverlap: number[] | undefined,
574
- moveClientOverlap: number[] | undefined,
575
- ): void {
576
- const firstGte = partials.firstGte(seq);
577
-
578
- let partialLengthEntry: PartialSequenceLength;
579
- if (firstGte?.seq === seq) {
580
- partialLengthEntry = firstGte;
581
- // Existing entry at this seq--this occurs for ops that insert/delete
582
- // more than one segment.
583
- partialLengthEntry.seglen += segmentLen;
584
- if (remoteObliteratedLen) {
585
- partialLengthEntry.remoteObliteratedLen ??= 0;
586
- partialLengthEntry.remoteObliteratedLen += remoteObliteratedLen;
587
- }
588
- if (removeClientOverlap) {
589
- PartialSequenceLengths.accumulateRemoveClientOverlap(
590
- firstGte,
591
- removeClientOverlap,
592
- obliterateOverlapLen,
593
- );
594
- }
594
+ const partials = isLocal
595
+ ? combinedPartialLengths.unsequencedRecords?.partialLengths
596
+ : combinedPartialLengths.partialLengths;
597
+ if (!partials) {
598
+ // Local partial but its computation isn't required
599
+ return;
600
+ }
595
601
 
596
- if (moveClientOverlap) {
597
- PartialSequenceLengths.accumulateMoveOverlapForExisting(
598
- obliterateOverlapLen,
599
- segment,
600
- firstGte,
601
- moveClientOverlap,
602
- );
603
- }
602
+ if (isLocal) {
603
+ partials.addOrUpdate({
604
+ seq: seqOrLocalSeq,
605
+ clientId,
606
+ len: 0,
607
+ seglen: segmentLen,
608
+ });
604
609
  } else {
605
- const overlapObliterateClients = moveClientOverlap
606
- ? PartialSequenceLengths.getMoveOverlapForExisting(
607
- segment,
608
- obliterateOverlapLen,
609
- moveClientOverlap,
610
- )
611
- : undefined;
612
-
613
- partialLengthEntry = {
614
- seq,
610
+ partials.addOrUpdate({
611
+ seq: seqOrLocalSeq,
615
612
  clientId,
616
613
  len: 0,
617
614
  seglen: segmentLen,
618
- remoteObliteratedLen,
619
- overlapRemoveClients: removeClientOverlap
620
- ? PartialSequenceLengths.getOverlapClients(removeClientOverlap, obliterateOverlapLen)
621
- : undefined,
622
- overlapObliterateClients,
623
- };
624
-
625
- partials.addOrUpdate(partialLengthEntry);
615
+ });
616
+ combinedPartialLengths.addClientAdjustment(clientId, seqOrLocalSeq, segmentLen);
626
617
  }
627
618
  }
628
619
 
629
620
  /**
630
- * Inserts length information about the insertion of `segment` into
631
- * `combinedPartialLengths.partialLengths`.
632
- *
633
- * Does not update the clientSeqNumbers field to account for this segment.
634
- *
635
- * If `removalInfo` or `moveInfo` are defined, this operation updates the
636
- * bookkeeping to account for the (re)moval of this segment at the (re)movedSeq
637
- * instead.
638
- *
639
- * When the insertion or (re)moval of the segment is un-acked and
640
- * `combinedPartialLengths` is meant to compute such records, this does the
641
- * analogous addition to the bookkeeping for the local segment in
642
- * `combinedPartialLengths.unsequencedRecords`.
621
+ * Inserts length information about the removal or obliteration of `segment` into
622
+ * `combinedPartialLengths.partialLengths` and the appropriate per-client adjustments.
643
623
  */
644
- private static insertSegment(
624
+ private static accountForRemoval(
645
625
  combinedPartialLengths: PartialSequenceLengths,
646
626
  segment: ISegmentPrivate,
647
- removalInfo?: IRemovalInfo,
648
- moveInfo?: IMoveInfo,
627
+ collabWindow: CollaborationWindow,
649
628
  ): void {
650
629
  assertInserted(segment);
651
630
 
631
+ const removalInfo = toRemovalInfo(segment);
632
+ const moveInfo = toMoveInfo(segment);
633
+ if (!removalInfo && !moveInfo) {
634
+ return;
635
+ }
636
+
637
+ if (
638
+ (removalInfo?.removedSeq !== undefined &&
639
+ seqLTE(removalInfo.removedSeq, collabWindow.minSeq)) ||
640
+ (moveInfo?.movedSeq !== undefined && seqLTE(moveInfo.movedSeq, collabWindow.minSeq))
641
+ ) {
642
+ combinedPartialLengths.minLength -= segment.cachedLength;
643
+ return;
644
+ }
645
+
652
646
  const removalIsLocal =
653
647
  !!removalInfo && removalInfo.removedSeq === UnassignedSequenceNumber;
654
648
  const moveIsLocal = !!moveInfo && moveInfo.movedSeq === UnassignedSequenceNumber;
655
- const isLocal =
656
- segment.seq === UnassignedSequenceNumber ||
657
- (!!removalInfo && removalIsLocal && (!moveInfo || moveIsLocal)) ||
658
- (!!moveInfo && moveIsLocal && (!removalInfo || removalIsLocal));
659
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
660
- let seqOrLocalSeq = isLocal ? segment.localSeq! : segment.seq;
661
- let segmentLen = segment.cachedLength;
662
- let clientId = segment.clientId;
663
- let removeClientOverlap: number[] | undefined;
664
- let moveClientOverlap: number[] | undefined;
665
- let remoteObliteratedLen: number | undefined;
649
+ const isLocalInsertion = segment.seq === UnassignedSequenceNumber;
650
+ const isOnlyLocalRemoval = removalIsLocal && (!moveInfo || moveIsLocal);
651
+ const isOnlyLocalMove = moveIsLocal && (!removalInfo || removalIsLocal);
652
+ const isLocal = isLocalInsertion || isOnlyLocalRemoval || isOnlyLocalMove;
653
+
654
+ if (
655
+ segment.seq === UnassignedSequenceNumber &&
656
+ !(removalIsLocal && (!moveInfo || moveIsLocal)) &&
657
+ !(moveIsLocal && (!removalInfo || removalIsLocal))
658
+ ) {
659
+ throw new Error("Should have handled this codepath in wasMovedOnInsertion");
660
+ }
661
+
662
+ const lenDelta = -segment.cachedLength;
663
+ let clientId: number;
664
+ let seqOrLocalSeq: number;
666
665
 
667
666
  // it's not possible to have an overlapping obliterate and remove that are both local
668
667
  assert(
@@ -670,6 +669,11 @@ export class PartialSequenceLengths {
670
669
  0x870 /* overlapping local obliterate and remove */,
671
670
  );
672
671
 
672
+ const clientsWithRemoveOrObliterate = new Set<number>([
673
+ ...(removalInfo?.removedClientIds ?? []),
674
+ ...(moveInfo?.movedClientIds ?? []),
675
+ ]);
676
+
673
677
  const removeHappenedFirst =
674
678
  removalInfo &&
675
679
  (!moveInfo ||
@@ -679,40 +683,20 @@ export class PartialSequenceLengths {
679
683
  if (removeHappenedFirst) {
680
684
  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
681
685
  seqOrLocalSeq = removalIsLocal ? removalInfo.localRemovedSeq! : removalInfo.removedSeq;
682
- segmentLen = -segmentLen;
683
686
  // The client who performed the remove is always stored
684
687
  // in the first position of removalInfo.
685
688
  clientId = removalInfo.removedClientIds[0];
686
- const hasOverlap = removalInfo.removedClientIds.length > 1;
687
- removeClientOverlap = hasOverlap ? removalInfo.removedClientIds : undefined;
688
- } else if (moveInfo) {
689
+ } else {
690
+ assert(
691
+ moveInfo !== undefined,
692
+ 0xab9 /* Expected move to exist if remove either did not exist or didn't happen first */,
693
+ );
689
694
  // The client who performed the move is always stored
690
695
  // in the first position of moveInfo.
691
696
  clientId = moveInfo.movedClientIds[0];
692
697
 
693
698
  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
694
699
  seqOrLocalSeq = moveIsLocal ? moveInfo.localMovedSeq! : moveInfo.movedSeq;
695
-
696
- if (moveInfo.wasMovedOnInsert) {
697
- assert(
698
- moveInfo.movedSeq !== -1,
699
- 0x871 /* wasMovedOnInsert should only be set on acked obliterates */,
700
- );
701
- segmentLen = 0;
702
- } else {
703
- segmentLen = -segmentLen;
704
- }
705
-
706
- const hasOverlap = moveInfo.movedClientIds.length > 1;
707
- moveClientOverlap = hasOverlap ? moveInfo.movedClientIds : undefined;
708
- } // BUG BUG: something fishy here around how/when move info is passed or not
709
- // this condition only hits if it is not passed, so we can't rely on the passed move info
710
- // and need to inspect the segment directly. maybe related to AB#15630.
711
- else if (toMoveInfo(segment)?.wasMovedOnInsert) {
712
- // if this segment was obliterated on insert, its length is only
713
- // visible to the client that inserted it
714
- segmentLen = 0;
715
- remoteObliteratedLen = segment.cachedLength;
716
700
  }
717
701
 
718
702
  const partials = isLocal
@@ -723,199 +707,92 @@ export class PartialSequenceLengths {
723
707
  return;
724
708
  }
725
709
 
726
- // overlapping move and remove, remove happened first
727
- if (moveInfo && removalInfo && removeHappenedFirst && !moveIsLocal) {
728
- // The client who performed the remove is always stored
729
- // in the first position of removalInfo.
730
- const moveClientId = moveInfo.movedClientIds[0];
731
- const hasOverlap = moveInfo.movedClientIds.length > 1;
732
-
733
- PartialSequenceLengths.updatePartialsAfterInsertion(
734
- segment,
735
- 0,
736
- -segment.cachedLength,
737
- segmentLen,
738
- partials,
739
- moveInfo.movedSeq,
740
- moveClientId,
741
- undefined,
742
- hasOverlap ? moveInfo.movedClientIds : undefined,
743
- );
744
- }
745
-
746
- if (removalInfo && !removeHappenedFirst && !removalIsLocal) {
747
- const removeSeqOrLocalSeq = removalIsLocal
748
- ? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
749
- removalInfo.localRemovedSeq!
750
- : removalInfo.removedSeq;
751
- // The client who performed the remove is always stored
752
- // in the first position of removalInfo.
753
- const removeClientId = removalInfo.removedClientIds[0];
754
- const hasOverlap = removalInfo.removedClientIds.length > 1;
755
-
756
- PartialSequenceLengths.updatePartialsAfterInsertion(
757
- segment,
758
- 0,
759
- -segment.cachedLength,
760
- segmentLen,
761
- partials,
762
- removeSeqOrLocalSeq,
763
- removeClientId,
764
- hasOverlap ? removalInfo.removedClientIds : undefined,
765
- undefined,
766
- );
767
- }
768
-
769
- PartialSequenceLengths.updatePartialsAfterInsertion(
770
- segment,
771
- segmentLen,
772
- remoteObliteratedLen,
773
- undefined,
774
- partials,
775
- seqOrLocalSeq,
776
- clientId,
777
- removeClientOverlap,
778
- moveClientOverlap,
779
- );
780
-
781
- // todo: the below block needs to be changed to handle obliterate, which
782
- // doesn't have great support for reconnect at the moment. see ADO #3714
783
- const { unsequencedRecords } = combinedPartialLengths;
784
- if (
785
- unsequencedRecords &&
786
- removeClientOverlap &&
787
- isRemoved(segment) &&
788
- segment.localRemovedSeq !== undefined
789
- ) {
790
- const localSeq = segment.localRemovedSeq;
791
- const localPartialLengthEntry: LocalPartialSequenceLength = {
710
+ if (isLocal) {
711
+ // The segment is either inserted only locally or removed/moved only locally.
712
+ // We already accounted for the insertion in the accountForInsertion codepath.
713
+ // Only thing left to do is account for the removal.
714
+ partials.addOrUpdate({
792
715
  seq: seqOrLocalSeq,
793
- localSeq,
794
716
  clientId,
795
717
  len: 0,
796
- seglen: segmentLen,
797
- };
798
- let localIndexFirstGTE = 0;
799
- for (
800
- ;
801
- localIndexFirstGTE < unsequencedRecords.overlappingRemoves.length;
802
- localIndexFirstGTE++
803
- ) {
804
- if (unsequencedRecords.overlappingRemoves[localIndexFirstGTE].seq >= seqOrLocalSeq) {
805
- break;
806
- }
807
- }
808
-
809
- insertIntoList(
810
- unsequencedRecords.overlappingRemoves,
811
- localIndexFirstGTE,
812
- localPartialLengthEntry,
813
- );
814
-
815
- const tweakedLocalPartialEntry = {
816
- ...localPartialLengthEntry,
817
- seq: localSeq,
818
- };
718
+ seglen: lenDelta,
719
+ });
720
+ } else {
721
+ partials.addOrUpdate({
722
+ seq: seqOrLocalSeq,
723
+ clientId,
724
+ len: 0,
725
+ seglen: lenDelta,
726
+ });
819
727
 
820
- unsequencedRecords.partialLengths.addOrUpdate(tweakedLocalPartialEntry);
821
- }
822
- }
728
+ for (const id of clientsWithRemoveOrObliterate) {
729
+ if (id === collabWindow.clientId) {
730
+ // The local client also removed or obliterated this segment.
731
+ const localSeq = moveInfo?.localMovedSeq ?? removalInfo?.localRemovedSeq;
732
+ if (localSeq === undefined) {
733
+ // Sure, the local client did it--but that change was already acked.
734
+ // No need to account for it in the unsequenced records.
735
+ continue;
736
+ }
737
+ const { unsequencedRecords } = combinedPartialLengths;
738
+ if (!unsequencedRecords) {
739
+ // Local partial but its computation isn't required.
740
+ continue;
741
+ }
742
+ assert(
743
+ localSeq !== undefined,
744
+ 0xaba /* Local client was in move/removed client ids but segment has no local seq for either */,
745
+ );
823
746
 
824
- private static addSeq(
825
- partialLengths: PartialSequenceLengthsSet,
826
- seq: number,
827
- seqSeglen: number,
828
- remoteObliteratedLen?: number,
829
- clientId?: number,
830
- ): void {
831
- let seqPartialLen: PartialSequenceLength | undefined;
832
- let penultPartialLen: PartialSequenceLength | undefined;
833
- let pLen = partialLengths.latestLeq(seq);
834
- if (pLen) {
835
- if (pLen.seq === seq) {
836
- seqPartialLen = pLen;
837
- pLen = partialLengths.latestLeq(seq - 1);
838
- if (pLen) {
839
- penultPartialLen = pLen;
747
+ unsequencedRecords.partialLengths.addOrUpdate({
748
+ seq: localSeq,
749
+ clientId: collabWindow.clientId,
750
+ seglen: lenDelta,
751
+ len: 0,
752
+ });
753
+
754
+ // Because we've included deltas which take effect when either of localSeq or refSeq are high enough,
755
+ // we need to offset this with an adjustment that takes effect when both are high enough.
756
+ combinedPartialLengths.addLocalAdjustment({
757
+ refSeq: seqOrLocalSeq,
758
+ localSeq,
759
+ // combinedPartialLengths.partialLengths has an entry removing this segment from a perspective >= seqOrLocalSeq.
760
+ // combinedPartialLengths.unsequencedRecords.partialLengths now has an entry removing this segment from a perspective
761
+ // with local seq >= `localSeq`.
762
+ // In order to only remove this segment once, we add back in the length (where this entry only takes effect when
763
+ // both above are true due to logic in computeOverallRefSeqAdjustment).
764
+ seglen: segment.cachedLength,
765
+ });
766
+ } else {
767
+ // Note that all clients that have a remove or obliterate operation on this segment
768
+ // use the seq of the winning move/obliterate in their per-client adjustments!
769
+ combinedPartialLengths.addClientAdjustment(id, seqOrLocalSeq, lenDelta);
770
+
771
+ // Also ensure that all these clients have seen the segment as inserted before being removed
772
+ // This is technically not necessary for removes (we never ask for the length of this block with
773
+ // respect to a refSeq which this entry would affect), but it's simpler to just add it here.
774
+ // We already add this entry as part of the accountForInsertion codepath for the client that
775
+ // actually did insert the segment, hence not doing so [again] here.
776
+ if (segment.seq > collabWindow.minSeq && id !== segment.clientId) {
777
+ combinedPartialLengths.addClientAdjustment(id, segment.seq, segment.cachedLength);
778
+ }
840
779
  }
841
- } else {
842
- penultPartialLen = pLen;
843
780
  }
844
781
  }
845
- const len = penultPartialLen === undefined ? seqSeglen : penultPartialLen.len + seqSeglen;
846
- if (seqPartialLen === undefined) {
847
- seqPartialLen = {
848
- clientId,
849
- len,
850
- seglen: seqSeglen,
851
- seq,
852
- remoteObliteratedLen,
853
- };
854
- partialLengths.addOrUpdate(seqPartialLen);
855
- } else {
856
- seqPartialLen.remoteObliteratedLen = remoteObliteratedLen;
857
- seqPartialLen.seglen = seqSeglen;
858
- seqPartialLen.len = len;
859
- // Assert client id matches
860
- }
861
782
  }
862
783
 
863
784
  /**
864
- * Length of the block this PartialSequenceLength corresponds to when viewed at `minSeq`.
865
- */
866
- private minLength = 0;
867
-
868
- /**
869
- * Total number of segments in the subtree rooted at the block this PartialSequenceLength corresponds to.
785
+ * If incremental update of partial lengths fails, this gets set to the seq of the failed update.
786
+ * When higher up blocks attempt to incrementally update, they first check if the seq they are updating for
787
+ * matches this value. If it does, they propagate a full refresh instead.
870
788
  */
871
- private segmentCount = 0;
872
-
873
- /**
874
- * List of PartialSequenceLength objects--ordered by increasing seq--giving length information about
875
- * the block associated with this PartialSequenceLengths object.
876
- *
877
- * `partialLengths[i].len` contains the length of this block considering only sequenced segments with
878
- * `sequenceNumber <= partialLengths[i].seq`.
879
- */
880
- private readonly partialLengths: PartialSequenceLengthsSet = new PartialSequenceLengthsSet();
881
-
882
- /**
883
- * clientSeqNumbers[clientId] is a list of partial lengths for sequenced ops which either:
884
- * - were submitted by `clientId`.
885
- * - deleted a range containing segments that were concurrently deleted by `clientId`
886
- *
887
- * The second case is referred to as the "overlapping delete" case. It is necessary to avoid double-counting
888
- * the removal of those segments in queries including clientId.
889
- */
890
- private readonly clientSeqNumbers: PartialSequenceLengthsSet[] = [];
891
-
892
- /**
893
- * Contains information required to answer queries for the length of this segment from the perspective of
894
- * the local client but not including all local segments (i.e., `localSeq !== collabWindow.localSeq`).
895
- * This field is only computed if requested in the constructor (i.e. `computeLocalPartials === true`).
896
- */
897
- private unsequencedRecords: UnsequencedPartialLengthInfo | undefined;
898
-
899
- constructor(
900
- /**
901
- * The minimumSequenceNumber as defined by the collab window used in the last call to `update`,
902
- * or if no such calls have been made, the one used on construction.
903
- */
904
- public minSeq: number,
905
- computeLocalPartials: boolean,
906
- ) {
907
- if (computeLocalPartials) {
908
- this.unsequencedRecords = {
909
- partialLengths: new PartialSequenceLengthsSet(),
910
- overlappingRemoves: [],
911
- cachedOverlappingByRefSeq: new Map(),
912
- };
913
- }
914
- }
789
+ private lastIncrementalInvalidationSeq = Number.NEGATIVE_INFINITY;
915
790
 
916
- // Assume: seq is latest sequence number; no structural change to sub-tree, but a segment
917
- // with sequence number seq has been added within the sub-tree (and `update` has been called
918
- // on all descendant PartialSequenceLengths)
791
+ // Assume: seq is latest sequence number; no structural change to sub-tree, but this partial lengths
792
+ // entry needs to account for the change made by the client with `clientId` at sequence number `seq`.
793
+ // (and `update` has been called on all descendant PartialSequenceLengths).
794
+ // This implementation does not support overlapping removes: callers should recompute partial lengths
795
+ // using `combine` when the change that has just been applied involves such an operation.
919
796
  // TODO: assert client id matches
920
797
  public update(
921
798
  node: MergeBlock,
@@ -924,8 +801,33 @@ export class PartialSequenceLengths {
924
801
 
925
802
  collabWindow: CollaborationWindow,
926
803
  ): void {
804
+ // In the current implementation, this method gets invoked multiple times for the same sequence number (i.e. mid-operation).
805
+ // We counter this by first zeroing out existing entries from previous updates, but it isn't ideal.
806
+ // Even if we fix this at the merge-tree level, the same type of issue can crop up with grouped batching enabled.
807
+ const latest = this.partialLengths.latestLeq(seq);
808
+ if (latest?.seq === seq) {
809
+ this.partialLengths.addOrUpdate({ seq, len: 0, seglen: -latest.seglen, clientId });
810
+ }
811
+
812
+ // .forEach natively ignores undefined entries.
813
+ // eslint-disable-next-line unicorn/no-array-for-each
814
+ this.perClientAdjustments.forEach((clientAdjustments) => {
815
+ const leqPartial = clientAdjustments.latestLeq(seq);
816
+ if (leqPartial && leqPartial.seq === seq) {
817
+ this.addClientAdjustment(clientId, seq, -leqPartial.seglen);
818
+ }
819
+ });
820
+
821
+ /**
822
+ * If any of the changes made by the client at `seq` necessitate partial length entries at sequence numbers other than `seq`,
823
+ * this flag is set to true. This propagates upwards when aggregating parents as well.
824
+ *
825
+ * Note: it seems feasible to update parents more incrementally by tracking the changes made to child blocks for a given update.
826
+ * There isn't a great place for this information to flow today.
827
+ */
828
+ let failIncrementalPropagation = false;
829
+
927
830
  let seqSeglen = 0;
928
- let remoteObliteratedLen = 0;
929
831
  let segCount = 0;
930
832
  // Compute length for seq across children
931
833
  for (let i = 0; i < node.childCount; i++) {
@@ -934,17 +836,6 @@ export class PartialSequenceLengths {
934
836
  const segment = child;
935
837
  const removalInfo = toRemovalInfo(segment);
936
838
  const moveInfo = toMoveInfo(segment);
937
-
938
- const removalIsLocal =
939
- !!removalInfo && removalInfo.removedSeq === UnassignedSequenceNumber;
940
- const moveIsLocal = !!moveInfo && moveInfo.movedSeq === UnassignedSequenceNumber;
941
-
942
- const removeHappenedFirst =
943
- removalInfo &&
944
- (!moveInfo ||
945
- moveIsLocal ||
946
- (!removalIsLocal && moveInfo.movedSeq > removalInfo.removedSeq));
947
-
948
839
  if (seq === segment.seq) {
949
840
  // if this segment was moved on insert, its length should
950
841
  // only be visible to the inserting client
@@ -954,74 +845,69 @@ export class PartialSequenceLengths {
954
845
  moveInfo.movedSeq < segment.seq &&
955
846
  moveInfo.wasMovedOnInsert
956
847
  ) {
957
- remoteObliteratedLen += segment.cachedLength;
848
+ this.addClientAdjustment(clientId, moveInfo.movedSeq, segment.cachedLength);
849
+ failIncrementalPropagation = true;
958
850
  } else {
959
851
  seqSeglen += segment.cachedLength;
852
+ this.addClientAdjustment(clientId, seq, segment.cachedLength);
960
853
  }
961
854
  }
962
855
 
963
- if (seq === removalInfo?.removedSeq) {
964
- // if the remove op happened before an overlapping obliterate,
965
- // all clients can see the remove at this seq. otherwise, only
966
- // the removing client is aware of the remove
967
- if (removeHappenedFirst) {
968
- seqSeglen -= segment.cachedLength;
969
- } else {
970
- remoteObliteratedLen -= segment.cachedLength;
856
+ const earlierDeletion = Math.min(
857
+ removalInfo?.removedSeq ?? Number.MAX_VALUE,
858
+ moveInfo?.movedSeq ?? Number.MAX_VALUE,
859
+ );
860
+ if (segment.seq !== UnassignedSequenceNumber && seq === earlierDeletion) {
861
+ seqSeglen -= segment.cachedLength;
862
+ if (clientId !== collabWindow.clientId) {
863
+ this.addClientAdjustment(clientId, seq, -segment.cachedLength);
864
+ if (segment.seq > collabWindow.minSeq && segment.clientId !== clientId) {
865
+ this.addClientAdjustment(clientId, segment.seq, segment.cachedLength);
866
+ failIncrementalPropagation = true;
867
+ }
971
868
  }
972
869
  }
973
870
 
974
- if (seq === moveInfo?.movedSeq) {
975
- if (removeHappenedFirst) {
976
- remoteObliteratedLen -= segment.cachedLength;
977
- } else if (
978
- moveInfo.wasMovedOnInsert &&
979
- segment.seq !== UnassignedSequenceNumber &&
980
- segment.seq !== undefined &&
981
- moveInfo.movedSeq > segment.seq
982
- ) {
983
- remoteObliteratedLen += segment.cachedLength;
984
- seqSeglen -= segment.cachedLength;
985
- } else if (segment.seq !== UnassignedSequenceNumber) {
986
- seqSeglen -= segment.cachedLength;
987
- }
988
- }
989
871
  segCount++;
990
872
  } else {
991
873
  const childBlock = child;
992
874
  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
993
875
  const branchPartialLengths = childBlock.partialLengths!;
876
+ if (branchPartialLengths.lastIncrementalInvalidationSeq === seq) {
877
+ // Bail out.
878
+ const newPartials = PartialSequenceLengths.combine(node, collabWindow, false);
879
+ newPartials.lastIncrementalInvalidationSeq = seq;
880
+ node.partialLengths = newPartials;
881
+ return;
882
+ }
994
883
  const partialLengths = branchPartialLengths.partialLengths;
995
884
  const leqPartial = partialLengths.latestLeq(seq);
996
885
  if (leqPartial && leqPartial.seq === seq) {
997
886
  seqSeglen += leqPartial.seglen;
998
- remoteObliteratedLen += leqPartial.remoteObliteratedLen ?? 0;
999
887
  }
1000
888
  segCount += branchPartialLengths.segmentCount;
889
+
890
+ // .forEach natively ignores undefined entries.
891
+ // eslint-disable-next-line unicorn/no-array-for-each
892
+ branchPartialLengths.perClientAdjustments.forEach((clientAdjustments) => {
893
+ const leqBranchPartial = clientAdjustments.latestLeq(seq);
894
+ if (leqBranchPartial && leqBranchPartial.seq === seq) {
895
+ this.addClientAdjustment(clientId, seq, leqBranchPartial.seglen);
896
+ }
897
+ });
1001
898
  }
1002
899
  }
900
+
901
+ if (failIncrementalPropagation) {
902
+ this.lastIncrementalInvalidationSeq = seq;
903
+ }
1003
904
  this.segmentCount = segCount;
1004
905
  this.unsequencedRecords = undefined;
906
+ this.partialLengths.addOrUpdate({ seq, seglen: seqSeglen, len: 0, clientId });
1005
907
 
1006
- PartialSequenceLengths.addSeq(
1007
- this.partialLengths,
1008
- seq,
1009
- seqSeglen,
1010
- remoteObliteratedLen,
1011
- clientId,
1012
- );
1013
- this.clientSeqNumbers[clientId] ??= new PartialSequenceLengthsSet();
1014
- PartialSequenceLengths.addSeq(
1015
- this.clientSeqNumbers[clientId],
1016
- seq,
1017
- seqSeglen + remoteObliteratedLen,
1018
- undefined,
1019
- clientId,
1020
- );
1021
908
  if (PartialSequenceLengths.options.zamboni) {
1022
909
  this.zamboni(collabWindow);
1023
910
  }
1024
-
1025
911
  PartialSequenceLengths.options.verifier?.(this);
1026
912
  }
1027
913
 
@@ -1036,23 +922,19 @@ export class PartialSequenceLengths {
1036
922
  * constructed with `computeLocalPartials` set to true and not subsequently updated with `update`.
1037
923
  */
1038
924
  public getPartialLength(refSeq: number, clientId: number, localSeq?: number): number {
1039
- let pLen = this.minLength;
1040
- const cliLatestIndex = this.cliLatest(clientId);
1041
- const cliSeq = this.clientSeqNumbers[clientId];
1042
- pLen += this.partialLengths.latestLeq(refSeq)?.len ?? 0;
925
+ let length = this.minLength;
926
+ length += this.partialLengths.latestLeq(refSeq)?.len ?? 0;
1043
927
 
1044
928
  if (localSeq === undefined) {
1045
- if (cliLatestIndex >= 0) {
1046
- const cliLatest = cliSeq.items[cliLatestIndex];
1047
- if (cliLatest.seq > refSeq) {
1048
- // The client has local edits after refSeq, add in the length adjustments
1049
- pLen += cliLatest.len;
1050
- const precedingCli = this.cliLatestLEQ(clientId, refSeq);
1051
- if (precedingCli) {
1052
- // Subtract out double-counted lengths: segments still in the collab window but before
1053
- // the refSeq submitted by the client we're querying for were counted in each addition above.
1054
- pLen -= precedingCli.len;
1055
- }
929
+ const latestClientEntry = this.latestClientEntry(clientId);
930
+ if (latestClientEntry !== undefined && latestClientEntry.seq > refSeq) {
931
+ // The client has local edits after refSeq, add in the length adjustments
932
+ length += latestClientEntry.len;
933
+ const precedingCli = this.latestClientEntryLEQ(clientId, refSeq);
934
+ if (precedingCli) {
935
+ // Subtract out double-counted lengths: segments still in the collab window but before
936
+ // the refSeq submitted by the client we're querying for were counted in each addition above.
937
+ length -= precedingCli.len;
1056
938
  }
1057
939
  }
1058
940
  } else {
@@ -1064,34 +946,18 @@ export class PartialSequenceLengths {
1064
946
  // Local segments at or before localSeq should also be included
1065
947
  const local = unsequencedPartialLengths.latestLeq(localSeq);
1066
948
  if (local) {
1067
- pLen += local.len;
949
+ length += local.len;
1068
950
 
1069
- // Lastly, we must subtract out any double-counted removes, which occur if a currently un-acked local
1070
- // remove overlaps with a remote client's remove that occurred at sequence number <=refSeq.
1071
- pLen -= this.computeOverlappingLocalRemoves(refSeq, localSeq);
951
+ // Lastly, we must add in any additional adjustment due to double-counting removes and obliterations
952
+ // removing local-only segments.
953
+ length += this.computeOverallRefSeqAdjustment(refSeq, localSeq);
1072
954
  }
1073
955
  }
1074
- return pLen;
956
+ return length;
1075
957
  }
1076
958
 
1077
959
  /**
1078
- * Computes the seglen for the double-counted removed overlap at (refSeq, localSeq). This logic is equivalent
1079
- * to the following:
1080
- *
1081
- * ```typescript
1082
- * let total = 0;
1083
- * for (const partialLength of this.unsequencedRecords!.overlappingRemoves) {
1084
- * if (partialLength.seq > refSeq) {
1085
- * break;
1086
- * }
1087
- *
1088
- * if (partialLength.localSeq <= localSeq) {
1089
- * total += partialLength.seglen;
1090
- * }
1091
- * }
1092
- *
1093
- * return total;
1094
- * ```
960
+ * Computes the seglen for the double-counted removed overlap at (refSeq, localSeq).
1095
961
  *
1096
962
  * Reconnect happens to only need to compute these lengths for two refSeq values: before and
1097
963
  * after the rebase. Since these lists potentially scale with O(collab window * number of local edits)
@@ -1099,27 +965,35 @@ export class PartialSequenceLengths {
1099
965
  * we cache the results for a given refSeq in `this.unsequencedRecords.cachedOverlappingByRefSeq` so
1100
966
  * that they can be binary-searched the same way the usual partialLengths lists are.
1101
967
  */
1102
- private computeOverlappingLocalRemoves(refSeq: number, localSeq: number): number {
968
+ private computeOverallRefSeqAdjustment(refSeq: number, localSeq: number): number {
1103
969
  if (this.unsequencedRecords === undefined) {
1104
970
  return 0;
1105
971
  }
1106
972
 
1107
- let cachedOverlapPartials = this.unsequencedRecords.cachedOverlappingByRefSeq.get(refSeq);
1108
- if (!cachedOverlapPartials) {
973
+ let cachedAdjustment = this.unsequencedRecords.cachedAdjustmentByRefSeq.get(refSeq);
974
+ if (!cachedAdjustment) {
1109
975
  const partials: PartialSequenceLengthsSet = new PartialSequenceLengthsSet();
1110
- for (const partial of this.unsequencedRecords.overlappingRemoves) {
1111
- if (partial.seq > refSeq) {
1112
- break;
976
+ for (const [
977
+ seq,
978
+ adjustments,
979
+ ] of this.unsequencedRecords.perRefSeqAdjustments.entries()) {
980
+ if (seq > refSeq) {
981
+ // TODO: Prior code path got away with an early exit here by sorting the entries by refSeq.
982
+ // We could do the same here if we wanted.
983
+ // Old codepath basically flattened the 2d array into a 1d array with both dimensions listed.
984
+ continue;
1113
985
  }
1114
986
 
1115
- partials.addOrUpdate({ ...partial, seq: partial.localSeq, len: 0 });
987
+ for (const partial of adjustments.items) {
988
+ // This coalesces entries with the same localSeq as well as computes overall lengths.
989
+ partials.addOrUpdate({ ...partial });
990
+ }
1116
991
  }
1117
- // This coalesces entries with the same localSeq as well as computes overall lengths.
1118
- cachedOverlapPartials = partials;
1119
- this.unsequencedRecords.cachedOverlappingByRefSeq.set(refSeq, cachedOverlapPartials);
992
+ cachedAdjustment = partials;
993
+ this.unsequencedRecords.cachedAdjustmentByRefSeq.set(refSeq, cachedAdjustment);
1120
994
  }
1121
995
 
1122
- const overlap = cachedOverlapPartials.latestLeq(localSeq);
996
+ const overlap = cachedAdjustment.latestLeq(localSeq);
1123
997
  return overlap?.len ?? 0;
1124
998
  }
1125
999
 
@@ -1130,12 +1004,12 @@ export class PartialSequenceLengths {
1130
1004
  }
1131
1005
 
1132
1006
  // eslint-disable-next-line @typescript-eslint/no-for-in-array, no-restricted-syntax
1133
- for (const clientId in this.clientSeqNumbers) {
1134
- if (this.clientSeqNumbers[clientId].size > 0) {
1007
+ for (const clientId in this.perClientAdjustments) {
1008
+ if (this.perClientAdjustments[clientId].size > 0) {
1135
1009
  buf += `Client `;
1136
1010
  buf += glc ? `${glc(+clientId)}` : `${clientId}`;
1137
1011
  buf += "[";
1138
- for (const partial of this.clientSeqNumbers[clientId].items) {
1012
+ for (const partial of this.perClientAdjustments[clientId].items) {
1139
1013
  buf += `(${partial.seq},${partial.len})`;
1140
1014
  }
1141
1015
  buf += "]";
@@ -1151,52 +1025,54 @@ export class PartialSequenceLengths {
1151
1025
  this.minLength += this.partialLengths.copyDown(segmentWindow.minSeq);
1152
1026
  this.minSeq = segmentWindow.minSeq;
1153
1027
  // eslint-disable-next-line @typescript-eslint/no-for-in-array, guard-for-in, no-restricted-syntax
1154
- for (const clientId in this.clientSeqNumbers) {
1155
- const cliPartials = this.clientSeqNumbers[clientId];
1028
+ for (const clientId in this.perClientAdjustments) {
1029
+ const cliPartials = this.perClientAdjustments[clientId];
1156
1030
  if (cliPartials) {
1157
1031
  cliPartials.copyDown(segmentWindow.minSeq);
1158
1032
  }
1159
1033
  }
1160
1034
  }
1161
1035
 
1162
- private addClientSeqNumber(clientId: number, seq: number, seglen: number): void {
1163
- this.clientSeqNumbers[clientId] ??= new PartialSequenceLengthsSet();
1164
- const cli = this.clientSeqNumbers[clientId];
1036
+ private addClientAdjustment(clientId: number, seq: number, seglen: number): void {
1037
+ this.perClientAdjustments[clientId] ??= new PartialSequenceLengthsSet();
1038
+ const cli = this.perClientAdjustments[clientId];
1165
1039
  cli.addOrUpdate({ seq, len: 0, seglen });
1166
1040
  }
1167
1041
 
1168
- // Assumes sequence number already coalesced and that this is called in increasing `seq` order.
1169
- private addClientSeqNumberFromPartial(partialLength: PartialSequenceLength): void {
1170
- const seglen = partialLength.seglen + (partialLength.remoteObliteratedLen ?? 0);
1171
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
1172
- this.addClientSeqNumber(partialLength.clientId!, partialLength.seq, seglen);
1173
- if (partialLength.overlapRemoveClients) {
1174
- partialLength.overlapRemoveClients.map((oc: Property<number, IOverlapClient>) => {
1175
- // Original client entry was handled above
1176
- if (partialLength.clientId !== oc.data.clientId) {
1177
- this.addClientSeqNumber(oc.data.clientId, partialLength.seq, oc.data.seglen);
1178
- }
1179
- return true;
1180
- });
1181
- }
1182
- if (partialLength.overlapObliterateClients) {
1183
- partialLength.overlapObliterateClients.map((oc: Property<number, IOverlapClient>) => {
1184
- // Original client entry was handled above
1185
- if (partialLength.clientId !== oc.data.clientId) {
1186
- this.addClientSeqNumber(oc.data.clientId, partialLength.seq, oc.data.seglen);
1187
- }
1188
- return true;
1189
- });
1190
- }
1042
+ private addLocalAdjustment({
1043
+ refSeq,
1044
+ localSeq,
1045
+ seglen,
1046
+ }: { refSeq: number; localSeq: number; seglen: number }): void {
1047
+ assert(
1048
+ this.unsequencedRecords !== undefined,
1049
+ 0xabb /* Local adjustment computed without partials */,
1050
+ );
1051
+ const adjustments =
1052
+ this.unsequencedRecords.perRefSeqAdjustments.get(refSeq) ??
1053
+ new PartialSequenceLengthsSet();
1054
+ this.unsequencedRecords.perRefSeqAdjustments.set(refSeq, adjustments);
1055
+ adjustments.addOrUpdate({ seq: localSeq, len: 0, seglen });
1191
1056
  }
1192
1057
 
1193
- private cliLatestLEQ(clientId: number, refSeq: number): PartialSequenceLength | undefined {
1194
- return this.clientSeqNumbers[clientId]?.latestLeq(refSeq);
1058
+ /**
1059
+ * Returns the partial lengths associated with the latest change associated with `clientId` at or before `refSeq`.
1060
+ * Returns undefined if no such change exists.
1061
+ */
1062
+ private latestClientEntryLEQ(
1063
+ clientId: number,
1064
+ refSeq: number,
1065
+ ): PartialSequenceLength | undefined {
1066
+ return this.perClientAdjustments[clientId]?.latestLeq(refSeq);
1195
1067
  }
1196
1068
 
1197
- private cliLatest(clientId: number): number {
1198
- const cliSeqs = this.clientSeqNumbers[clientId];
1199
- return cliSeqs && cliSeqs.size > 0 ? cliSeqs.size - 1 : -1;
1069
+ /**
1070
+ * Get the partial lengths associated with the most recent change received by `clientId`, or undefined
1071
+ * if this client has made no changes in this block within the collab window.
1072
+ */
1073
+ private latestClientEntry(clientId: number): PartialSequenceLength | undefined {
1074
+ const cliSeqs = this.perClientAdjustments[clientId];
1075
+ return cliSeqs && cliSeqs.size > 0 ? cliSeqs.items[cliSeqs.size - 1] : undefined;
1200
1076
  }
1201
1077
  }
1202
1078
 
@@ -1254,30 +1130,6 @@ function verifyPartialLengthsInner(
1254
1130
  assert(false, 0x057 /* "Negative length after length adjustment!" */);
1255
1131
  }
1256
1132
  }
1257
-
1258
- if (partialLength.overlapRemoveClients) {
1259
- // Only the flat partialLengths can have overlapRemoveClients, the per client view shouldn't
1260
- assert(
1261
- !clientPartials,
1262
- 0x058 /* "Both overlapRemoveClients and clientPartials are set!" */,
1263
- );
1264
-
1265
- // Each overlap client counts as one, but the first remove to sequence was already counted.
1266
- // (this aligns with the logic to omit the removing client in `addClientSeqNumberFromPartial`)
1267
- count += partialLength.overlapRemoveClients.size() - 1;
1268
- }
1269
-
1270
- if (partialLength.overlapObliterateClients) {
1271
- // Only the flat partialLengths can have overlapObliterateClients, the per client view shouldn't
1272
- assert(
1273
- !clientPartials,
1274
- 0x872 /* Both overlapObliterateClients and clientPartials are set! */,
1275
- );
1276
-
1277
- // Each overlap client counts as one, but the first move to sequence was already counted.
1278
- // (this aligns with the logic to omit the moving client in `addClientSeqNumberFromPartial`)
1279
- count += partialLength.overlapObliterateClients.size() - 1;
1280
- }
1281
1133
  }
1282
1134
  return count;
1283
1135
  }
@@ -1314,16 +1166,28 @@ export function verifyExpectedPartialLengths(
1314
1166
  }
1315
1167
 
1316
1168
  if (expected !== partialLen) {
1169
+ const nonIncrementalPartials = PartialSequenceLengths.combine(
1170
+ node,
1171
+ mergeTree.collabWindow,
1172
+ false,
1173
+ true,
1174
+ );
1175
+ const nonIncrementalLength = nonIncrementalPartials.getPartialLength(
1176
+ refSeq,
1177
+ clientId,
1178
+ localSeq,
1179
+ );
1317
1180
  node.partialLengths?.getPartialLength(refSeq, clientId, localSeq);
1181
+
1318
1182
  throw new Error(
1319
- `expected partial length of ${expected} but found ${partialLen}. refSeq: ${refSeq}, clientId: ${clientId}`,
1183
+ `expected partial length of ${expected} but found ${partialLen}. refSeq: ${refSeq}, clientId: ${clientId}. (non-incremental codepath returned ${nonIncrementalLength})`,
1320
1184
  );
1321
1185
  }
1322
1186
  }
1323
1187
 
1324
1188
  export function verifyPartialLengths(partialSeqLengths: PartialSequenceLengths): void {
1325
- if (partialSeqLengths["clientSeqNumbers"]) {
1326
- for (const cliSeq of partialSeqLengths["clientSeqNumbers"]) {
1189
+ if (partialSeqLengths["perClientAdjustments"]) {
1190
+ for (const cliSeq of partialSeqLengths["perClientAdjustments"]) {
1327
1191
  if (cliSeq) {
1328
1192
  verifyPartialLengthsInner(partialSeqLengths, cliSeq, true);
1329
1193
  }
@@ -1346,72 +1210,6 @@ export function verifyPartialLengths(partialSeqLengths: PartialSequenceLengths):
1346
1210
  }
1347
1211
  /* eslint-enable @typescript-eslint/dot-notation */
1348
1212
 
1349
- /**
1350
- * Clones an `overlapRemoveClients` red-black tree.
1351
- */
1352
- function cloneOverlapRemoveClients(
1353
- oldTree: RedBlackTree<number, IOverlapClient> | undefined,
1354
- ): RedBlackTree<number, IOverlapClient> | undefined {
1355
- if (!oldTree) {
1356
- return undefined;
1357
- }
1358
- const newTree = new RedBlackTree<number, IOverlapClient>(compareNumbers);
1359
- oldTree.map((bProp: Property<number, IOverlapClient>) => {
1360
- newTree.put(bProp.data.clientId, { ...bProp.data });
1361
- return true;
1362
- });
1363
- return newTree;
1364
- }
1365
-
1366
- function combineForOverlapClients(
1367
- treeA: RedBlackTree<number, IOverlapClient> | undefined,
1368
- treeB: RedBlackTree<number, IOverlapClient> | undefined,
1369
- ): RedBlackTree<number, IOverlapClient> | undefined {
1370
- if (treeA) {
1371
- if (treeB) {
1372
- treeB.map((bProp: Property<number, IOverlapClient>) => {
1373
- const aProp = treeA.get(bProp.key);
1374
- if (aProp) {
1375
- aProp.data.seglen += bProp.data.seglen;
1376
- } else {
1377
- treeA.put(bProp.data.clientId, { ...bProp.data });
1378
- }
1379
- return true;
1380
- });
1381
- }
1382
- } else {
1383
- return cloneOverlapRemoveClients(treeB);
1384
- }
1385
- }
1386
-
1387
- /**
1388
- * Combines the `overlapRemoveClients` and `overlapObliterateClients` fields of
1389
- * two `PartialSequenceLength` objects, modifying the first PartialSequenceLength's
1390
- * bookkeeping in-place.
1391
- *
1392
- * Combination is performed additively on `seglen` on a per-client basis.
1393
- */
1394
- export function combineOverlapClients(
1395
- a: PartialSequenceLength,
1396
- b: PartialSequenceLength,
1397
- ): void {
1398
- const overlapRemoveClients = combineForOverlapClients(
1399
- a.overlapRemoveClients,
1400
- b.overlapRemoveClients,
1401
- );
1402
- if (overlapRemoveClients) {
1403
- a.overlapRemoveClients = overlapRemoveClients;
1404
- }
1405
-
1406
- const overlapObliterateClients = combineForOverlapClients(
1407
- a.overlapObliterateClients,
1408
- b.overlapObliterateClients,
1409
- );
1410
- if (overlapObliterateClients) {
1411
- a.overlapObliterateClients = overlapObliterateClients;
1412
- }
1413
- }
1414
-
1415
1213
  /**
1416
1214
  * Given a number of seq-sorted `partialLength` lists, merges them into a combined seq-sorted `partialLength`
1417
1215
  * list. This merge includes coalescing `PartialSequenceLength` entries at the same seq.
@@ -1433,10 +1231,6 @@ function mergePartialLengths(
1433
1231
  for (const partialLength of mergeSortedListsBySeq(childPartialLengths)) {
1434
1232
  mergedLengths.addOrUpdate({
1435
1233
  ...partialLength,
1436
- overlapRemoveClients: cloneOverlapRemoveClients(partialLength.overlapRemoveClients),
1437
- overlapObliterateClients: cloneOverlapRemoveClients(
1438
- partialLength.overlapObliterateClients,
1439
- ),
1440
1234
  });
1441
1235
  }
1442
1236
  return mergedLengths;
@@ -1492,14 +1286,3 @@ function mergeSortedListsBySeq<T extends PartialSequenceLength>(lists: T[][]): I
1492
1286
 
1493
1287
  return { [Symbol.iterator]: () => new PartialSequenceLengthIterator(lists) };
1494
1288
  }
1495
-
1496
- function insertIntoList<T>(list: T[], index: number, elem: T): void {
1497
- if (index < list.length) {
1498
- for (let k = list.length; k > index; k--) {
1499
- list[k] = list[k - 1];
1500
- }
1501
- list[index] = elem;
1502
- } else {
1503
- list.push(elem);
1504
- }
1505
- }