articulated 0.1.0 → 0.2.0

package/src/id_list.ts

@@ -1,13 +1,100 @@
-import { ElementId, equalsId } from "./id";
+import { SparseIndices } from "sparse-array-rled";
+import { ElementId } from "./id";
 import { SavedIdList } from "./saved_id_list";
 
-interface ListElement {
-  id: ElementId;
-  isDeleted: boolean;
+// Most exports are only for tests. See index.ts for public exports.
+
+/*
+ IdList implementation using a modified B+Tree.
+
+ See tests/id_list_simple.ts for a simpler implementation with the same API but
+ impractical efficiency (linear time ops; one object in memory per id).
+ The fuzz tests compare that implementation to this one.
+
+ The B+Tree is unusual in that it has no keys, only values (= ids). The order on the values
+ is determined "by fiat" using insertAfter/insertBefore instead of using sorted keys.
+
+ The leaves in the B+Tree are not individual ids; instead, each leaf is a compressed representation of a groups of ids
+ with the same bunchId and sequential counters. Each leaf also contains a `present`
+ field to track which of its ids are deleted.
+ (Unlike in a SavedIdList, we do not separate adjacent ids with different isDeleted statuses.)
+
+ Note that it is possible for adjacent leaves to be mergeable (i.e., they could be one leaf) but not merged.
+ This happens if you insert the middle ids later (e.g., 0, 2, 1).
+ It has a slight perf penalty that goes away once you reload.
+ Note that save() needs to work around this possibility - see pushSaveItem.
+
+ The B+Tree also stores two statistics about each subtree: its size (# of present ids)
+ and its knownSize (# of known ids). These allow indexed access in log time.
+
+ Unlike some B+Trees, we do not store a linked list of leaves. Iteration instead uses a depth-first search.
+*/
+
+export interface LeafNode {
+  readonly bunchId: string;
+  readonly startCounter: number;
+  readonly count: number;
+  /**
+   * The present counter values in this leaf node.
+   *
+   * Note that it is indexed by counter, not by (counter - this.startCounter).
+   */
+  readonly present: SparseIndices;
+}
+
+/**
+ * An inner node with inner-node children.
+ */
+export class InnerNodeInner {
+  readonly size: number;
+  readonly knownSize: number;
+
+  constructor(readonly children: readonly InnerNode[]) {
+    let size = 0;
+    let knownSize = 0;
+    for (const child of children) {
+      size += child.size;
+      knownSize += child.knownSize;
+    }
+    this.size = size;
+    this.knownSize = knownSize;
+  }
+}
+
+/**
+ * An inner node with leaf children.
+ */
+export class InnerNodeLeaf {
+  readonly size: number;
+  readonly knownSize: number;
+
+  constructor(readonly children: readonly LeafNode[]) {
+    let size = 0;
+    let knownSize = 0;
+    for (const child of children) {
+      size += child.present.count();
+      knownSize += child.count;
+    }
+    this.size = size;
+    this.knownSize = knownSize;
+  }
 }
 
+export type InnerNode = InnerNodeInner | InnerNodeLeaf;
+
+/**
+ * The B+Tree's branching factor, i.e., the max number of children of a node.
+ *
+ * Note that our B+Tree has no keys - in particular, no keys in internal nodes.
+ *
+ * Wiki B+Tree: "B+ trees can also be used for data stored in RAM.
+ * In this case a reasonable choice for block size would be the size of [the] processor's cache line."
+ * (64 byte cache line) / (8 byte pointer) = 8.
+ */
+export const M = 8;
+
 /**
- * A list of ElementIds.
+ * A list of ElementIds, as a persistent (immutable) data structure.
  *
  * An IdList helps you assign a unique immutable id to each element of a list, such
  * as a todo-list or a text document (= list of characters). That way, you can keep track
@@ -16,11 +103,14 @@ interface ListElement {
  *
  * Any id that has been inserted into an IdList remains **known** to that list indefinitely,
  * allowing you to reference it in insertAfter/insertBefore operations. Calling {@link delete}
- * merely marks an id as deleted (not present); it remains in memory as a "tombstone".
+ * merely marks an id as deleted (= not present); a deleted id does not count towards the length of the list or index-based accessors, but it does remain in memory as a "tombstone".
  * This is useful in collaborative settings, since another user might instruct you to
  * call `insertAfter(before, newId)` when you have already deleted `before` locally.
- * If that is not a concern and you truly want to make an id no longer known, instead
- * call {@link uninsert}.
+ *
+ * To enable easy and efficient rollbacks, such as in a
+ * [server reconciliation](https://mattweidner.com/2024/06/04/server-architectures.html#1-server-reconciliation)
+ * architecture, IdList is a persistent (immutable) data structure. Mutating methods
+ * return a new IdList, sharing memory with the old IdList where possible.
  *
  * See {@link ElementId} for advice on generating ElementIds. IdList is optimized for
  * the case where sequential ElementIds often have the same bunchId and sequential counters.
@@ -28,30 +118,53 @@ interface ListElement {
  * cause such ids to be separated, partially deleted, or even reordered.
  */
 export class IdList {
-  private readonly state: ListElement[];
-  private _length: number;
+  /**
+   * Internal - construct an IdList using a static method (e.g. `IdList.new`).
+   */
+  private constructor(private readonly root: InnerNode) {}
 
   /**
    * Constructs an empty list.
    *
-   * To begin with a non-empty list, use {@link IdList.from} or {@link IdList.fromIds}.
+   * To begin with a non-empty list, use {@link IdList.from}, {@link IdList.fromIds},
+   * or {@link IdList.load}.
    */
-  constructor() {
-    this.state = [];
-    this._length = 0;
+  static new() {
+    return new this(new InnerNodeLeaf([]));
   }
 
   /**
    * Constructs a list with the given known ids and their isDeleted status, in list order.
    */
-  static from(state: Iterable<{ id: ElementId; isDeleted: boolean }>) {
-    const list = new IdList();
-    for (const { id, isDeleted } of state) {
-      // Clone to prevent aliasing.
-      list.state.push({ id, isDeleted });
-      if (!isDeleted) list._length++;
+  static from(
+    knownIds: Iterable<{ id: ElementId; isDeleted: boolean }>
+  ): IdList {
+    // Convert knownIds to a saved state and load that.
+    const savedState: SavedIdList = [];
+
+    for (const { id, isDeleted } of knownIds) {
+      if (savedState.length !== 0) {
+        const current = savedState.at(-1)!;
+        if (
+          id.bunchId === current.bunchId &&
+          id.counter === current.startCounter + current.count &&
+          isDeleted === current.isDeleted
+        ) {
+          // @ts-expect-error Mutating for convenience; no aliasing to worry about.
+          current.count++;
+          continue;
+        }
+      }
+
+      savedState.push({
+        bunchId: id.bunchId,
+        startCounter: id.counter,
+        count: 1,
+        isDeleted,
+      });
     }
-    return list;
+
+    return IdList.load(savedState);
   }
 
   /**
@@ -61,17 +174,18 @@ export class IdList {
    * specify known-but-deleted ids. That way, you can reference the known-but-deleted ids
    * in future insertAfter/insertBefore operations.
    */
-  static fromIds(ids: Iterable<ElementId>) {
-    const list = new IdList();
-    for (const id of ids) {
-      list.state.push({ id, isDeleted: false });
-      list._length++;
-    }
-    return list;
+  static fromIds(ids: Iterable<ElementId>): IdList {
+    return this.from(
+      (function* () {
+        for (const id of ids) yield { id, isDeleted: false };
+      })()
+    );
   }
 
   /**
    * Inserts `newId` immediately after the given id (`before`), which may be deleted.
+   * A new IdList is returned and the current list remains unchanged.
+   *
    * All ids to the right of `before` are shifted one index to the right, in the manner
    * of [Array.splice](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice).
    *
@@ -81,30 +195,108 @@ export class IdList {
    * @param count Provide this to bulk-insert `count` ids from left-to-right,
    * starting with newId and proceeding with the same bunchId and sequential counters.
    * @throws If `before` is not known.
-   * @throws If `newId` is already known.
+   * @throws If any inserted id is already known.
    */
-  insertAfter(before: ElementId | null, newId: ElementId, count = 1) {
-    if (this.isKnown(newId)) {
-      throw new Error("newId is already known");
+  insertAfter(before: ElementId | null, newId: ElementId, count = 1): IdList {
+    if (!(Number.isSafeInteger(newId.counter) && newId.counter >= 0)) {
+      throw new Error(`Invalid counter: ${newId.counter}`);
+    }
+    if (!(Number.isSafeInteger(count) && count >= 0)) {
+      throw new Error(`Invalid count: ${count}`);
+    }
+    if (count !== 0 && isAnyKnown(newId, count, this.root)) {
+      throw new Error("An inserted id is already known");
     }
 
-    let index: number;
     if (before === null) {
-      // -1 so index + 1 is 0: insert at the beginning of the list.
-      index = -1;
-    } else {
-      index = this.state.findIndex((elt) => equalsId(elt.id, before));
-      if (index === -1) {
-        throw new Error("before is not known");
+      if (count === 0) return this;
+
+      if (this.root.children.length === 0) {
+        // Insert the first leaf as a child of root.
+        const present = SparseIndices.new();
+        present.set(newId.counter, count);
+        return new IdList(
+          new InnerNodeLeaf([
+            {
+              bunchId: newId.bunchId,
+              startCounter: newId.counter,
+              count,
+              present,
+            },
+          ])
+        );
+      } else {
+        // Insert before the first known id.
+        return this.insertBefore(firstId(this.root), newId, count);
       }
     }
 
-    this.state.splice(index + 1, 0, ...expandElements(newId, false, count));
-    this._length += count;
+    const located = locate(before, this.root);
+    if (located === null) {
+      throw new Error("before is not known");
+    }
+    if (count === 0) return this;
+    const leaf = located[0].node;
+
+    if (before.counter === leaf.startCounter + leaf.count - 1) {
+      // before is leaf's last id: we insert directly after leaf.
+      if (
+        leaf.bunchId === newId.bunchId &&
+        leaf.startCounter + leaf.count === newId.counter
+      ) {
+        // Extending leaf forwards.
+        const present = leaf.present.clone();
+        present.set(newId.counter, count);
+        return this.replaceLeaf(located, {
+          ...leaf,
+          count: leaf.count + count,
+          present,
+        });
+      } else {
+        const present = SparseIndices.new();
+        present.set(newId.counter, count);
+        return this.replaceLeaf(located, leaf, {
+          bunchId: newId.bunchId,
+          startCounter: newId.counter,
+          count,
+          present,
+        });
+      }
+    } else {
+      // before is not leaf's last id: we need to split leaf and insert there.
+      const newPresent = SparseIndices.new();
+      newPresent.set(newId.counter, count);
+      const [leftPresent, rightPresent] = splitPresent(
+        leaf.present,
+        before.counter + 1
+      );
+      return this.replaceLeaf(
+        located,
+        {
+          ...leaf,
+          count: before.counter + 1 - leaf.startCounter,
+          present: leftPresent,
+        },
+        {
+          bunchId: newId.bunchId,
+          startCounter: newId.counter,
+          count,
+          present: newPresent,
+        },
+        {
+          ...leaf,
+          startCounter: before.counter + 1,
+          count: leaf.count - (before.counter + 1 - leaf.startCounter),
+          present: rightPresent,
+        }
+      );
+    }
   }
 
   /**
    * Inserts `newId` immediately before the given id (`after`), which may be deleted.
+   * A new IdList is returned and the current list remains unchanged.
+   *
    * All ids to the right of `after`, plus `after` itself, are shifted one index to the right, in the manner
    * of [Array.splice](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice).
    *
@@ -117,80 +309,153 @@ export class IdList {
    * @throws If `after` is not known.
    * @throws If `newId` is already known.
    */
-  insertBefore(after: ElementId | null, newId: ElementId, count = 1) {
-    if (this.isKnown(newId)) {
-      throw new Error("newId is already known");
+  insertBefore(after: ElementId | null, newId: ElementId, count = 1): IdList {
+    if (!(Number.isSafeInteger(newId.counter) && newId.counter >= 0)) {
+      throw new Error(`Invalid counter: ${newId.counter}`);
+    }
+    if (!(Number.isSafeInteger(count) && count >= 0)) {
+      throw new Error(`Invalid count: ${count}`);
+    }
+    if (count !== 0 && isAnyKnown(newId, count, this.root)) {
+      throw new Error("An inserted id is already known");
     }
 
-    let index: number;
     if (after === null) {
-      index = this.state.length;
-    } else {
-      index = this.state.findIndex((elt) => equalsId(elt.id, after));
-      if (index === -1) {
-        throw new Error("after is not known");
-      }
+      if (count === 0) return this;
+
+      // Insert after the last known id, or at the beginning if empty.
+      return this.insertAfter(
+        this.root.knownSize === 0 ? null : lastId(this.root),
+        newId,
+        count
+      );
     }
 
-    // We insert the bunch from left-to-right even though it's insertBefore.
-    this.state.splice(index, 0, ...expandElements(newId, false, count));
-    this._length += count;
-  }
+    const located = locate(after, this.root);
+    if (located === null) {
+      throw new Error("after is not known");
+    }
+    if (count === 0) return this;
+    const leaf = located[0].node;
 
-  /**
-   * Un-inserts `id` from the list, making it no longer known or present in this list.
-   *
-   * Typically, you instead want to call {@link delete}, which marks `id` as deleted while
-   * it remains known. That way, you can reference `id` in future insertAfter/insertBefore
-   * operations, including ones sent concurrently by other devices.
-   *
-   * If `id` is already not known, this method does nothing.
-   */
-  uninsert(id: ElementId) {
-    const index = this.state.findIndex((elt) => equalsId(elt.id, id));
-    if (index !== -1) {
-      this.state.splice(index, 1);
-      this._length--;
+    if (after.counter === leaf.startCounter) {
+      // after is leaf's first id: we insert directly before leaf.
+      if (
+        leaf.bunchId === newId.bunchId &&
+        leaf.startCounter === newId.counter + count
+      ) {
+        // Extending leaf backwards.
+        const present = leaf.present.clone();
+        present.set(newId.counter, count);
+        return this.replaceLeaf(located, {
+          ...leaf,
+          startCounter: leaf.startCounter - count,
+          count: leaf.count + count,
+          present,
+        });
+      } else {
+        const present = SparseIndices.new();
+        present.set(newId.counter, count);
+        return this.replaceLeaf(
+          located,
+          {
+            bunchId: newId.bunchId,
+            startCounter: newId.counter,
+            count,
+            present,
+          },
+          leaf
+        );
+      }
+    } else {
+      // after is not leaf's first id: we need to split leaf and insert there.
+      const present = SparseIndices.new();
+      present.set(newId.counter, count);
+      const [leftPresent, rightPresent] = splitPresent(
+        leaf.present,
+        after.counter
+      );
+      return this.replaceLeaf(
+        located,
+        {
+          ...leaf,
+          count: after.counter - leaf.startCounter,
+          present: leftPresent,
+        },
+        {
+          bunchId: newId.bunchId,
+          startCounter: newId.counter,
+          count,
+          present,
+        },
+        {
+          ...leaf,
+          startCounter: after.counter,
+          count: leaf.count - (after.counter - leaf.startCounter),
+          present: rightPresent,
+        }
+      );
     }
   }
 
   /**
-   * Marks `id` as deleted from this list. The id remains known (a "tombstone").
+   * Marks `id` as deleted from this list.
+   * A new IdList is returned and the current list remains unchanged.
    *
+   * Once deleted, `id` does not count towards the length of the list or index-based accessors.
+   * However, it remains known (a "tombstone").
    * Because `id` is still known, you can reference it in future insertAfter/insertBefore
    * operations, including ones sent concurrently by other devices.
-   * However, it does occupy space in memory (compressed in common cases).
-   *
-   * For an exact inverse to `insertAfter(-, id)` or `insertBefore(-, id)`
-   * that makes `id` no longer known, see {@link uninsert}.
+   * This does have a memory cost, but it is compressed in common cases.
    *
-   * If `id` is already deleted or not known, this method does nothing.
+   * If `id` is already deleted or is not known, this method does nothing.
    */
   delete(id: ElementId) {
-    const elt = this.state.find((elt) => equalsId(elt.id, id));
-    if (elt !== undefined && !elt.isDeleted) {
-      elt.isDeleted = true;
-      this._length--;
-    }
+    const located = locate(id, this.root);
+    if (located === null) return this;
+
+    const leaf = located[0].node;
+    if (!leaf.present.has(id.counter)) return this;
+
+    const newPresent = leaf.present.clone();
+    newPresent.delete(id.counter);
+
+    return this.replaceLeaf(located, { ...leaf, present: newPresent });
   }
 
   /**
    * Un-marks `id` as deleted from this list, making it present again.
-   * This is an exact inverse to {@link delete}.
+   * A new IdList is returned and the current list remains unchanged.
+   *
+   * This method is an exact inverse to {@link delete}.
    *
    * If `id` is already present, this method does nothing.
    *
    * @throws If `id` is not known.
    */
   undelete(id: ElementId) {
-    const elt = this.state.find((elt) => equalsId(elt.id, id));
-    if (elt === undefined) {
+    const located = locate(id, this.root);
+    if (located === null) {
       throw new Error("id is not known");
     }
-    if (elt.isDeleted) {
-      elt.isDeleted = false;
-      this._length++;
-    }
+
+    const leaf = located[0].node;
+    if (leaf.present.has(id.counter)) return this;
+
+    const newPresent = leaf.present.clone();
+    newPresent.set(id.counter);
+
+    return this.replaceLeaf(located, { ...leaf, present: newPresent });
+  }
+
+  /**
+   * Replaces the leaf at the given path with newLeaves.
+   * Returns a proper (sufficiently balanced) B+Tree with updated sizes.
+   *
+   * newLeaves.length must be in [1, M].
+   */
+  private replaceLeaf(located: Located, ...newLeaves: LeafNode[]): IdList {
+    return new IdList(replaceNode(located, this.root, newLeaves, 0));
   }
 
   // Accessors
@@ -203,9 +468,9 @@ export class IdList {
    * Compare to {@link isKnown}.
    */
   has(id: ElementId): boolean {
-    const elt = this.state.find((elt) => equalsId(elt.id, id));
-    if (elt === undefined) return false;
-    return !elt.isDeleted;
+    const located = locate(id, this.root);
+    if (located === null) return false;
+    return located[0].node.present.has(id.counter);
   }
 
   /**
@@ -214,7 +479,16 @@ export class IdList {
    * Compare to {@link has}.
    */
   isKnown(id: ElementId): boolean {
-    return this.state.some((elt) => equalsId(elt.id, id));
+    return locate(id, this.root) !== null;
+  }
+
+  /**
+   * The length of the list, counting only present ids.
+   *
+   * To include known but deleted ids, use `this.knownIds.length`.
+   */
+  get length() {
+    return this.root.size;
   }
 
   /**
@@ -228,14 +502,36 @@ export class IdList {
     }
 
     let remaining = index;
-    for (const elt of this.state) {
-      if (!elt.isDeleted) {
-        if (remaining === 0) return elt.id;
-        remaining--;
+    let curParent = this.root;
+    // eslint-disable-next-line no-constant-condition
+    recurse: while (true) {
+      if (curParent instanceof InnerNodeInner) {
+        for (const child of curParent.children) {
+          if (remaining < child.size) {
+            // Recurse.
+            curParent = child;
+            continue recurse;
+          } else {
+            remaining -= child.size;
+          }
+        }
+      } else {
+        for (const child of curParent.children) {
+          const childSize = child.present.count();
+          if (remaining < childSize) {
+            // Found it.
+            return {
+              bunchId: child.bunchId,
+              counter: child.present.indexOfCount(remaining),
+            };
+          } else {
+            remaining -= childSize;
+          }
+        }
       }
-    }
 
-    throw new Error("Internal error");
+      throw new Error("Internal error");
+    }
   }
 
   /**
@@ -250,36 +546,47 @@ export class IdList {
    * @throws If `id` is not known.
    */
   indexOf(id: ElementId, bias: "none" | "left" | "right" = "none"): number {
+    const located = locate(id, this.root);
+    if (located === null) throw new Error("id is not known");
+
     /**
      * The number of present ids less than id.
      * Equivalently, the index id would have if present.
      */
     let index = 0;
-    for (const elt of this.state) {
-      if (equalsId(elt.id, id)) {
-        // Found it.
-        if (elt.isDeleted) {
-          switch (bias) {
-            case "none":
-              return -1;
-            case "left":
-              return index - 1;
-            case "right":
-              return index;
-          }
-        } else return index;
+
+    // Lesser siblings of parent, grandparent, etc.
+    for (let i = 1; i < located.length; i++) {
+      const parent = (
+        i === located.length - 1 ? this.root : located[i + 1].node
+      ) as InnerNodeInner;
+      for (let c = 0; c < located[i].indexInParent; c++) {
+        index += parent.children[c].size;
       }
-      if (!elt.isDeleted) index++;
     }
 
-    throw new Error("id is not known");
-  }
+    // Siblings of id's leaf.
+    const leafParent = (
+      located.length === 1 ? this.root : located[1].node
+    ) as InnerNodeLeaf;
+    for (let c = 0; c < located[0].indexInParent; c++) {
+      index += leafParent.children[c].present.count();
+    }
 
-  /**
-   * The length of the list.
-   */
-  get length(): number {
-    return this._length;
+    // id's index within leaf.
+    const [count, has] = located[0].node.present._countHas(id.counter);
+    index += count;
+    if (has) return index;
+    else {
+      switch (bias) {
+        case "none":
+          return -1;
+        case "left":
+          return index - 1;
+        case "right":
+          return index;
+      }
+    }
   }
 
   // Iterators and views
@@ -287,10 +594,8 @@ export class IdList {
   /**
    * Iterates over all present ids in the list.
    */
-  *[Symbol.iterator](): IterableIterator<ElementId> {
-    for (const elt of this.state) {
-      if (!elt.isDeleted) yield elt.id;
-    }
+  [Symbol.iterator](): IterableIterator<ElementId> {
+    return iterateNode(this.root, false);
   }
 
   /**
@@ -303,26 +608,22 @@ export class IdList {
   /**
    * Iterates over all __known__ ids in the list, indicating which are deleted.
    */
-  valuesWithDeleted(): IterableIterator<{ id: ElementId; isDeleted: boolean }> {
-    return this.state.values();
-  }
-
-  /**
-   * Returns an independent copy of this list, including known but deleted ids.
-   */
-  clone(): IdList {
-    return IdList.from(this.state);
+  valuesWithIsDeleted(): IterableIterator<{
+    id: ElementId;
+    isDeleted: boolean;
+  }> {
+    return iterateNodeWithIsDeleted(this.root);
   }
 
   private _knownIds?: KnownIdView;
 
   /**
-   * A live-updating view of this list that treats all known ids as present.
-   * That is, it ignores isDeleted status when computing list indices or iterating.
+   * A view of this list that treats all known ids as present.
+   * That is, it ignores is-deleted status when computing list indices or iterating.
    */
   get knownIds(): KnownIdView {
     if (this._knownIds === undefined) {
-      this._knownIds = new KnownIdView(this, this.state);
+      this._knownIds = new KnownIdView(this, this.root);
     }
     return this._knownIds;
   }
@@ -336,66 +637,89 @@ export class IdList {
    * See {@link SavedIdList} for a description of the save format.
    */
   save(): SavedIdList {
-    const ans: SavedIdList = [];
+    const acc: SavedIdList = [];
+    saveNode(this.root, acc);
+    return acc;
+  }
+
+  /**
+   * Loads a saved state returned by {@link save}.
+   */
+  static load(savedState: SavedIdList) {
+    // 1. Determine the leaves.
 
-    for (const { id, isDeleted } of this.state) {
-      if (ans.length !== 0) {
-        const current = ans[ans.length - 1];
+    const leaves: LeafNode[] = [];
+    for (let i = 0; i < savedState.length; i++) {
+      const item = savedState[i];
+
+      if (!(Number.isSafeInteger(item.count) && item.count >= 0)) {
+        throw new Error(`Invalid count: ${item.count}`);
+      }
+      if (
+        !(Number.isSafeInteger(item.startCounter) && item.startCounter >= 0)
+      ) {
+        throw new Error(`Invalid startCounter: ${item.startCounter}`);
+      }
+
+      if (item.count === 0) continue;
+
+      if (leaves.length !== 0) {
+        const lastLeaf = leaves.at(-1)!;
         if (
-          id.bunchId === current.bunchId &&
-          id.counter === current.startCounter + current.count &&
-          isDeleted === current.isDeleted
+          item.bunchId === lastLeaf.bunchId &&
+          item.startCounter === lastLeaf.startCounter + lastLeaf.count
         ) {
-          current.count++;
+          // Extend lastLeaf.
+          // Okay to mutate in-place since we haven't referenced it anywhere else yet.
+          // @ts-expect-error Mutate in place
+          lastLeaf.count += item.count;
+          if (!item.isDeleted) {
+            lastLeaf.present.set(item.startCounter, item.count);
+          }
           continue;
         }
       }
 
-      ans.push({
-        bunchId: id.bunchId,
-        startCounter: id.counter,
-        count: 1,
-        isDeleted,
+      // If we get to here, we need a new leaf.
+      const present = SparseIndices.new();
+      if (!item.isDeleted) present.set(item.startCounter, item.count);
+      leaves.push({
+        bunchId: item.bunchId,
+        startCounter: item.startCounter,
+        count: item.count,
+        present,
       });
     }
 
-    return ans;
-  }
-
-  /**
-   * Loads a saved state returned by {@link save}, __overwriting__ the current state of this list.
-   */
-  load(savedState: SavedIdList) {
-    this.state.length = 0;
-    this._length = 0;
+    // 2. Create a B+Tree with the given leaves.
+    // We do a "direct" balanced construction that takes O(n) time, instead of inserting
+    // leaves one-by-one, which would take O(n log(n)) time.
 
-    for (const { bunchId, startCounter, count, isDeleted } of savedState) {
-      if (!(Number.isSafeInteger(count) && count >= 0)) {
-        throw new Error(`Invalid length: ${count}`);
-      }
+    if (leaves.length === 0) return IdList.new();
 
-      for (let i = 0; i < count; i++) {
-        this.state.push({
-          id: { bunchId, counter: startCounter + i },
-          isDeleted,
-        });
-      }
-      if (!isDeleted) this._length += count;
-    }
+    // Depth of the B+Tree (number of non-root nodes on any path from a leaf to the root).
+    // A fully balanced B+Tree of depth d has between [M^{d-1} + 1, M^d] leaves.
+    const depth =
+      leaves.length === 1
+        ? 1
+        : Math.ceil(Math.log(leaves.length) / Math.log(M));
+    return new IdList(buildTree(leaves, 0, depth));
   }
 }
 
 /**
- * A live-updating view of an IdList that treats all known ids as present.
- * That is, this class ignores the underlying list's isDeleted status when computing list indices.
+ * A view of an IdList that treats all known ids as present.
+ * That is, this class ignores the underlying list's is-deleted status when computing list indices.
+ * Access using {@link IdList.knownIds}.
  *
- * To mutate, call methods on the original IdList (`this.list`).
+ * Like IdList, KnownIdView is immutable. To mutate, use a mutating method on the original IdList
+ * and access the returned list's `knownIds`.
  */
 export class KnownIdView {
   /**
    * Internal use only. Use {@link IdList.knownIds} instead.
    */
-  constructor(readonly list: IdList, private readonly state: ListElement[]) {}
+  constructor(readonly list: IdList, private readonly root: InnerNode) {}
 
   // Mutators are omitted - mutate this.list instead.
 
@@ -413,14 +737,72 @@ export class KnownIdView {
       throw new Error(`Index out of bounds: ${index} (length: ${this.length}`);
     }
 
-    return this.state[index].id;
+    let remaining = index;
+    let curParent = this.root;
+    // eslint-disable-next-line no-constant-condition
+    recurse: while (true) {
+      if (curParent instanceof InnerNodeInner) {
+        for (const child of curParent.children) {
+          if (remaining < child.knownSize) {
+            // Recurse.
+            curParent = child;
+            continue recurse;
+          } else {
+            remaining -= child.knownSize;
+          }
+        }
+      } else {
+        for (const child of curParent.children) {
+          if (remaining < child.count) {
+            // Found it.
+            return {
+              bunchId: child.bunchId,
+              counter: child.startCounter + remaining,
+            };
+          } else {
+            remaining -= child.count;
+          }
+        }
+      }
+
+      throw new Error("Internal error");
+    }
   }
 
   /**
    * Returns the index of `id` in this view, or -1 if it is not known.
    */
   indexOf(id: ElementId): number {
-    return this.state.findIndex((elt) => equalsId(elt.id, id));
+    const located = locate(id, this.root);
+    if (located === null) throw new Error("id is not known");
+
+    /**
+     * The number of present ids less than id.
+     * Equivalently, the index id would have if present.
+     */
+    let index = 0;
+
+    // Lesser siblings of parent, grandparent, etc.
+    for (let i = 1; i < located.length; i++) {
+      const parent = (
+        i === located.length - 1 ? this.root : located[i + 1].node
+      ) as InnerNodeInner;
+      for (let c = 0; c < located[i].indexInParent; c++) {
+        index += parent.children[c].knownSize;
+      }
+    }
+
+    // Siblings of id's leaf.
+    const leafParent = (
+      located.length === 1 ? this.root : located[1].node
+    ) as InnerNodeLeaf;
+    for (let c = 0; c < located[0].indexInParent; c++) {
+      const child = leafParent.children[c];
+      index += child.count;
+    }
+
+    // id's index with leaf.
+    return index + (id.counter - located[0].node.startCounter);
   }
 
   /**
@@ -429,7 +811,7 @@ export class KnownIdView {
    * Equivalently, the number of known ids in `this.list`.
    */
   get length(): number {
-    return this.state.length;
+    return this.root.knownSize;
   }
 
   // Iterators
@@ -437,10 +819,8 @@ export class KnownIdView {
   /**
    * Iterates over all ids in this view, i.e., all known ids in `this.list`.
    */
-  *[Symbol.iterator](): IterableIterator<ElementId> {
-    for (const elt of this.state) {
-      yield elt.id;
-    }
+  [Symbol.iterator](): IterableIterator<ElementId> {
+    return iterateNode(this.root, true);
   }
 
   /**
@@ -451,21 +831,319 @@ export class KnownIdView {
   }
 }
 
-function expandElements(
-  startId: ElementId,
-  isDeleted: boolean,
-  count: number
-): ListElement[] {
-  if (!(Number.isSafeInteger(count) && count >= 0)) {
-    throw new Error(`Invalid count: ${count}`);
+/**
+ * Returns the first (leftmost) known ElementId in node's subtree.
+ */
+function firstId(node: InnerNode): ElementId {
+  let currentInner = node;
+  while (!(currentInner instanceof InnerNodeLeaf)) {
+    currentInner = currentInner.children[0];
+  }
+  const firstLeaf = currentInner.children[0];
+  return {
+    bunchId: firstLeaf.bunchId,
+    counter: firstLeaf.startCounter,
+  };
+}
+
+/**
+ * Returns the last (rightmost) known ElementId in node's subtree.
+ */
+function lastId(node: InnerNode): ElementId {
+  let currentInner = node;
+  while (!(currentInner instanceof InnerNodeLeaf)) {
+    currentInner = currentInner.children.at(-1)!;
+  }
+  const lastLeaf = currentInner.children.at(-1)!;
+  return {
+    bunchId: lastLeaf.bunchId,
+    counter: lastLeaf.startCounter + lastLeaf.count - 1,
+  };
+}
+
+type Located = [
+  { node: LeafNode; indexInParent: number },
+  // Index 1 will be an InnerNodeLeaf if it exists.
+  ...{ node: InnerNode; indexInParent: number }[]
+];
+
+/**
+ * Returns the path from id's leaf node to the root, or null if id is not found.
+ *
+ * The path contains each node and its index in its parent's node, starting with id's
+ * LeafNode and ending at a child of the root.
+ */
+export function locate(id: ElementId, node: InnerNode): Located | null {
+  if (node instanceof InnerNodeInner) {
+    for (let i = 0; i < node.children.length; i++) {
+      const child = node.children[i];
+      const childLocated = locate(id, child);
+      if (childLocated !== null) {
+        childLocated.push({ node: child, indexInParent: i });
+        return childLocated;
+      }
+    }
+  } else {
+    for (let i = 0; i < node.children.length; i++) {
+      const child = node.children[i];
+      if (
+        child.bunchId === id.bunchId &&
+        child.startCounter <= id.counter &&
+        id.counter < child.startCounter + child.count
+      ) {
+        return [{ node: child, indexInParent: i }];
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * Returns true if any of the given bulk ids are known within node's subtree.
+ *
+ * Assumes count > 0.
+ */
+function isAnyKnown(id: ElementId, count: number, node: InnerNode): boolean {
+  if (node instanceof InnerNodeInner) {
+    for (const child of node.children) {
+      if (isAnyKnown(id, count, child)) return true;
+    }
+  } else {
+    for (const child of node.children) {
+      if (child.bunchId === id.bunchId) {
+        // Test if there is any overlap between the child's counter range [a, b]
+        // and the bulk id's counter range [c, d].
+        const a = child.startCounter;
+        const b = child.startCounter + child.count - 1;
+        const c = id.counter;
+        const d = id.counter + count - 1;
+        if (a <= d && c <= b) return true;
+      }
+    }
+  }
+  return false;
+}
+
+/**
+ * Replace located[i].node with newNodes.
+ *
+ * newNodes.length must be in [1, M].
+ */
+function replaceNode(
+  located: Located,
+  root: InnerNode,
+  newNodes: InnerNode[] | LeafNode[],
+  i: number
+): InnerNode {
+  const parent =
+    i === located.length - 1 ? root : (located[i + 1].node as InnerNode);
+  const indexInParent = located[i].indexInParent;
+  // Copy-on-write version of parent.children.splice(indexInParent, 1, ...newNodes)
+  const newChildren = parent.children
+    .slice(0, indexInParent)
+    .concat(newNodes, parent.children.slice(indexInParent + 1));
+
+  if (newChildren.length > M) {
+    // Split the parent to maintain BTree property (# children <= M).
+    const split = Math.ceil(newChildren.length / 2);
+    const newParents = [
+      newChildren.slice(0, split),
+      newChildren.slice(split),
+    ].map((children) =>
+      i === 0
+        ? new InnerNodeLeaf(children as LeafNode[])
+        : new InnerNodeInner(children as InnerNode[])
+    );
+    if (i === located.length - 1) {
+      // newParents replace root. We need a new root to hold them.
+      return new InnerNodeInner(newParents);
+    } else {
+      return replaceNode(located, root, newParents, i + 1);
+    }
+  } else {
+    const newParent =
+      i === 0
+        ? new InnerNodeLeaf(newChildren as LeafNode[])
+        : new InnerNodeInner(newChildren as InnerNode[]);
+    if (i === located.length - 1) {
+      // Replaces root.
+      return newParent;
+    } else {
+      return replaceNode(located, root, [newParent], i + 1);
+    }
+  }
+}
+
+/**
+ * Splits present into two SparseIndices at the given counter.
+ */
+function splitPresent(
+  present: SparseIndices,
+  splitCounter: number
+): [leftPresent: SparseIndices, rightPresent: SparseIndices] {
+  const leftPresent = SparseIndices.new();
+  const rightPresent = SparseIndices.new();
+  const leafSlicer = present.newSlicer();
+  for (const [index, count] of leafSlicer.nextSlice(splitCounter)) {
+    leftPresent.set(index, count);
+  }
+  for (const [index, count] of leafSlicer.nextSlice(null)) {
+    rightPresent.set(index, count);
+  }
+  return [leftPresent, rightPresent];
+}
+
+function* iterateNode(
+  node: InnerNode,
+  includeDeleted: boolean
+): IterableIterator<ElementId> {
+  if (node instanceof InnerNodeInner) {
+    for (const child of node.children) {
+      yield* iterateNode(child, includeDeleted);
+    }
+  } else {
+    for (const child of node.children) {
+      if (includeDeleted) {
+        for (let i = 0; i < child.count; i++) {
+          yield { bunchId: child.bunchId, counter: child.startCounter + i };
+        }
+      } else {
+        for (const counter of child.present.keys()) {
+          yield { bunchId: child.bunchId, counter };
+        }
+      }
+    }
+  }
+}
+
+function* iterateNodeWithIsDeleted(
+  node: InnerNode
+): IterableIterator<{ id: ElementId; isDeleted: boolean }> {
+  if (node instanceof InnerNodeInner) {
+    for (const child of node.children) {
+      yield* iterateNodeWithIsDeleted(child);
+    }
+  } else {
+    for (const child of node.children) {
+      let nextIndex = child.startCounter;
+      for (const index of child.present.keys()) {
+        while (nextIndex < index) {
+          yield {
+            id: { bunchId: child.bunchId, counter: nextIndex },
+            isDeleted: true,
+          };
+          nextIndex++;
+        }
+        yield {
+          id: { bunchId: child.bunchId, counter: index },
+          isDeleted: false,
+        };
+        nextIndex++;
+      }
+      while (nextIndex < child.startCounter + child.count) {
+        yield {
+          id: { bunchId: child.bunchId, counter: nextIndex },
+          isDeleted: true,
+        };
+        nextIndex++;
+      }
+    }
+  }
+}
+
+/**
+ * Updates acc to account for node's subtree, as part of a depth-first search
+ * in list order.
+ */
+function saveNode(node: InnerNode, acc: SavedIdList) {
+  if (node instanceof InnerNodeInner) {
+    for (const child of node.children) {
+      saveNode(child, acc);
+    }
+  } else {
+    for (const child of node.children) {
+      let nextIndex = child.startCounter;
+      for (const [index, count] of child.present.items()) {
+        if (nextIndex < index) {
+          // Need a deleted item.
+          pushSaveItem(acc, {
+            bunchId: child.bunchId,
+            startCounter: nextIndex,
+            count: index - nextIndex,
+            isDeleted: true,
+          });
+        }
+        pushSaveItem(acc, {
+          bunchId: child.bunchId,
+          startCounter: index,
+          count,
+          isDeleted: false,
+        });
+        nextIndex = index + count;
+      }
+      if (nextIndex < child.startCounter + child.count) {
+        pushSaveItem(acc, {
+          bunchId: child.bunchId,
+          startCounter: nextIndex,
+          count: child.startCounter + child.count - nextIndex,
+          isDeleted: true,
+        });
+      }
+    }
+  }
+}
+
+/**
+ * Pushes a save item onto acc, combing it with the previous item if possible.
+ *
+ * This function is necessary because we don't guarantee that adjacent leaves are fully merged.
+ * Specifically, if you insert a bunch's ids with counter values (0, 2, 1)
+ * in that order, then counter 1 will extend one of the existing leaves
+ * but not merge with the other leaf.
+ *
+ * This situation won't appear in typical usage, and its perf penalty
+ * will go away once you reload. Thus we tolerate it instead of figuring out
+ * how to delete leaves from a B+Tree.
+ */
+function pushSaveItem(acc: SavedIdList, item: SavedIdList[number]) {
+  if (acc.length > 0) {
+    const previous = acc.at(-1)!;
+    if (
+      previous.isDeleted === item.isDeleted &&
+      previous.bunchId === item.bunchId &&
+      previous.startCounter + previous.count === item.startCounter
+    ) {
+      // Combine items.
+      // @ts-expect-error Mutating for convenience; no aliasing to worry about.
+      previous.count += item.count;
+      return;
+    }
   }
+  acc.push(item);
+}
 
-  const ans: ListElement[] = [];
-  for (let i = 0; i < count; i++) {
-    ans.push({
-      id: { bunchId: startId.bunchId, counter: startId.counter + i },
-      isDeleted,
-    });
+/**
+ * Builds a tree with the given leaves. Used by IdList.load.
+ *
+ * In contrast to inserting the leaves one-by-one, this function balances the
+ * tree, with full inner nodes (M children) whenever possible,
+ * and runs in O(L) time instead of O(L log(L)).
+ */
+function buildTree(
+  leaves: LeafNode[],
+  startIndex: number,
+  depthRemaining: number
+): InnerNode {
+  if (depthRemaining === 1) {
+    return new InnerNodeLeaf(leaves.slice(startIndex, startIndex + M));
+  } else {
+    const children: InnerNode[] = [];
+    const childLeafCount = Math.pow(M, depthRemaining - 1);
+    for (let i = 0; i < M; i++) {
+      const childStartIndex = startIndex + i * childLeafCount;
+      if (childStartIndex >= leaves.length) break;
+      children.push(buildTree(leaves, childStartIndex, depthRemaining - 1));
+    }
+    return new InnerNodeInner(children);
   }
-  return ans;
 }