articulated 0.1.0 → 0.3.0

package/src/id_list.ts

@@ -1,13 +1,146 @@
-import { ElementId, equalsId } from "./id";
+import { SparseIndices } from "sparse-array-rled";
+import { ElementId } from "./id";
+import { LeafMap, MutableLeafMap } from "./internal/leaf_map";
+import { getAndBumpNextSeq, MutableSeqMap, SeqMap } from "./internal/seq_map";
 import { SavedIdList } from "./saved_id_list";
 
-interface ListElement {
-  id: ElementId;
-  isDeleted: boolean;
+// Most exports are only for tests. See index.ts for public exports.
+
+/*
+ IdList implementation using a modified B+Tree.
+
+ See tests/id_list_simple.ts for a simpler implementation with the same API but
+ impractical efficiency (linear time ops; one object in memory per id).
+ The fuzz tests compare that implementation to this one.
+
+ The B+Tree is unusual in that it has no keys, only values (= ids). The order on the values
+ is determined "by fiat" using insertAfter/insertBefore instead of using sorted keys.
+
+ The leaves in the B+Tree are not individual ids; instead, each leaf is a compressed representation of a groups of ids
+ with the same bunchId and sequential counters. Each leaf also contains a `present`
+ field to track which of its ids are deleted.
+ (Unlike in a SavedIdList, we do not separate adjacent ids with different isDeleted statuses.)
+
+ Note that it is possible for adjacent leaves to be mergeable (i.e., they could be one leaf) but not merged.
+ This happens if you insert the middle ids later (e.g., 0, 2, 1).
+ It has a slight perf penalty that goes away once you reload.
+ Note that save() needs to work around this possibility - see pushSaveItem.
+
+ The B+Tree also stores two statistics about each subtree: its size (# of present ids)
+ and its knownSize (# of known ids). These allow indexed access in log time.
+
+ Unlike some B+Trees, we do not store a linked list of leaves. Iteration instead uses a depth-first search.
+
+ Finally, we also store a "bottom-up" view of the B+Tree, in order to quickly find the leaf or
+ tree path corresponding to an ElementId. Each inner node is assigned a unique sequence number
+ (seq), and we store a persistent map from each leaf to its parent's seq (leafMap)
+ and from each inner node's seq to its parent's seq (parentSeqs). Because leafMap is sorted
+ by (LeafNode.bunchId, LeafNode.startCounter), we can also use it to lookup the leaf corresponding
+ to an ElementId, e.g., for IdList.has.
+*/
+
+export interface LeafNode {
+  readonly bunchId: string;
+  readonly startCounter: number;
+  readonly count: number;
+  /**
+   * The present counter values in this leaf node.
+   *
+   * Note that it is indexed by counter, not by (counter - this.startCounter).
+   */
+  readonly present: SparseIndices;
+}
+
+/**
+ * An inner node with inner-node children.
+ */
+export class InnerNodeInner {
+  readonly size: number;
+  readonly knownSize: number;
+
+  constructor(
+    /**
+     * A unique identifer for this node within its IdTree.
+     */
+    readonly seq: number,
+    readonly children: readonly InnerNode[],
+    /**
+     * We add entries for the children to this map, overwriting any existing parentSeqs.
+     *
+     * Pass null to skip when you are doing it yourself. Regardless, you need to
+     * delete any outdated entries yourself.
+     */
+    parentSeqsMut: MutableSeqMap | null
+  ) {
+    let size = 0;
+    let knownSize = 0;
+    for (const child of children) {
+      size += child.size;
+      knownSize += child.knownSize;
+      if (parentSeqsMut) {
+        parentSeqsMut.value = parentSeqsMut.value.set(child.seq, seq);
+      }
+    }
+    this.size = size;
+    this.knownSize = knownSize;
+  }
+}
+
+/**
+ * An inner node with leaf children.
+ */
+export class InnerNodeLeaf {
+  readonly size: number;
+  readonly knownSize: number;
+
+  constructor(
+    /**
+     * A unique identifer for this node within its IdTree.
+     */
+    readonly seq: number,
+    readonly children: readonly LeafNode[],
+    /**
+     * We add entries for the children to this map, overwriting any existing parentSeqs.
+     *
+     * Pass null to skip when you are doing it yourself.
+     */
+    leafMapMut: MutableLeafMap | null
+  ) {
+    let size = 0;
+    let knownSize = 0;
+    for (const child of children) {
+      size += child.present.count();
+      knownSize += child.count;
+      if (leafMapMut) {
+        leafMapMut.value = leafMapMut.value.set(child, seq);
+      }
+    }
+    this.size = size;
+    this.knownSize = knownSize;
+  }
 }
 
+export type InnerNode = InnerNodeInner | InnerNodeLeaf;
+
+type Located = [
+  { node: LeafNode; indexInParent: number },
+  // Index 1 will be an InnerNodeLeaf if it exists.
+  ...{ node: InnerNode; indexInParent: number }[]
+];
+
 /**
- * A list of ElementIds.
+ * The B+Tree's branching factor, i.e., the max number of children of a node.
+ *
+ * Note that our B+Tree has no keys - in particular, no keys in internal nodes.
+ *
+ * Wiki B+Tree: "B+ trees can also be used for data stored in RAM.
+ * In this case a reasonable choice for block size would be the size of [the] processor's cache line."
+ * (64 byte cache line) / (8 byte pointer) = 8.
+ */
+export const M = 8;
+
+/**
+ * A list of ElementIds, as a persistent (immutable) data structure.
  *
  * An IdList helps you assign a unique immutable id to each element of a list, such
  * as a todo-list or a text document (= list of characters). That way, you can keep track
@@ -16,11 +149,14 @@ interface ListElement {
  *
  * Any id that has been inserted into an IdList remains **known** to that list indefinitely,
  * allowing you to reference it in insertAfter/insertBefore operations. Calling {@link delete}
- * merely marks an id as deleted (not present); it remains in memory as a "tombstone".
+ * merely marks an id as deleted (= not present); a deleted id does not count towards the length of the list or index-based accessors, but it does remain in memory as a "tombstone".
  * This is useful in collaborative settings, since another user might instruct you to
  * call `insertAfter(before, newId)` when you have already deleted `before` locally.
- * If that is not a concern and you truly want to make an id no longer known, instead
- * call {@link uninsert}.
+ *
+ * To enable easy and efficient rollbacks, such as in a
+ * [server reconciliation](https://mattweidner.com/2024/06/04/server-architectures.html#1-server-reconciliation)
+ * architecture, IdList is a persistent (immutable) data structure. Mutating methods
+ * return a new IdList, sharing memory with the old IdList where possible.
  *
  * See {@link ElementId} for advice on generating ElementIds. IdList is optimized for
  * the case where sequential ElementIds often have the same bunchId and sequential counters.
@@ -28,30 +164,77 @@ interface ListElement {
  * cause such ids to be separated, partially deleted, or even reordered.
  */
 export class IdList {
-  private readonly state: ListElement[];
-  private _length: number;
+  /**
+   * A persistent map from each InnerNode's seq to its parent node's seq.
+   *
+   * We map the root's seq to 0 (in our constructor).
+   */
+  private readonly parentSeqs: SeqMap;
+
+  /**
+   * Internal - construct an IdList using a static method (e.g. `IdList.new`).
+   */
+  private constructor(
+    private readonly root: InnerNode,
+    /**
+     * A persistent sorted map from each leaf to its parent node's seq.
+     *
+     * Besides parentSeqs, we also use this to lookup leaves by ElementId.
+     */
+    private readonly leafMap: LeafMap,
+    parentSeqs: SeqMap
+  ) {
+    this.parentSeqs = parentSeqs.set(root.seq, 0);
+  }
 
   /**
    * Constructs an empty list.
    *
-   * To begin with a non-empty list, use {@link IdList.from} or {@link IdList.fromIds}.
+   * To begin with a non-empty list, use {@link IdList.from}, {@link IdList.fromIds},
+   * or {@link IdList.load}.
    */
-  constructor() {
-    this.state = [];
-    this._length = 0;
+  static new() {
+    const leafMapMut = { value: LeafMap.new() };
+    const parentSeqsMut = { value: SeqMap.new() };
+    return new this(
+      new InnerNodeLeaf(getAndBumpNextSeq(parentSeqsMut), [], leafMapMut),
+      leafMapMut.value,
+      parentSeqsMut.value
+    );
   }
 
   /**
    * Constructs a list with the given known ids and their isDeleted status, in list order.
    */
-  static from(state: Iterable<{ id: ElementId; isDeleted: boolean }>) {
-    const list = new IdList();
-    for (const { id, isDeleted } of state) {
-      // Clone to prevent aliasing.
-      list.state.push({ id, isDeleted });
-      if (!isDeleted) list._length++;
+  static from(
+    knownIds: Iterable<{ id: ElementId; isDeleted: boolean }>
+  ): IdList {
+    // Convert knownIds to a saved state and load that.
+    const savedState: SavedIdList = [];
+
+    for (const { id, isDeleted } of knownIds) {
+      if (savedState.length !== 0) {
+        const current = savedState.at(-1)!;
+        if (
+          id.bunchId === current.bunchId &&
+          id.counter === current.startCounter + current.count &&
+          isDeleted === current.isDeleted
+        ) {
+          // @ts-expect-error Mutating for convenience; no aliasing to worry about.
+          current.count++;
+          continue;
+        }
+      }
+
+      savedState.push({
+        bunchId: id.bunchId,
+        startCounter: id.counter,
+        count: 1,
+        isDeleted,
+      });
     }
-    return list;
+
+    return IdList.load(savedState);
   }
 
   /**
@@ -61,17 +244,18 @@ export class IdList {
    * specify known-but-deleted ids. That way, you can reference the known-but-deleted ids
    * in future insertAfter/insertBefore operations.
    */
-  static fromIds(ids: Iterable<ElementId>) {
-    const list = new IdList();
-    for (const id of ids) {
-      list.state.push({ id, isDeleted: false });
-      list._length++;
-    }
-    return list;
+  static fromIds(ids: Iterable<ElementId>): IdList {
+    return this.from(
+      (function* () {
+        for (const id of ids) yield { id, isDeleted: false };
+      })()
+    );
   }
 
   /**
    * Inserts `newId` immediately after the given id (`before`), which may be deleted.
+   * A new IdList is returned and the current list remains unchanged.
+   *
    * All ids to the right of `before` are shifted one index to the right, in the manner
    * of [Array.splice](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice).
    *
@@ -81,30 +265,111 @@ export class IdList {
    * @param count Provide this to bulk-insert `count` ids from left-to-right,
    * starting with newId and proceeding with the same bunchId and sequential counters.
    * @throws If `before` is not known.
-   * @throws If `newId` is already known.
+   * @throws If any inserted id is already known.
    */
-  insertAfter(before: ElementId | null, newId: ElementId, count = 1) {
-    if (this.isKnown(newId)) {
-      throw new Error("newId is already known");
+  insertAfter(before: ElementId | null, newId: ElementId, count = 1): IdList {
+    if (!(Number.isSafeInteger(newId.counter) && newId.counter >= 0)) {
+      throw new Error(`Invalid counter: ${newId.counter}`);
+    }
+    if (!(Number.isSafeInteger(count) && count >= 0)) {
+      throw new Error(`Invalid count: ${count}`);
+    }
+    if (this.isAnyKnown(newId, count)) {
+      throw new Error("An inserted id is already known");
     }
 
-    let index: number;
     if (before === null) {
-      // -1 so index + 1 is 0: insert at the beginning of the list.
-      index = -1;
-    } else {
-      index = this.state.findIndex((elt) => equalsId(elt.id, before));
-      if (index === -1) {
-        throw new Error("before is not known");
+      if (count === 0) return this;
+
+      if (this.root.children.length === 0) {
+        // Insert the first leaf as a child of root.
+        const present = SparseIndices.new();
+        present.set(newId.counter, count);
+        const leaf: LeafNode = {
+          bunchId: newId.bunchId,
+          startCounter: newId.counter,
+          count,
+          present,
+        };
+
+        const leafMapMut = { value: this.leafMap };
+        return new IdList(
+          new InnerNodeLeaf(this.root.seq, [leaf], leafMapMut),
+          leafMapMut.value,
+          this.parentSeqs
+        );
+      } else {
+        // Insert before the first known id.
+        return this.insertBefore(firstId(this.root), newId, count);
       }
     }
 
-    this.state.splice(index + 1, 0, ...expandElements(newId, false, count));
-    this._length += count;
+    const located = this.locate(before);
+    if (located === null) {
+      throw new Error("before is not known");
+    }
+    if (count === 0) return this;
+    const leaf = located[0].node;
+
+    if (before.counter === leaf.startCounter + leaf.count - 1) {
+      // before is leaf's last id: we insert directly after leaf.
+      if (
+        leaf.bunchId === newId.bunchId &&
+        leaf.startCounter + leaf.count === newId.counter
+      ) {
+        // Extending leaf forwards.
+        const present = leaf.present.clone();
+        present.set(newId.counter, count);
+        return this.replaceLeaf(located, {
+          ...leaf,
+          count: leaf.count + count,
+          present,
+        });
+      } else {
+        const present = SparseIndices.new();
+        present.set(newId.counter, count);
+        return this.replaceLeaf(located, leaf, {
+          bunchId: newId.bunchId,
+          startCounter: newId.counter,
+          count,
+          present,
+        });
+      }
+    } else {
+      // before is not leaf's last id: we need to split leaf and insert there.
+      const newPresent = SparseIndices.new();
+      newPresent.set(newId.counter, count);
+      const [leftPresent, rightPresent] = splitPresent(
+        leaf.present,
+        before.counter + 1
+      );
+      return this.replaceLeaf(
+        located,
+        {
+          ...leaf,
+          count: before.counter + 1 - leaf.startCounter,
+          present: leftPresent,
+        },
+        {
+          bunchId: newId.bunchId,
+          startCounter: newId.counter,
+          count,
+          present: newPresent,
+        },
+        {
+          ...leaf,
+          startCounter: before.counter + 1,
+          count: leaf.count - (before.counter + 1 - leaf.startCounter),
+          present: rightPresent,
+        }
+      );
+    }
   }
 
   /**
    * Inserts `newId` immediately before the given id (`after`), which may be deleted.
+   * A new IdList is returned and the current list remains unchanged.
+   *
    * All ids to the right of `after`, plus `after` itself, are shifted one index to the right, in the manner
    * of [Array.splice](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice).
    *
@@ -117,80 +382,218 @@ export class IdList {
    * @throws If `after` is not known.
    * @throws If `newId` is already known.
    */
-  insertBefore(after: ElementId | null, newId: ElementId, count = 1) {
-    if (this.isKnown(newId)) {
-      throw new Error("newId is already known");
+  insertBefore(after: ElementId | null, newId: ElementId, count = 1): IdList {
+    if (!(Number.isSafeInteger(newId.counter) && newId.counter >= 0)) {
+      throw new Error(`Invalid counter: ${newId.counter}`);
+    }
+    if (!(Number.isSafeInteger(count) && count >= 0)) {
+      throw new Error(`Invalid count: ${count}`);
+    }
+    if (this.isAnyKnown(newId, count)) {
+      throw new Error("An inserted id is already known");
     }
 
-    let index: number;
     if (after === null) {
-      index = this.state.length;
-    } else {
-      index = this.state.findIndex((elt) => equalsId(elt.id, after));
-      if (index === -1) {
-        throw new Error("after is not known");
-      }
+      if (count === 0) return this;
+
+      // Insert after the last known id, or at the beginning if empty.
+      return this.insertAfter(
+        this.root.knownSize === 0 ? null : lastId(this.root),
+        newId,
+        count
+      );
     }
 
-    // We insert the bunch from left-to-right even though it's insertBefore.
-    this.state.splice(index, 0, ...expandElements(newId, false, count));
-    this._length += count;
-  }
+    const located = this.locate(after);
+    if (located === null) {
+      throw new Error("after is not known");
+    }
+    if (count === 0) return this;
+    const leaf = located[0].node;
 
-  /**
-   * Un-inserts `id` from the list, making it no longer known or present in this list.
-   *
-   * Typically, you instead want to call {@link delete}, which marks `id` as deleted while
-   * it remains known. That way, you can reference `id` in future insertAfter/insertBefore
-   * operations, including ones sent concurrently by other devices.
-   *
-   * If `id` is already not known, this method does nothing.
-   */
-  uninsert(id: ElementId) {
-    const index = this.state.findIndex((elt) => equalsId(elt.id, id));
-    if (index !== -1) {
-      this.state.splice(index, 1);
-      this._length--;
+    if (after.counter === leaf.startCounter) {
+      // after is leaf's first id: we insert directly before leaf.
+      if (
+        leaf.bunchId === newId.bunchId &&
+        leaf.startCounter === newId.counter + count
+      ) {
+        // Extending leaf backwards.
+        const present = leaf.present.clone();
+        present.set(newId.counter, count);
+        return this.replaceLeaf(located, {
+          ...leaf,
+          startCounter: leaf.startCounter - count,
+          count: leaf.count + count,
+          present,
+        });
+      } else {
+        const present = SparseIndices.new();
+        present.set(newId.counter, count);
+        return this.replaceLeaf(
+          located,
+          {
+            bunchId: newId.bunchId,
+            startCounter: newId.counter,
+            count,
+            present,
+          },
+          leaf
+        );
+      }
+    } else {
+      // after is not leaf's first id: we need to split leaf and insert there.
+      const present = SparseIndices.new();
+      present.set(newId.counter, count);
+      const [leftPresent, rightPresent] = splitPresent(
+        leaf.present,
+        after.counter
+      );
+      return this.replaceLeaf(
+        located,
+        {
+          ...leaf,
+          count: after.counter - leaf.startCounter,
+          present: leftPresent,
+        },
+        {
+          bunchId: newId.bunchId,
+          startCounter: newId.counter,
+          count,
+          present,
+        },
+        {
+          ...leaf,
+          startCounter: after.counter,
+          count: leaf.count - (after.counter - leaf.startCounter),
+          present: rightPresent,
+        }
+      );
     }
   }
 
   /**
-   * Marks `id` as deleted from this list. The id remains known (a "tombstone").
+   * Marks `id` as deleted from this list.
+   * A new IdList is returned and the current list remains unchanged.
    *
+   * Once deleted, `id` does not count towards the length of the list or index-based accessors.
+   * However, it remains known (a "tombstone").
    * Because `id` is still known, you can reference it in future insertAfter/insertBefore
    * operations, including ones sent concurrently by other devices.
-   * However, it does occupy space in memory (compressed in common cases).
-   *
-   * For an exact inverse to `insertAfter(-, id)` or `insertBefore(-, id)`
-   * that makes `id` no longer known, see {@link uninsert}.
+   * This does have a memory cost, but it is compressed in common cases.
    *
-   * If `id` is already deleted or not known, this method does nothing.
+   * If `id` is already deleted or is not known, this method does nothing.
    */
   delete(id: ElementId) {
-    const elt = this.state.find((elt) => equalsId(elt.id, id));
-    if (elt !== undefined && !elt.isDeleted) {
-      elt.isDeleted = true;
-      this._length--;
-    }
+    const located = this.locate(id);
+    if (located === null) return this;
+
+    const leaf = located[0].node;
+    if (!leaf.present.has(id.counter)) return this;
+
+    const newPresent = leaf.present.clone();
+    newPresent.delete(id.counter);
+
+    return this.replaceLeaf(located, { ...leaf, present: newPresent });
   }
 
   /**
    * Un-marks `id` as deleted from this list, making it present again.
-   * This is an exact inverse to {@link delete}.
+   * A new IdList is returned and the current list remains unchanged.
+   *
+   * This method is an exact inverse to {@link delete}.
    *
    * If `id` is already present, this method does nothing.
    *
    * @throws If `id` is not known.
    */
   undelete(id: ElementId) {
-    const elt = this.state.find((elt) => equalsId(elt.id, id));
-    if (elt === undefined) {
+    const located = this.locate(id);
+    if (located === null) {
       throw new Error("id is not known");
     }
-    if (elt.isDeleted) {
-      elt.isDeleted = false;
-      this._length++;
+
+    const leaf = located[0].node;
+    if (leaf.present.has(id.counter)) return this;
+
+    const newPresent = leaf.present.clone();
+    newPresent.set(id.counter);
+
+    return this.replaceLeaf(located, { ...leaf, present: newPresent });
+  }
+
+  /**
+   * Returns the path from id's leaf node to the root, or null if id is not found.
+   *
+   * The path contains each node and its index in its parent's node, starting with id's
+   * LeafNode and ending at a child of the root.
+   */
+  private locate(id: ElementId): Located | null {
+    // Find the leaf containing id, if any.
+    const [leaf, parentSeq] = this.leafMap.getLeaf(id.bunchId, id.counter);
+    if (leaf === undefined) return null;
+    if (
+      !(
+        leaf.bunchId === id.bunchId &&
+        leaf.startCounter <= id.counter &&
+        id.counter < leaf.startCounter + leaf.count
+      )
+    ) {
+      return null;
+    }
+
+    // Find the seqs on the path (leaf, root].
+    const innerSeqs: number[] = [];
+    let curSeq = parentSeq;
+    while (curSeq !== 0) {
+      innerSeqs.push(curSeq);
+      curSeq = this.parentSeqs.get(curSeq);
+    }
+
+    // Find the nodes and indexInParent's on the path (root, leaf),
+    // using seqs to find the appropriate child of each node.
+    const innerNodes: { node: InnerNode; indexInParent: number }[] = [];
+    let curParent = this.root;
+    // Start at the root child's seq and proceed to the leaf parent's seq.
+    for (let i = innerSeqs.length - 2; i >= 0; i--) {
+      const children = (curParent as InnerNodeInner).children;
+      const childIndex = children.findIndex(
+        (child) => child.seq === innerSeqs[i]
+      );
+      if (childIndex === -1) throw new Error("Internal error");
+      const child = children[childIndex];
+
+      innerNodes.push({ node: child, indexInParent: childIndex });
+      curParent = child;
     }
+
+    // Now curParent is the leaf's parent. Find leaf in its children and return.
+    const leafChildIndex = (curParent as InnerNodeLeaf).children.indexOf(leaf);
+    if (leafChildIndex === -1) throw new Error("Internal error");
+    return [
+      { node: leaf, indexInParent: leafChildIndex },
+      ...innerNodes.reverse(),
+    ];
+  }
+
+  /**
+   * Replaces the leaf at the given path with newLeaves.
+   * Returns a proper (sufficiently balanced) B+Tree with updated sizes.
+   *
+   * newLeaves.length must be in [1, M].
+   */
+  private replaceLeaf(located: Located, ...newLeaves: LeafNode[]): IdList {
+    const leafMapMut = { value: this.leafMap };
+    const parentSeqsMut = { value: this.parentSeqs };
+
+    const newRoot = replaceNode(
+      located,
+      this.root,
+      leafMapMut,
+      parentSeqsMut,
+      newLeaves,
+      0
+    );
+    return new IdList(newRoot, leafMapMut.value, parentSeqsMut.value);
   }
 
   // Accessors
@@ -203,9 +606,13 @@ export class IdList {
    * Compare to {@link isKnown}.
    */
   has(id: ElementId): boolean {
-    const elt = this.state.find((elt) => equalsId(elt.id, id));
-    if (elt === undefined) return false;
-    return !elt.isDeleted;
+    // Find the LeafNode that would contain id if known.
+    const [leaf] = this.leafMap.getLeaf(id.bunchId, id.counter);
+    if (leaf && leaf.bunchId === id.bunchId) {
+      return leaf.present.has(id.counter);
+    }
+
+    return false;
   }
 
   /**
@@ -214,7 +621,49 @@ export class IdList {
    * Compare to {@link has}.
    */
   isKnown(id: ElementId): boolean {
-    return this.state.some((elt) => equalsId(elt.id, id));
+    // Find the LeafNode that would contain id if known.
+    const [leaf] = this.leafMap.getLeaf(id.bunchId, id.counter);
+    if (leaf && leaf.bunchId === id.bunchId) {
+      return (
+        leaf.startCounter <= id.counter &&
+        id.counter < leaf.startCounter + leaf.count
+      );
+    }
+
+    return false;
+  }
+
+  // TODO: Make public?
+  /**
+   * Returns true if any of the given bulk ids are known.
+   */
+  private isAnyKnown(id: ElementId, count: number): boolean {
+    if (count === 0) return false;
+
+    // Find the leaf containing the last id, or the previous leaf.
+    // If any leaf knows any of the ids, this leaf must know an id too.
+    const [leaf] = this.leafMap.getLeaf(id.bunchId, id.counter + count - 1);
+
+    if (leaf && leaf.bunchId === id.bunchId) {
+      // Test if there is any overlap between the leaf's counter range [a, b]
+      // and the bulk ids' counter range [c, d].
+      const a = leaf.startCounter;
+      const b = leaf.startCounter + leaf.count - 1;
+      const c = id.counter;
+      const d = id.counter + count - 1;
+      return a <= d && c <= b;
+    }
+
+    return false;
+  }
+
+  /**
+   * The length of the list, counting only present ids.
+   *
+   * To include known but deleted ids, use `this.knownIds.length`.
+   */
+  get length() {
+    return this.root.size;
   }
 
   /**
@@ -228,14 +677,36 @@ export class IdList {
     }
 
     let remaining = index;
-    for (const elt of this.state) {
-      if (!elt.isDeleted) {
-        if (remaining === 0) return elt.id;
-        remaining--;
+    let curParent = this.root;
+    // eslint-disable-next-line no-constant-condition
+    recurse: while (true) {
+      if (curParent instanceof InnerNodeInner) {
+        for (const child of curParent.children) {
+          if (remaining < child.size) {
+            // Recurse.
+            curParent = child;
+            continue recurse;
+          } else {
+            remaining -= child.size;
+          }
+        }
+      } else {
+        for (const child of curParent.children) {
+          const childSize = child.present.count();
+          if (remaining < childSize) {
+            // Found it.
+            return {
+              bunchId: child.bunchId,
+              counter: child.present.indexOfCount(remaining),
+            };
+          } else {
+            remaining -= childSize;
+          }
+        }
       }
-    }
 
-    throw new Error("Internal error");
+      throw new Error("Internal error");
+    }
   }
 
   /**
@@ -250,36 +721,47 @@ export class IdList {
    * @throws If `id` is not known.
    */
   indexOf(id: ElementId, bias: "none" | "left" | "right" = "none"): number {
+    const located = this.locate(id);
+    if (located === null) throw new Error("id is not known");
+
     /**
      * The number of present ids less than id.
      * Equivalently, the index id would have if present.
      */
     let index = 0;
-    for (const elt of this.state) {
-      if (equalsId(elt.id, id)) {
-        // Found it.
-        if (elt.isDeleted) {
-          switch (bias) {
-            case "none":
-              return -1;
-            case "left":
-              return index - 1;
-            case "right":
-              return index;
-          }
-        } else return index;
+
+    // Lesser siblings of parent, grandparent, etc.
+    for (let i = 1; i < located.length; i++) {
+      const parent = (
+        i === located.length - 1 ? this.root : located[i + 1].node
+      ) as InnerNodeInner;
+      for (let c = 0; c < located[i].indexInParent; c++) {
+        index += parent.children[c].size;
       }
-      if (!elt.isDeleted) index++;
     }
 
-    throw new Error("id is not known");
-  }
+    // Siblings of id's leaf.
+    const leafParent = (
+      located.length === 1 ? this.root : located[1].node
+    ) as InnerNodeLeaf;
+    for (let c = 0; c < located[0].indexInParent; c++) {
+      index += leafParent.children[c].present.count();
+    }
 
-  /**
-   * The length of the list.
-   */
-  get length(): number {
-    return this._length;
+    // id's index within leaf.
+    const [count, has] = located[0].node.present._countHas(id.counter);
+    index += count;
+    if (has) return index;
+    else {
+      switch (bias) {
+        case "none":
+          return -1;
+        case "left":
+          return index - 1;
+        case "right":
+          return index;
+      }
+    }
   }
 
   // Iterators and views
@@ -287,10 +769,8 @@ export class IdList {
   /**
    * Iterates over all present ids in the list.
    */
-  *[Symbol.iterator](): IterableIterator<ElementId> {
-    for (const elt of this.state) {
-      if (!elt.isDeleted) yield elt.id;
-    }
+  [Symbol.iterator](): IterableIterator<ElementId> {
+    return iterateNode(this.root, false);
   }
 
   /**
@@ -303,26 +783,22 @@ export class IdList {
   /**
    * Iterates over all __known__ ids in the list, indicating which are deleted.
    */
-  valuesWithDeleted(): IterableIterator<{ id: ElementId; isDeleted: boolean }> {
-    return this.state.values();
-  }
-
-  /**
-   * Returns an independent copy of this list, including known but deleted ids.
-   */
-  clone(): IdList {
-    return IdList.from(this.state);
+  valuesWithIsDeleted(): IterableIterator<{
+    id: ElementId;
+    isDeleted: boolean;
+  }> {
+    return iterateNodeWithIsDeleted(this.root);
   }
 
   private _knownIds?: KnownIdView;
 
   /**
-   * A live-updating view of this list that treats all known ids as present.
-   * That is, it ignores isDeleted status when computing list indices or iterating.
+   * A view of this list that treats all known ids as present.
+   * That is, it ignores is-deleted status when computing list indices or iterating.
    */
   get knownIds(): KnownIdView {
     if (this._knownIds === undefined) {
-      this._knownIds = new KnownIdView(this, this.state);
+      this._knownIds = new KnownIdView(this, this.root);
     }
     return this._knownIds;
   }
@@ -336,66 +812,98 @@ export class IdList {
    * See {@link SavedIdList} for a description of the save format.
    */
   save(): SavedIdList {
-    const ans: SavedIdList = [];
+    const acc: SavedIdList = [];
+    saveNode(this.root, acc);
+    return acc;
+  }
+
+  /**
+   * Loads a saved state returned by {@link save}.
+   */
+  static load(savedState: SavedIdList) {
+    // 1. Determine the leaves in list order.
 
-    for (const { id, isDeleted } of this.state) {
-      if (ans.length !== 0) {
-        const current = ans[ans.length - 1];
+    const leaves: LeafNode[] = [];
+    for (let i = 0; i < savedState.length; i++) {
+      const item = savedState[i];
+
+      if (!(Number.isSafeInteger(item.count) && item.count >= 0)) {
+        throw new Error(`Invalid count: ${item.count}`);
+      }
+      if (
+        !(Number.isSafeInteger(item.startCounter) && item.startCounter >= 0)
+      ) {
+        throw new Error(`Invalid startCounter: ${item.startCounter}`);
+      }
+
+      if (item.count === 0) continue;
+
+      if (leaves.length !== 0) {
+        const lastLeaf = leaves.at(-1)!;
         if (
-          id.bunchId === current.bunchId &&
-          id.counter === current.startCounter + current.count &&
-          isDeleted === current.isDeleted
+          item.bunchId === lastLeaf.bunchId &&
+          item.startCounter === lastLeaf.startCounter + lastLeaf.count
         ) {
-          current.count++;
+          // Extend lastLeaf.
+          // Okay to mutate in-place since we haven't referenced it anywhere else yet.
+          // @ts-expect-error Mutate in place
+          lastLeaf.count += item.count;
+          if (!item.isDeleted) {
+            lastLeaf.present.set(item.startCounter, item.count);
+          }
           continue;
         }
       }
 
-      ans.push({
-        bunchId: id.bunchId,
-        startCounter: id.counter,
-        count: 1,
-        isDeleted,
+      // If we get to here, we need a new leaf.
+      const present = SparseIndices.new();
+      if (!item.isDeleted) present.set(item.startCounter, item.count);
+      leaves.push({
+        bunchId: item.bunchId,
+        startCounter: item.startCounter,
+        count: item.count,
+        present,
       });
     }
 
-    return ans;
-  }
+    // 2. Create a B+Tree with the given leaves.
+    // We do a "direct" balanced construction that takes O(L) time, instead of inserting
+    // leaves one-by-one, which would take O(L log(L)) time.
+    // However, constructing the sorted leafMap brings the overall runtime to O(L log(L)).
 
-  /**
-   * Loads a saved state returned by {@link save}, __overwriting__ the current state of this list.
-   */
-  load(savedState: SavedIdList) {
-    this.state.length = 0;
-    this._length = 0;
+    if (leaves.length === 0) return IdList.new();
 
-    for (const { bunchId, startCounter, count, isDeleted } of savedState) {
-      if (!(Number.isSafeInteger(count) && count >= 0)) {
-        throw new Error(`Invalid length: ${count}`);
-      }
+    // TODO: Test the aux data structures after loading.
+    // E.g. reload and then call checkAll again.
+    // Also should do insertions to test splitting of the full tree.
 
-      for (let i = 0; i < count; i++) {
-        this.state.push({
-          id: { bunchId, counter: startCounter + i },
-          isDeleted,
-        });
-      }
-      if (!isDeleted) this._length += count;
-    }
+    const leafMapMut = { value: LeafMap.new() };
+    const parentSeqsMut = { value: SeqMap.new() };
+
+    // Depth of the B+Tree (number of non-root nodes on any path from a leaf to the root).
+    // A full B+Tree of depth d has between [M^{d-1} + 1, M^d] leaves.
+    const depth =
+      leaves.length === 1
+        ? 1
+        : Math.ceil(Math.log(leaves.length) / Math.log(M));
+    const root = buildTree(leaves, leafMapMut, parentSeqsMut, 0, depth);
+    return new IdList(root, leafMapMut.value, parentSeqsMut.value);
   }
 }
 
 /**
- * A live-updating view of an IdList that treats all known ids as present.
- * That is, this class ignores the underlying list's isDeleted status when computing list indices.
+ * A view of an IdList that treats all known ids as present.
+ * That is, this class ignores the underlying list's is-deleted status when computing list indices.
+ * Access using {@link IdList.knownIds}.
  *
- * To mutate, call methods on the original IdList (`this.list`).
+ * Like IdList, KnownIdView is immutable. To mutate, use a mutating method on the original IdList
+ * and access the returned list's `knownIds`.
  */
 export class KnownIdView {
   /**
    * Internal use only. Use {@link IdList.knownIds} instead.
    */
-  constructor(readonly list: IdList, private readonly state: ListElement[]) {}
+  constructor(readonly list: IdList, private readonly root: InnerNode) {}
 
   // Mutators are omitted - mutate this.list instead.
 
@@ -413,14 +921,73 @@ export class KnownIdView {
       throw new Error(`Index out of bounds: ${index} (length: ${this.length}`);
     }
 
-    return this.state[index].id;
+    let remaining = index;
+    let curParent = this.root;
+    // eslint-disable-next-line no-constant-condition
+    recurse: while (true) {
+      if (curParent instanceof InnerNodeInner) {
+        for (const child of curParent.children) {
+          if (remaining < child.knownSize) {
+            // Recurse.
+            curParent = child;
+            continue recurse;
+          } else {
+            remaining -= child.knownSize;
+          }
+        }
+      } else {
+        for (const child of curParent.children) {
+          if (remaining < child.count) {
+            // Found it.
+            return {
+              bunchId: child.bunchId,
+              counter: child.startCounter + remaining,
+            };
+          } else {
+            remaining -= child.count;
+          }
+        }
+      }
+
+      throw new Error("Internal error");
+    }
   }
 
   /**
    * Returns the index of `id` in this view, or -1 if it is not known.
    */
   indexOf(id: ElementId): number {
-    return this.state.findIndex((elt) => equalsId(elt.id, id));
+    // @ts-expect-error Ignore private
+    const located = this.list.locate(id);
+    if (located === null) throw new Error("id is not known");
+
+    /**
+     * The number of present ids less than id.
+     * Equivalently, the index id would have if present.
+     */
+    let index = 0;
+
+    // Lesser siblings of parent, grandparent, etc.
+    for (let i = 1; i < located.length; i++) {
+      const parent = (
+        i === located.length - 1 ? this.root : located[i + 1].node
+      ) as InnerNodeInner;
+      for (let c = 0; c < located[i].indexInParent; c++) {
+        index += parent.children[c].knownSize;
+      }
+    }
+
+    // Siblings of id's leaf.
+    const leafParent = (
+      located.length === 1 ? this.root : located[1].node
+    ) as InnerNodeLeaf;
+    for (let c = 0; c < located[0].indexInParent; c++) {
+      const child = leafParent.children[c];
+      index += child.count;
+    }
+
+    // id's index with leaf.
+    return index + (id.counter - located[0].node.startCounter);
   }
 
   /**
@@ -429,7 +996,7 @@ export class KnownIdView {
    * Equivalently, the number of known ids in `this.list`.
    */
   get length(): number {
-    return this.state.length;
+    return this.root.knownSize;
   }
 
   // Iterators
@@ -437,10 +1004,8 @@ export class KnownIdView {
   /**
    * Iterates over all ids in this view, i.e., all known ids in `this.list`.
    */
-  *[Symbol.iterator](): IterableIterator<ElementId> {
-    for (const elt of this.state) {
-      yield elt.id;
-    }
+  [Symbol.iterator](): IterableIterator<ElementId> {
+    return iterateNode(this.root, true);
   }
 
   /**
@@ -451,21 +1016,331 @@ export class KnownIdView {
   }
 }
 
-function expandElements(
-  startId: ElementId,
-  isDeleted: boolean,
-  count: number
-): ListElement[] {
-  if (!(Number.isSafeInteger(count) && count >= 0)) {
-    throw new Error(`Invalid count: ${count}`);
+/**
+ * Returns the first (leftmost) known ElementId in node's subtree.
+ */
+function firstId(node: InnerNode): ElementId {
+  let currentInner = node;
+  while (!(currentInner instanceof InnerNodeLeaf)) {
+    currentInner = currentInner.children[0];
   }
+  const firstLeaf = currentInner.children[0];
+  return {
+    bunchId: firstLeaf.bunchId,
+    counter: firstLeaf.startCounter,
+  };
+}
+
+/**
+ * Returns the last (rightmost) known ElementId in node's subtree.
+ */
+function lastId(node: InnerNode): ElementId {
+  let currentInner = node;
+  while (!(currentInner instanceof InnerNodeLeaf)) {
+    currentInner = currentInner.children.at(-1)!;
+  }
+  const lastLeaf = currentInner.children.at(-1)!;
+  return {
+    bunchId: lastLeaf.bunchId,
+    counter: lastLeaf.startCounter + lastLeaf.count - 1,
+  };
+}
+
+/**
+ * Replace located[i].node with newNodes.
+ *
+ * newNodes.length must be in [1, M].
+ *
+ * The returned node's descendants are recorded in leafMapMut and parentSeqsMut,
+ * but the node itself is not (since we don't know its parent here).
+ */
+function replaceNode(
+  located: Located,
+  root: InnerNode,
+  leafMapMut: MutableLeafMap,
+  parentSeqsMut: MutableSeqMap,
+  newNodes: InnerNode[] | LeafNode[],
+  i: number
+): InnerNode {
+  const parent =
+    i === located.length - 1 ? root : (located[i + 1].node as InnerNode);
+  const indexInParent = located[i].indexInParent;
+  // Copy-on-write version of parent.children.splice(indexInParent, 1, ...newNodes)
+  const newChildren = parent.children
+    .slice(0, indexInParent)
+    .concat(newNodes, parent.children.slice(indexInParent + 1));
+
+  if (newChildren.length > M) {
+    // Split the parent to maintain BTree property (# children <= M).
+    // Treat the right parent as "new", getting a new seq.
+    const split = Math.ceil(newChildren.length / 2);
+    const seqs = [parent.seq, getAndBumpNextSeq(parentSeqsMut)];
+    const newParents = [
+      newChildren.slice(0, split),
+      newChildren.slice(split),
+    ].map((children, j) =>
+      i === 0
+        ? new InnerNodeLeaf(seqs[j], children as LeafNode[], leafMapMut)
+        : new InnerNodeInner(seqs[j], children as InnerNode[], parentSeqsMut)
+    );
+    if (i === located.length - 1) {
+      // newParents replace root. We need a new root to hold them.
+      return new InnerNodeInner(
+        getAndBumpNextSeq(parentSeqsMut),
+        newParents,
+        parentSeqsMut
+      );
+    } else {
+      return replaceNode(
+        located,
+        root,
+        leafMapMut,
+        parentSeqsMut,
+        newParents,
+        i + 1
+      );
+    }
+  } else {
+    // "Replace" parent, reusing its seq.
+    // To avoid doing newChildren.length sets every time (which makes replaceLeaf
+    // do >=(M/2)*log(L) total sets, even when none were necessary),
+    // we bypass the InnerNode constructors' leafMap/parentSeq operations,
+    // instead doing them ourselves only on the changed children.
+    let newParent: InnerNode;
+    if (i === 0) {
+      newParent = new InnerNodeLeaf(
+        parent.seq,
+        newChildren as LeafNode[],
+        null
+      );
+      // Important to delete the replaced leaf's entry, so that it doesn't corrupt by-ElementId searches.
+      leafMapMut.value = leafMapMut.value.delete(located[0].node);
+      for (const newNode of newNodes as LeafNode[]) {
+        leafMapMut.value = leafMapMut.value.set(newNode, parent.seq);
+      }
+    } else {
+      newParent = new InnerNodeInner(
+        parent.seq,
+        newChildren as InnerNode[],
+        null
+      );
+      for (const newNode of newNodes as InnerNode[]) {
+        if (newNode.seq !== (located[i].node as InnerNode).seq) {
+          parentSeqsMut.value = parentSeqsMut.value.set(
+            newNode.seq,
+            parent.seq
+          );
+        }
+      }
+      // If the replaced node isn't represented in newNodes (i.e., same seq is not reused),
+      // we could delete its entry to save memory, but it is not necessary.
+    }
+
+    if (i === located.length - 1) {
+      // Replaces root.
+      return newParent;
+    } else {
+      return replaceNode(
+        located,
+        root,
+        leafMapMut,
+        parentSeqsMut,
+        [newParent],
+        i + 1
+      );
+    }
+  }
+}
+
+/**
+ * Splits present into two SparseIndices at the given counter.
+ */
+function splitPresent(
+  present: SparseIndices,
+  splitCounter: number
+): [leftPresent: SparseIndices, rightPresent: SparseIndices] {
+  const leftPresent = SparseIndices.new();
+  const rightPresent = SparseIndices.new();
+  const leafSlicer = present.newSlicer();
+  for (const [index, count] of leafSlicer.nextSlice(splitCounter)) {
+    leftPresent.set(index, count);
+  }
+  for (const [index, count] of leafSlicer.nextSlice(null)) {
+    rightPresent.set(index, count);
+  }
+  return [leftPresent, rightPresent];
+}
 
-  const ans: ListElement[] = [];
-  for (let i = 0; i < count; i++) {
-    ans.push({
-      id: { bunchId: startId.bunchId, counter: startId.counter + i },
-      isDeleted,
-    });
+function* iterateNode(
+  node: InnerNode,
+  includeDeleted: boolean
+): IterableIterator<ElementId> {
+  if (node instanceof InnerNodeInner) {
+    for (const child of node.children) {
+      yield* iterateNode(child, includeDeleted);
+    }
+  } else {
+    for (const child of node.children) {
+      if (includeDeleted) {
+        for (let i = 0; i < child.count; i++) {
+          yield { bunchId: child.bunchId, counter: child.startCounter + i };
+        }
+      } else {
+        for (const counter of child.present.keys()) {
+          yield { bunchId: child.bunchId, counter };
+        }
+      }
+    }
+  }
+}
+
+function* iterateNodeWithIsDeleted(
+  node: InnerNode
+): IterableIterator<{ id: ElementId; isDeleted: boolean }> {
+  if (node instanceof InnerNodeInner) {
+    for (const child of node.children) {
+      yield* iterateNodeWithIsDeleted(child);
+    }
+  } else {
+    for (const child of node.children) {
+      let nextIndex = child.startCounter;
+      for (const index of child.present.keys()) {
+        while (nextIndex < index) {
+          yield {
+            id: { bunchId: child.bunchId, counter: nextIndex },
+            isDeleted: true,
+          };
+          nextIndex++;
+        }
+        yield {
+          id: { bunchId: child.bunchId, counter: index },
+          isDeleted: false,
+        };
+        nextIndex++;
+      }
+      while (nextIndex < child.startCounter + child.count) {
+        yield {
+          id: { bunchId: child.bunchId, counter: nextIndex },
+          isDeleted: true,
+        };
+        nextIndex++;
+      }
+    }
+  }
+}
+
+/**
+ * Updates acc to account for node's subtree, as part of a depth-first search
+ * in list order.
+ */
+function saveNode(node: InnerNode, acc: SavedIdList) {
+  if (node instanceof InnerNodeInner) {
+    for (const child of node.children) {
+      saveNode(child, acc);
+    }
+  } else {
+    for (const child of node.children) {
+      let nextIndex = child.startCounter;
+      for (const [index, count] of child.present.items()) {
+        if (nextIndex < index) {
+          // Need a deleted item.
+          pushSaveItem(acc, {
+            bunchId: child.bunchId,
+            startCounter: nextIndex,
+            count: index - nextIndex,
+            isDeleted: true,
+          });
+        }
+        pushSaveItem(acc, {
+          bunchId: child.bunchId,
+          startCounter: index,
+          count,
+          isDeleted: false,
+        });
+        nextIndex = index + count;
+      }
+      if (nextIndex < child.startCounter + child.count) {
+        pushSaveItem(acc, {
+          bunchId: child.bunchId,
+          startCounter: nextIndex,
+          count: child.startCounter + child.count - nextIndex,
+          isDeleted: true,
+        });
+      }
+    }
+  }
+}
+
+/**
+ * Pushes a save item onto acc, combing it with the previous item if possible.
+ *
+ * This function is necessary because we don't guarantee that adjacent leaves are fully merged.
+ * Specifically, if you insert a bunch's ids with counter values (0, 2, 1)
+ * in that order, then counter 1 will extend one of the existing leaves
+ * but not merge with the other leaf.
+ *
+ * This situation won't appear in typical usage, and its perf penalty
+ * will go away once you reload. Thus we tolerate it instead of figuring out
+ * how to delete leaves from a B+Tree.
+ */
+function pushSaveItem(acc: SavedIdList, item: SavedIdList[number]) {
+  if (acc.length > 0) {
+    const previous = acc.at(-1)!;
+    if (
+      previous.isDeleted === item.isDeleted &&
+      previous.bunchId === item.bunchId &&
+      previous.startCounter + previous.count === item.startCounter
+    ) {
+      // Combine items.
+      // @ts-expect-error Mutating for convenience; no aliasing to worry about.
+      previous.count += item.count;
+      return;
+    }
+  }
+  acc.push(item);
+}
+
+/**
+ * Builds a tree with the given leaves. Used by IdList.load.
+ *
+ * The returned node's descendants are recorded in leafMapMut and parentSeqsMut,
+ * but not the node itself (since we don't know its parent here).
+ *
+ * In contrast to inserting the leaves one-by-one, this function fills nodes
+ * with M children whenever possible,
+ * and the B+Tree parts run in O(L) time instead of O(L log(L)).
+ * However, the overall runtime is O(L log(L)) from constructing the sorted leafMap.
+ */
+function buildTree(
+  leaves: LeafNode[],
+  leafMapMut: MutableLeafMap,
+  parentSeqsMut: MutableSeqMap,
+  startIndex: number,
+  depthRemaining: number
+): InnerNode {
+  const parentSeq = getAndBumpNextSeq(parentSeqsMut);
+  if (depthRemaining === 1) {
+    return new InnerNodeLeaf(
+      parentSeq,
+      leaves.slice(startIndex, startIndex + M),
+      leafMapMut
+    );
+  } else {
+    const children: InnerNode[] = [];
+    const childLeafCount = Math.pow(M, depthRemaining - 1);
+    for (let i = 0; i < M; i++) {
+      const childStartIndex = startIndex + i * childLeafCount;
+      if (childStartIndex >= leaves.length) break;
+      children.push(
+        buildTree(
+          leaves,
+          leafMapMut,
+          parentSeqsMut,
+          childStartIndex,
+          depthRemaining - 1
+        )
+      );
+    }
+    return new InnerNodeInner(parentSeq, children, parentSeqsMut);
   }
-  return ans;
 }