articulated 0.1.0 → 0.3.0

package/README.md

@@ -23,22 +23,23 @@ yarn add articulated
 ```typescript
 import { IdList } from "articulated";
 
-// Create an empty list
-const list = new IdList();
+// Create an empty list.
+let list = IdList.new();
 
-// Insert a new element at the beginning
-list.insertAfter(null, { bunchId: "user1", counter: 0 });
+// Insert a new ElementId at the beginning.
+// Note: Persistent (immutable) data structure! Mutators return a new IdList.
+list = list.insertAfter(null, { bunchId: "user1", counter: 0 });
 
-// Insert another element after the first
-list.insertAfter(
+// Insert another ElementId after the first.
+list = list.insertAfter(
   { bunchId: "user1", counter: 0 },
   { bunchId: "user1", counter: 1 }
 );
 
-// Delete an element (marks as deleted but keeps as known)
-list.delete({ bunchId: "user1", counter: 0 });
+// Delete an ElementId (marks as deleted but keeps as known).
+list = list.delete({ bunchId: "user1", counter: 0 });
 
-// Check if elements are present/known
+// Check if ElementIds are present/known.
 console.log(list.has({ bunchId: "user1", counter: 0 })); // false (deleted)
 console.log(list.isKnown({ bunchId: "user1", counter: 0 })); // true (known but deleted)
 ```
@@ -50,9 +51,9 @@ console.log(list.isKnown({ bunchId: "user1", counter: 0 })); // true (known but
 An `ElementId` is a globally unique identifier for a list element, composed of:
 
 - `bunchId`: A string UUID or similar globally unique ID
-- `counter`: A numeric value to distinguish elements in the same bunch
+- `counter`: A numeric value to distinguish ElementIds in the same bunch
 
-For optimal compression, when inserting multiple elements in sequence, use the same `bunchId` with sequential `counter` values.
+For optimal compression, when inserting multiple ElementIds in a left-to-right sequence, use the same `bunchId` with sequential `counter` values.
 
 ```typescript
 // Example of IDs that will compress well
@@ -63,39 +64,38 @@ const id3 = { bunchId: "abc123", counter: 2 };
 
 ### IdList Operations
 
+To enable easy and efficient rollbacks, such as in a [server reconciliation](https://mattweidner.com/2024/06/04/server-architectures.html#1-server-reconciliation) architecture, IdList is a persistent (immutable) data structure. Mutating methods return a new IdList, sharing memory with the old IdList where possible.
+
 #### Basic Operations
 
-- `insertAfter(before, newId)`: Insert after a specific element
-- `insertBefore(after, newId)`: Insert before a specific element
-- `delete(id)`: Mark an element as deleted (remains known)
-- `undelete(id)`: Restore a deleted element
+- `insertAfter(before, newId): IdList`: Insert after a specific ElementId
+- `insertBefore(after, newId): IdList`: Insert before a specific ElementId
+- `delete(id): IdList`: Mark an ElementId as deleted (remains known)
+- `undelete(id): IdList`: Restore a deleted ElementId
 
-#### Advanced Operations
+#### Basic Accessors
 
-- `uninsert(id)`: Remove an element completely (no longer known)
-- `at(index)`: Get the element ID at a specific index
-- `indexOf(id, bias)`: Get the index of an element with optional bias for deleted elements
-- `clone()`: Create a deep copy of the list
+- `at(index)`: Get the ElementId at a specific index
+- `indexOf(id, bias: "none" | "left" | "right" = "none")`: Get the index of an ElementId, with optional bias for deleted-but-known ElementIds
 
 #### Bulk Operations
 
 ```typescript
 // Insert multiple sequential ids at once
-list.insertAfter(null, { bunchId: "user1", counter: 0 }, 5);
+list = list.insertAfter(null, { bunchId: "user1", counter: 0 }, 5);
 // Inserts 5 ids with bunchId="user1" and counters 0, 1, 2, 3, 4
 ```
 
-### Persistence
+#### Save and load
 
-Save and restore the list state:
+Save and load the list state in JSON form:
 
 ```typescript
 // Save list state
 const savedState = list.save();
 
-// Later, restore from saved state
-const newList = new IdList();
-newList.load(savedState);
+// Later, load from saved state
+let newList = IdList.load(savedState);
 ```
 
 ## Use Cases
@@ -104,3 +104,30 @@ newList.load(savedState);
 - Todo lists with collaborative editing
 - Any list where elements' positions change but need stable identifiers
 - Conflict-free replicated data type (CRDT) implementations
+
+## Internals
+
+IdList stores its state as a modified [B+Tree](https://en.wikipedia.org/wiki/B%2B_tree), described at the top of [its source code](./src/id_list.ts). Each leaf in the B+Tree represents multiple ElementIds (sharing a bunchId and sequential counters) in a compressed way; for normal collaborative text editing, expect 10-20 ElementIds per leaf.
+
+To speed up searches, we also maintain a "bottom-up" tree that maps from each node to a sequence number identifying its parent. (Using sequence numbers instead of pointers is necessary for persistence.) The map is implemented using persistent balanced trees from [functional-red-black-tree](https://www.npmjs.com/package/functional-red-black-tree).
+
+Asymptotic runtimes are given in terms of the number of leaves `L` and the maximum "fragmentation" of a leaf `F`, which is the number of times its ElementIds alternate between deleted vs present.
+
+- insertAfter, insertBefore: `O(log^2(L) + F)`.
+  - The bottleneck is finding the B+Tree path of the before/after ElementId. This requires `O(log(L))` lookups in the bottom-up tree's map, each of which takes `O(log(L))` time. See the implementation of `IdList.locate`.
+- delete, undelete: `O(log^2(L) + F)`.
+- indexOf: `O(log^2(L) + F)`.
+  - Bottleneck is locating the id.
+- at: `O(log(L) + F)`.
+  - Simple B+Tree search.
+- has, isKnown: `O(log(L) + F)`
+  - Part of the bottom-up tree is a sorted map with leaf keys; due to the sort, we can also use that map to look up the leaf corresponding to an ElementId, in `O(log(L))` time.
+- length: `O(1)`.
+  - Cached.
+- save: `O(S + L)`, where `S <= L * F` is the saved state's length.
+- load: `O(S * log(S))`
+  - The bottleneck is constructing the bottom-up tree: specifically, the map from each leaf to its parent's sequence number (`leafMap`). That map is itself a sorted tree, hence takes `O(L * log(L))` time to construct, and `L <= S`.
+
+If you want to get a sense of what IdList is or how to implement your own version, consider reading the source code for [IdListSimple](./test/id_list_simple.ts), which behaves identically to IdList. It is short (<300 SLOC) and direct, using an array and `Array.splice`. The downside is that IdListSimple does not compress ElementIds, and all of its operations take `O(# ids)` time. We use it as a known-good implementation in our fuzz tests.
+
+<!-- TODO: related work: CRDTs, ropes, list-positions, ?? -->

package/build/commonjs/id.d.ts

@@ -3,12 +3,12 @@
  *
  * ElementIds are conceptually the same as UUIDs (or nanoids, etc.).
  * However, when a single thread generates a series of ElementIds, you are
- * allowed to optimize by generating a single UUID/nanoid/etc. and using that as the "bunchId"
+ * allowed to optimize by generating a single UUID/nanoid/etc. and using that as the `bunchId`
  * for a "bunch" of elements, with varying `counter`.
  * The resulting ElementIds compress better than a set of UUIDs, but they are
- * still globally unique, even if another thread/user/device generates ElementIds concurrently.
+ * still globally unique, even if another thread/device/user generates ElementIds concurrently.
  *
- * For example, if a user types a sentence from left to right, you may generate a
+ * For example, if a user types a sentence from left to right, you can generate a
  * single `bunchId` and assign their characters the sequential ElementIds
  * `{ bunchId, counter: 0 }, { bunchId, counter: 1 }, { bunchId, counter: 2 }, ...`.
  * An IdList will store all of these as a single object instead of
@@ -31,9 +31,6 @@ export interface ElementId {
      * IdList is optimized for this case, but it is not mandatory.
      * In particular, it is okay if future edits cause the sequential ids to be
      * separated, partially deleted, or even reordered.
-     *
-     * Negative integers are supported by IdList (e.g., for optimized right-to-left insertions),
-     * though you may choose to avoid these in your application, to make serialization easier.
      */
     readonly counter: number;
 }

package/build/commonjs/id.js.map

@@ -1 +1 @@
-{"version":3,"file":"id.js","sourceRoot":"","sources":["../../src/id.ts"],"names":[],"mappings":";;;AAwCA;;GAEG;AACH,SAAgB,QAAQ,CAAC,CAAY,EAAE,CAAY;IACjD,OAAO,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,CAAC;AAC5D,CAAC;AAFD,4BAEC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAgB,SAAS,CAAC,OAAkB,EAAE,KAAa;IACzD,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAVD,8BAUC"}
+{"version":3,"file":"id.js","sourceRoot":"","sources":["../../src/id.ts"],"names":[],"mappings":";;;AAqCA;;GAEG;AACH,SAAgB,QAAQ,CAAC,CAAY,EAAE,CAAY;IACjD,OAAO,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,CAAC;AAC5D,CAAC;AAFD,4BAEC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAgB,SAAS,CAAC,OAAkB,EAAE,KAAa;IACzD,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAVD,8BAUC"}

package/build/commonjs/id_list.d.ts

@@ -1,11 +1,79 @@
+import { SparseIndices } from "sparse-array-rled";
 import { ElementId } from "./id";
+import { MutableLeafMap } from "./internal/leaf_map";
+import { MutableSeqMap } from "./internal/seq_map";
 import { SavedIdList } from "./saved_id_list";
-interface ListElement {
-    id: ElementId;
-    isDeleted: boolean;
+export interface LeafNode {
+    readonly bunchId: string;
+    readonly startCounter: number;
+    readonly count: number;
+    /**
+     * The present counter values in this leaf node.
+     *
+     * Note that it is indexed by counter, not by (counter - this.startCounter).
+     */
+    readonly present: SparseIndices;
+}
+/**
+ * An inner node with inner-node children.
+ */
+export declare class InnerNodeInner {
+    /**
+     * A unique identifer for this node within its IdTree.
+     */
+    readonly seq: number;
+    readonly children: readonly InnerNode[];
+    readonly size: number;
+    readonly knownSize: number;
+    constructor(
+    /**
+     * A unique identifer for this node within its IdTree.
+     */
+    seq: number, children: readonly InnerNode[], 
+    /**
+     * We add entries for the children to this map, overwriting any existing parentSeqs.
+     *
+     * Pass null to skip when you are doing it yourself. Regardless, you need to
+     * delete any outdated entries yourself.
+     */
+    parentSeqsMut: MutableSeqMap | null);
 }
 /**
- * A list of ElementIds.
+ * An inner node with leaf children.
+ */
+export declare class InnerNodeLeaf {
+    /**
+     * A unique identifer for this node within its IdTree.
+     */
+    readonly seq: number;
+    readonly children: readonly LeafNode[];
+    readonly size: number;
+    readonly knownSize: number;
+    constructor(
+    /**
+     * A unique identifer for this node within its IdTree.
+     */
+    seq: number, children: readonly LeafNode[], 
+    /**
+     * We add entries for the children to this map, overwriting any existing parentSeqs.
+     *
+     * Pass null to skip when you are doing it yourself.
+     */
+    leafMapMut: MutableLeafMap | null);
+}
+export type InnerNode = InnerNodeInner | InnerNodeLeaf;
+/**
+ * The B+Tree's branching factor, i.e., the max number of children of a node.
+ *
+ * Note that our B+Tree has no keys - in particular, no keys in internal nodes.
+ *
+ * Wiki B+Tree: "B+ trees can also be used for data stored in RAM.
+ * In this case a reasonable choice for block size would be the size of [the] processor's cache line."
+ * (64 byte cache line) / (8 byte pointer) = 8.
+ */
+export declare const M = 8;
+/**
+ * A list of ElementIds, as a persistent (immutable) data structure.
  *
  * An IdList helps you assign a unique immutable id to each element of a list, such
  * as a todo-list or a text document (= list of characters). That way, you can keep track
@@ -14,11 +82,14 @@ interface ListElement {
  *
  * Any id that has been inserted into an IdList remains **known** to that list indefinitely,
  * allowing you to reference it in insertAfter/insertBefore operations. Calling {@link delete}
- * merely marks an id as deleted (not present); it remains in memory as a "tombstone".
+ * merely marks an id as deleted (= not present); a deleted id does not count towards the length of the list or index-based accessors, but it does remain in memory as a "tombstone".
  * This is useful in collaborative settings, since another user might instruct you to
  * call `insertAfter(before, newId)` when you have already deleted `before` locally.
- * If that is not a concern and you truly want to make an id no longer known, instead
- * call {@link uninsert}.
+ *
+ * To enable easy and efficient rollbacks, such as in a
+ * [server reconciliation](https://mattweidner.com/2024/06/04/server-architectures.html#1-server-reconciliation)
+ * architecture, IdList is a persistent (immutable) data structure. Mutating methods
+ * return a new IdList, sharing memory with the old IdList where possible.
  *
  * See {@link ElementId} for advice on generating ElementIds. IdList is optimized for
  * the case where sequential ElementIds often have the same bunchId and sequential counters.
@@ -26,18 +97,34 @@ interface ListElement {
  * cause such ids to be separated, partially deleted, or even reordered.
  */
 export declare class IdList {
-    private readonly state;
-    private _length;
+    private readonly root;
+    /**
+     * A persistent sorted map from each leaf to its parent node's seq.
+     *
+     * Besides parentSeqs, we also use this to lookup leaves by ElementId.
+     */
+    private readonly leafMap;
+    /**
+     * A persistent map from each InnerNode's seq to its parent node's seq.
+     *
+     * We map the root's seq to 0 (in our constructor).
+     */
+    private readonly parentSeqs;
+    /**
+     * Internal - construct an IdList using a static method (e.g. `IdList.new`).
+     */
+    private constructor();
     /**
      * Constructs an empty list.
      *
-     * To begin with a non-empty list, use {@link IdList.from} or {@link IdList.fromIds}.
+     * To begin with a non-empty list, use {@link IdList.from}, {@link IdList.fromIds},
+     * or {@link IdList.load}.
      */
-    constructor();
+    static new(): IdList;
     /**
      * Constructs a list with the given known ids and their isDeleted status, in list order.
      */
-    static from(state: Iterable<{
+    static from(knownIds: Iterable<{
         id: ElementId;
         isDeleted: boolean;
     }>): IdList;
@@ -51,6 +138,8 @@ export declare class IdList {
     static fromIds(ids: Iterable<ElementId>): IdList;
     /**
      * Inserts `newId` immediately after the given id (`before`), which may be deleted.
+     * A new IdList is returned and the current list remains unchanged.
+     *
      * All ids to the right of `before` are shifted one index to the right, in the manner
      * of [Array.splice](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice).
      *
@@ -60,11 +149,13 @@ export declare class IdList {
      * @param count Provide this to bulk-insert `count` ids from left-to-right,
      * starting with newId and proceeding with the same bunchId and sequential counters.
      * @throws If `before` is not known.
-     * @throws If `newId` is already known.
+     * @throws If any inserted id is already known.
      */
-    insertAfter(before: ElementId | null, newId: ElementId, count?: number): void;
+    insertAfter(before: ElementId | null, newId: ElementId, count?: number): IdList;
     /**
      * Inserts `newId` immediately before the given id (`after`), which may be deleted.
+     * A new IdList is returned and the current list remains unchanged.
+     *
      * All ids to the right of `after`, plus `after` itself, are shifted one index to the right, in the manner
      * of [Array.splice](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice).
      *
@@ -77,39 +168,45 @@ export declare class IdList {
      * @throws If `after` is not known.
      * @throws If `newId` is already known.
      */
-    insertBefore(after: ElementId | null, newId: ElementId, count?: number): void;
+    insertBefore(after: ElementId | null, newId: ElementId, count?: number): IdList;
     /**
-     * Un-inserts `id` from the list, making it no longer known or present in this list.
+     * Marks `id` as deleted from this list.
+     * A new IdList is returned and the current list remains unchanged.
      *
-     * Typically, you instead want to call {@link delete}, which marks `id` as deleted while
-     * it remains known. That way, you can reference `id` in future insertAfter/insertBefore
+     * Once deleted, `id` does not count towards the length of the list or index-based accessors.
+     * However, it remains known (a "tombstone").
+     * Because `id` is still known, you can reference it in future insertAfter/insertBefore
      * operations, including ones sent concurrently by other devices.
+     * This does have a memory cost, but it is compressed in common cases.
      *
-     * If `id` is already not known, this method does nothing.
+     * If `id` is already deleted or is not known, this method does nothing.
      */
-    uninsert(id: ElementId): void;
+    delete(id: ElementId): IdList;
     /**
-     * Marks `id` as deleted from this list. The id remains known (a "tombstone").
+     * Un-marks `id` as deleted from this list, making it present again.
+     * A new IdList is returned and the current list remains unchanged.
      *
-     * Because `id` is still known, you can reference it in future insertAfter/insertBefore
-     * operations, including ones sent concurrently by other devices.
-     * However, it does occupy space in memory (compressed in common cases).
+     * This method is an exact inverse to {@link delete}.
      *
-     * For an exact inverse to `insertAfter(-, id)` or `insertBefore(-, id)`
-     * that makes `id` no longer known, see {@link uninsert}.
+     * If `id` is already present, this method does nothing.
      *
-     * If `id` is already deleted or not known, this method does nothing.
+     * @throws If `id` is not known.
      */
-    delete(id: ElementId): void;
+    undelete(id: ElementId): IdList;
     /**
-     * Un-marks `id` as deleted from this list, making it present again.
-     * This is an exact inverse to {@link delete}.
+     * Returns the path from id's leaf node to the root, or null if id is not found.
      *
-     * If `id` is already present, this method does nothing.
+     * The path contains each node and its index in its parent's node, starting with id's
+     * LeafNode and ending at a child of the root.
+     */
+    private locate;
+    /**
+     * Replaces the leaf at the given path with newLeaves.
+     * Returns a proper (sufficiently balanced) B+Tree with updated sizes.
      *
-     * @throws If `id` is not known.
+     * newLeaves.length must be in [1, M].
      */
-    undelete(id: ElementId): void;
+    private replaceLeaf;
     /**
      * Returns whether `id` is present in the list, i.e., it is known and not deleted.
      *
@@ -124,6 +221,16 @@ export declare class IdList {
      * Compare to {@link has}.
      */
     isKnown(id: ElementId): boolean;
+    /**
+     * Returns true if any of the given bulk ids are known.
+     */
+    private isAnyKnown;
+    /**
+     * The length of the list, counting only present ids.
+     *
+     * To include known but deleted ids, use `this.knownIds.length`.
+     */
+    get length(): number;
     /**
      * Returns the id at the given index in the list.
      *
@@ -142,10 +249,6 @@ export declare class IdList {
      * @throws If `id` is not known.
      */
     indexOf(id: ElementId, bias?: "none" | "left" | "right"): number;
-    /**
-     * The length of the list.
-     */
-    get length(): number;
     /**
      * Iterates over all present ids in the list.
      */
@@ -157,18 +260,14 @@ export declare class IdList {
     /**
      * Iterates over all __known__ ids in the list, indicating which are deleted.
      */
-    valuesWithDeleted(): IterableIterator<{
+    valuesWithIsDeleted(): IterableIterator<{
         id: ElementId;
         isDeleted: boolean;
     }>;
-    /**
-     * Returns an independent copy of this list, including known but deleted ids.
-     */
-    clone(): IdList;
     private _knownIds?;
     /**
-     * A live-updating view of this list that treats all known ids as present.
-     * That is, it ignores isDeleted status when computing list indices or iterating.
+     * A view of this list that treats all known ids as present.
+     * That is, it ignores is-deleted status when computing list indices or iterating.
      */
     get knownIds(): KnownIdView;
     /**
@@ -179,23 +278,25 @@ export declare class IdList {
      */
     save(): SavedIdList;
     /**
-     * Loads a saved state returned by {@link save}, __overwriting__ the current state of this list.
+     * Loads a saved state returned by {@link save}.
      */
-    load(savedState: SavedIdList): void;
+    static load(savedState: SavedIdList): IdList;
 }
 /**
- * A live-updating view of an IdList that treats all known ids as present.
- * That is, this class ignores the underlying list's isDeleted status when computing list indices.
+ * A view of an IdList that treats all known ids as present.
+ * That is, this class ignores the underlying list's is-deleted status when computing list indices.
+ * Access using {@link IdList.knownIds}.
  *
- * To mutate, call methods on the original IdList (`this.list`).
+ * Like IdList, KnownIdView is immutable. To mutate, use a mutating method on the original IdList
+ * and access the returned list's `knownIds`.
  */
 export declare class KnownIdView {
     readonly list: IdList;
-    private readonly state;
+    private readonly root;
     /**
      * Internal use only. Use {@link IdList.knownIds} instead.
      */
-    constructor(list: IdList, state: ListElement[]);
+    constructor(list: IdList, root: InnerNode);
     /**
      * Returns the id at the given index in this view.
      *
@@ -223,4 +324,3 @@ export declare class KnownIdView {
      */
     values(): IterableIterator<ElementId>;
 }
-export {};