@loro-extended/change 5.3.0 → 5.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loro-extended/change",
-  "version": "5.3.0",
+  "version": "5.4.0",
   "description": "A schema-driven, type-safe wrapper for Loro CRDT that provides natural JavaScript syntax for collaborative data mutations",
   "author": "Duane Johnson",
   "license": "MIT",
@@ -25,13 +25,14 @@
     "tsup": "^8.5.0",
     "tsx": "^4.20.3",
     "typescript": "^5.9.2",
-    "vitest": "^3.2.4"
+    "vitest": "^4.0.17"
   },
   "peerDependencies": {
     "loro-crdt": "^1.10.3"
   },
   "scripts": {
     "build": "tsup",
+    "test": "verify logic",
     "verify": "verify"
   }
 }

package/src/conversion.ts

@@ -108,17 +108,53 @@ function convertStructInput(
   }
 
   const map = new LoroMap()
-  for (const [k, v] of Object.entries(value)) {
+
+  // Iterate over schema keys to ensure all nested containers are materialized
+  for (const k of Object.keys(shape.shapes)) {
     const nestedSchema = shape.shapes[k]
-    if (nestedSchema) {
+    const v = value[k]
+
+    if (v !== undefined) {
       const convertedValue = convertInputToRef(v, nestedSchema)
       if (isContainer(convertedValue)) {
         map.setContainer(k, convertedValue)
       } else {
         map.set(k, convertedValue)
       }
-    } else {
-      map.set(k, value)
+    } else if (isContainerShape(nestedSchema)) {
+      // If value is missing but it's a container shape, create an empty container
+      // This ensures deterministic container IDs across peers
+      let emptyValue: any
+      if (nestedSchema._type === "struct" || nestedSchema._type === "record") {
+        emptyValue = {}
+      } else if (
+        nestedSchema._type === "list" ||
+        nestedSchema._type === "movableList"
+      ) {
+        emptyValue = []
+      } else if (nestedSchema._type === "text") {
+        emptyValue = ""
+      } else if (nestedSchema._type === "counter") {
+        emptyValue = 0
+      }
+
+      if (emptyValue !== undefined) {
+        const convertedValue = convertInputToRef(emptyValue, nestedSchema)
+        if (isContainer(convertedValue)) {
+          map.setContainer(k, convertedValue)
+        }
+      }
+    }
+  }
+
+  // Also handle keys present in value but not in schema (if any, though for structs this shouldn't happen ideally)
+  // But for backward compatibility or loose typing, we might want to preserve them?
+  // The original code did:
+  // if (nestedSchema) { ... } else { map.set(k, value) }
+  // So it allowed extra keys.
+  for (const [k, v] of Object.entries(value)) {
+    if (!shape.shapes[k]) {
+      map.set(k, v)
     }
   }
 

package/src/functional-helpers.ts

@@ -1,11 +1,11 @@
-import type {
-  LoroCounter,
+import {
+  type LoroCounter,
   LoroDoc,
-  LoroList,
-  LoroMap,
-  LoroMovableList,
-  LoroText,
-  LoroTree,
+  type LoroList,
+  type LoroMap,
+  type LoroMovableList,
+  type LoroText,
+  type LoroTree,
 } from "loro-crdt"
 import { loro } from "./loro.js"
 import type {
@@ -274,6 +274,52 @@ export function getLoroContainer(
   return loro(ref as any).container
 }
 
+/**
+ * Creates a new TypedDoc as a fork of the current document.
+ * The forked doc contains all history up to the current version.
+ * The forked doc has a different PeerID from the original by default.
+ *
+ * For raw LoroDoc access, use: `loro(doc).doc.fork()`
+ *
+ * @param doc - The TypedDoc to fork
+ * @param options - Optional settings
+ * @param options.preservePeerId - If true, copies the original doc's peer ID to the fork
+ * @returns A new TypedDoc with the same schema at the current version
+ *
+ * @example
+ * ```typescript
+ * import { fork, loro } from "@loro-extended/change"
+ *
+ * const doc = createTypedDoc(schema);
+ * doc.title.update("Hello");
+ *
+ * // Fork the document
+ * const forkedDoc = fork(doc);
+ * forkedDoc.title.update("World");
+ *
+ * console.log(doc.title.toString()); // "Hello"
+ * console.log(forkedDoc.title.toString()); // "World"
+ *
+ * // Fork with same peer ID (for World/Worldview pattern)
+ * const worldview = fork(world, { preservePeerId: true });
+ * ```
+ */
+export function fork<Shape extends DocShape>(
+  doc: TypedDoc<Shape>,
+  options?: { preservePeerId?: boolean },
+): TypedDoc<Shape> {
+  const loroDoc = loro(doc).doc
+  const forkedLoroDoc = loroDoc.fork()
+  const shape = loro(doc).docShape as Shape
+
+  // Optionally preserve the peer ID (useful for World/Worldview pattern)
+  if (options?.preservePeerId) {
+    forkedLoroDoc.setPeerId(loroDoc.peerId)
+  }
+
+  return createTypedDoc(shape, forkedLoroDoc)
+}
+
 /**
  * Creates a new TypedDoc at a specified version (frontiers).
  * The forked doc will only contain history before the specified frontiers.
@@ -309,3 +355,71 @@ export function forkAt<Shape extends DocShape>(
   const shape = loro(doc).docShape as Shape
   return createTypedDoc(shape, forkedLoroDoc)
 }
+
+/**
+ * Creates a new TypedDoc at a specified version using a shallow snapshot.
+ * Unlike `forkAt`, this creates a "garbage-collected" snapshot that only
+ * contains the current state and history since the specified frontiers.
+ *
+ * This is more memory-efficient than `forkAt` for documents with long history,
+ * especially useful for the fork-and-merge pattern in LEA where we only need:
+ * 1. Read current state
+ * 2. Apply changes
+ * 3. Export delta and merge back
+ *
+ * The shallow fork has a different PeerID from the original by default.
+ * Use `preservePeerId: true` to copy the original's peer ID (useful for
+ * fork-and-merge patterns where you want consistent frontier progression).
+ *
+ * @param doc - The TypedDoc to fork
+ * @param frontiers - The version to fork at (obtained from `loro(doc).doc.frontiers()`)
+ * @param options - Optional settings
+ * @param options.preservePeerId - If true, copies the original doc's peer ID to the fork
+ * @returns A new TypedDoc with the same schema at the specified version (shallow)
+ *
+ * @example
+ * ```typescript
+ * import { shallowForkAt, loro } from "@loro-extended/change"
+ *
+ * const doc = createTypedDoc(schema);
+ * doc.title.update("Hello");
+ * const frontiers = loro(doc).doc.frontiers();
+ *
+ * // Create a shallow fork (memory-efficient)
+ * const shallowDoc = shallowForkAt(doc, frontiers, { preservePeerId: true });
+ *
+ * // Modify the shallow doc
+ * shallowDoc.title.update("World");
+ *
+ * // Merge changes back
+ * const update = loro(shallowDoc).doc.export({
+ *   mode: "update",
+ *   from: loro(doc).doc.version()
+ * });
+ * loro(doc).doc.import(update);
+ * ```
+ */
+export function shallowForkAt<Shape extends DocShape>(
+  doc: TypedDoc<Shape>,
+  frontiers: Frontiers,
+  options?: { preservePeerId?: boolean },
+): TypedDoc<Shape> {
+  const loroDoc = loro(doc).doc
+  const shape = loro(doc).docShape as Shape
+
+  // Export a shallow snapshot at the specified frontiers
+  const shallowBytes = loroDoc.export({
+    mode: "shallow-snapshot",
+    frontiers,
+  })
+
+  // Create a new LoroDoc from the shallow snapshot
+  const shallowLoroDoc = LoroDoc.fromSnapshot(shallowBytes)
+
+  // Optionally preserve the peer ID for consistent frontier progression
+  if (options?.preservePeerId) {
+    shallowLoroDoc.setPeerId(loroDoc.peerId)
+  }
+
+  return createTypedDoc(shape, shallowLoroDoc)
+}

package/src/index.ts

@@ -4,13 +4,17 @@ export {
   derivePlaceholder,
   deriveShapePlaceholder,
 } from "./derive-placeholder.js"
+
 // Functional helpers (recommended API)
 export {
   change,
+  fork,
   forkAt,
   getLoroContainer,
   getLoroDoc,
+  shallowForkAt,
 } from "./functional-helpers.js"
+
 // The loro() escape hatch for CRDT internals
 export {
   LORO_SYMBOL,
@@ -23,6 +27,7 @@ export {
   type LoroTypedDocRef,
   loro,
 } from "./loro.js"
+
 export { mergeValue, overlayPlaceholder } from "./overlay.js"
 // Path selector DSL exports
 export { createPathBuilder } from "./path-builder.js"
@@ -35,43 +40,42 @@ export type {
   PathSelector,
 } from "./path-selector.js"
 export { createPlaceholderProxy } from "./placeholder-proxy.js"
+export { replayDiff } from "./replay-diff.js"
+// Shape utilities
+// Container shapes
+// Value shapes
 export type {
-  // Escape hatch shapes for untyped integration
   AnyContainerShape,
   AnyValueShape,
   ArrayValueShape,
   ContainerOrValueShape,
   ContainerShape,
   ContainerType as RootContainerType,
-  // Container shapes
   CounterContainerShape,
-  // Discriminated union for tagged unions
+  // Tagged union
   DiscriminatedUnionValueShape,
-  // Schema node types
   DocShape,
   ListContainerShape,
-  /** @deprecated Use StructContainerShape instead */
-  MapContainerShape,
   MovableListContainerShape,
-  /** @deprecated Use StructValueShape instead */
-  ObjectValueShape,
+  NumberValueShape,
   RecordContainerShape,
   RecordValueShape,
+  StringValueShape,
   StructContainerShape,
   StructValueShape,
   TextContainerShape,
   TreeContainerShape,
-  // Tree-related types
   TreeNodeJSON,
   TreeRefInterface,
+  // Union of two or more plain value types
   UnionValueShape,
-  // Value shapes
   ValueShape,
   // WithNullable type for shapes that support .nullable()
   WithNullable,
   // WithPlaceholder type for shapes that support .placeholder()
   WithPlaceholder,
 } from "./shape.js"
+
 // Schema and type exports
 export { Shape } from "./shape.js"
 export type { Frontiers, TypedDoc } from "./typed-doc.js"

package/src/loro.ts

@@ -28,6 +28,7 @@ import type {
   Container,
   LoroCounter,
   LoroDoc,
+  LoroEventBatch,
   LoroList,
   LoroMap,
   LoroMovableList,
@@ -85,7 +86,7 @@ export interface LoroRefBase {
    * @param callback - Function called when the container changes
    * @returns Subscription that can be used to unsubscribe
    */
-  subscribe(callback: (event: unknown) => void): Subscription
+  subscribe(callback: (event: LoroEventBatch) => void): Subscription
 }
 
 /**

package/src/nested-container-materialization.test.ts

@@ -0,0 +1,336 @@
+import { describe, expect, it } from "vitest"
+import { convertInputToRef } from "./conversion.js"
+import { createTypedDoc, loro, Shape } from "./index.js"
+
+describe("Nested Container Materialization", () => {
+  it("syncs correctly between peers when struct has nested empty record", () => {
+    // Define a schema with a nested map
+    const schema = Shape.doc({
+      items: Shape.record(
+        Shape.struct({
+          id: Shape.plain.string(),
+          metadata: Shape.record(Shape.struct({ key: Shape.plain.string() })), // Nested map of structs
+        }),
+      ),
+    })
+
+    // Create two separate documents (simulating two peers)
+    const clientDoc = createTypedDoc(schema)
+    const serverDoc = createTypedDoc(schema)
+
+    // Client creates an item with empty nested map
+    clientDoc.items.set("item-1", {
+      id: "item-1",
+      metadata: {}, // Empty nested map - BUG: may not materialize properly
+    })
+
+    // Sync client -> server
+    const clientSnapshot = loro(clientDoc).doc.export({ mode: "snapshot" })
+    loro(serverDoc).doc.import(clientSnapshot)
+
+    // Server writes to the nested map
+    const serverEntry = serverDoc.items.get("item-1")
+    expect(serverEntry).toBeDefined()
+    serverEntry?.metadata.set("entry-1", { key: "value" })
+
+    // Sync server -> client
+    const serverUpdate = loro(serverDoc).doc.export({
+      mode: "update",
+      from: loro(clientDoc).doc.version(),
+    })
+    loro(clientDoc).doc.import(serverUpdate)
+
+    // BUG: Client's metadata is EMPTY!
+    const clientEntry = clientDoc.items.get("item-1")
+    expect(clientEntry?.metadata.toJSON()).toEqual({
+      "entry-1": { key: "value" },
+    })
+  })
+
+  it("handles concurrent creation of nested containers", () => {
+    const schema = Shape.doc({
+      items: Shape.record(
+        Shape.struct({
+          id: Shape.plain.string(),
+          metadata: Shape.record(Shape.plain.string()),
+        }),
+      ),
+    })
+
+    const doc1 = createTypedDoc(schema)
+    const doc2 = createTypedDoc(schema)
+
+    // Peer 1 creates item with empty metadata
+    doc1.items.set("item-1", {
+      id: "item-1",
+      metadata: {},
+    })
+
+    // Sync 1 -> 2
+    loro(doc2).doc.import(loro(doc1).doc.export({ mode: "snapshot" }))
+
+    // Peer 1 writes to metadata
+    doc1.items.get("item-1")?.metadata.set("p1", "v1")
+
+    // Peer 2 writes to metadata (concurrently)
+    doc2.items.get("item-1")?.metadata.set("p2", "v2")
+
+    // Sync 1 -> 2
+    loro(doc2).doc.import(
+      loro(doc1).doc.export({ mode: "update", from: loro(doc2).doc.version() }),
+    )
+
+    // Sync 2 -> 1
+    loro(doc1).doc.import(
+      loro(doc2).doc.export({ mode: "update", from: loro(doc1).doc.version() }),
+    )
+
+    // Both should have both values
+    const json1 = doc1.items.get("item-1")?.metadata.toJSON()
+    const json2 = doc2.items.get("item-1")?.metadata.toJSON()
+
+    console.log("JSON1", json1)
+    console.log("JSON2", json2)
+
+    expect(json1).toEqual({ p1: "v1", p2: "v2" })
+    expect(json2).toEqual({ p1: "v1", p2: "v2" })
+  })
+
+  it("materializes nested containers in Tree nodes", () => {
+    const schema = Shape.doc({
+      tree: Shape.tree(
+        Shape.struct({
+          id: Shape.plain.string(),
+          tags: Shape.record(Shape.plain.boolean()),
+        }),
+      ),
+    })
+
+    const doc1 = createTypedDoc(schema)
+    const doc2 = createTypedDoc(schema)
+
+    // Create a node with empty tags
+    const node = doc1.tree.createNode({
+      id: "node-1",
+      tags: {},
+    })
+
+    // Get the node ID for lookup
+    const nodeId = loro(node).container.id
+
+    // Sync 1 -> 2
+    loro(doc2).doc.import(loro(doc1).doc.export({ mode: "snapshot" }))
+
+    // Verify the container exists in doc1
+    const tags = node.data.tags
+    expect(tags).toBeDefined()
+
+    // Verify it's materialized by checking if we can get its container ID
+    const tagsContainer = loro(tags).container
+    expect(tagsContainer).toBeDefined()
+
+    // Now check doc2 using getNodeByID
+    const tree2 = loro(doc2.tree).container
+    const node2 = tree2.getNodeByID(nodeId)
+
+    expect(node2).toBeDefined()
+    if (node2) {
+      const data = node2.data // LoroMap
+      const tagsMap = data.get("tags") // Should be LoroMap
+      expect(tagsMap).toBeDefined()
+      // @ts-expect-error - kind() exists at runtime
+      expect(tagsMap.kind()).toBe("Map")
+    }
+  })
+
+  it("handles missing containers gracefully (schema evolution)", async () => {
+    // Simulate an old document that doesn't have a nested container
+    const { LoroDoc } = await import("loro-crdt")
+    const oldDoc = new LoroDoc()
+    const map = oldDoc.getMap("root")
+    map.set("existing", "value")
+    // "newField" is missing
+    oldDoc.commit()
+
+    // Load with new schema that expects "newField" as a container
+    const schema = Shape.doc({
+      root: Shape.struct({
+        existing: Shape.plain.string(),
+        newField: Shape.record(Shape.plain.string()),
+      }),
+    })
+
+    const doc = createTypedDoc(schema)
+    loro(doc).doc.import(oldDoc.export({ mode: "snapshot" }))
+
+    // Access missing container
+    const root = doc.root
+    // Should not crash
+    expect(root.existing).toBe("value")
+
+    // Accessing the missing container ref should work (it creates the wrapper)
+    const newField = root.newField
+    expect(newField).toBeDefined()
+
+    // But the underlying container doesn't exist yet?
+    // getOrCreateRef creates the wrapper.
+    // The wrapper's getContainer() calls getOrCreateContainer().
+
+    // If we read from it:
+    expect(newField.keys()).toEqual([])
+
+    // If we write to it:
+    newField.set("k", "v")
+
+    // It should create the container lazily (or eagerly on set)
+    expect(newField.get("k")).toBe("v")
+
+    // Verify in raw doc
+    const rawMap = loro(doc).doc.getMap("root")
+    const rawNewField = rawMap.get("newField")
+    expect(rawNewField).toBeDefined()
+  })
+
+  // Task 1.2: Test struct with deeply nested empty containers
+  it("materializes all levels of deeply nested structs", () => {
+    const schema = Shape.doc({
+      root: Shape.struct({
+        level1: Shape.struct({
+          level2: Shape.struct({
+            level3: Shape.record(Shape.plain.string()),
+          }),
+        }),
+      }),
+    })
+
+    const doc1 = createTypedDoc(schema)
+    const doc2 = createTypedDoc(schema)
+
+    // Initialize with empty nested structure - all levels should materialize
+    // Note: For structs, we need to access them to trigger materialization
+    // since the doc root is created lazily
+    const _root = doc1.root
+    const _level1 = doc1.root.level1
+    const _level2 = doc1.root.level1.level2
+    const _level3 = doc1.root.level1.level2.level3
+
+    // Sync doc1 -> doc2
+    loro(doc2).doc.import(loro(doc1).doc.export({ mode: "snapshot" }))
+
+    // Peer 2 writes to the deeply nested container
+    doc2.root.level1.level2.level3.set("deep-key", "deep-value")
+
+    // Sync doc2 -> doc1
+    loro(doc1).doc.import(
+      loro(doc2).doc.export({ mode: "update", from: loro(doc1).doc.version() }),
+    )
+
+    // Verify the deeply nested value is visible in doc1
+    expect(doc1.root.level1.level2.level3.get("deep-key")).toBe("deep-value")
+  })
+
+  // Task 1.3: Test list push with nested empty container
+  it("materializes nested containers when pushing to list", () => {
+    const schema = Shape.doc({
+      items: Shape.list(
+        Shape.struct({
+          id: Shape.plain.string(),
+          metadata: Shape.record(Shape.plain.string()),
+        }),
+      ),
+    })
+
+    const doc1 = createTypedDoc(schema)
+    const doc2 = createTypedDoc(schema)
+
+    // Push an item with empty nested metadata
+    doc1.items.push({
+      id: "item-1",
+      metadata: {},
+    })
+
+    // Sync doc1 -> doc2
+    loro(doc2).doc.import(loro(doc1).doc.export({ mode: "snapshot" }))
+
+    // Peer 2 writes to the nested metadata
+    const item = doc2.items[0]
+    expect(item).toBeDefined()
+    item?.metadata.set("key", "value")
+
+    // Sync doc2 -> doc1
+    loro(doc1).doc.import(
+      loro(doc2).doc.export({ mode: "update", from: loro(doc1).doc.version() }),
+    )
+
+    // Verify the nested value is visible in doc1
+    expect(doc1.items[0]?.metadata.get("key")).toBe("value")
+  })
+
+  // Task 1.5: Test conversion API with nested empty container
+  it("convertInputToRef creates containers for empty nested values", async () => {
+    const loroCrdt = await import("loro-crdt")
+
+    const structShape = Shape.struct({
+      id: Shape.plain.string(),
+      nested: Shape.record(Shape.plain.string()),
+    })
+
+    // Convert a plain object with empty nested record
+    const result = convertInputToRef({ id: "test", nested: {} }, structShape)
+
+    // Result should be a LoroMap
+    expect(result).toBeInstanceOf(loroCrdt.LoroMap)
+
+    // The nested field should also be a LoroMap (not undefined or plain object)
+    const nestedContainer = (result as typeof loroCrdt.LoroMap.prototype).get(
+      "nested",
+    )
+    expect(nestedContainer).toBeDefined()
+    expect(nestedContainer).toBeInstanceOf(loroCrdt.LoroMap)
+  })
+
+  // Tree Node Full Sync Test (from test-plan)
+  it("syncs nested containers in Tree nodes between peers", () => {
+    const schema = Shape.doc({
+      tree: Shape.tree(
+        Shape.struct({
+          id: Shape.plain.string(),
+          tags: Shape.record(Shape.plain.boolean()),
+        }),
+      ),
+    })
+
+    const doc1 = createTypedDoc(schema)
+    const doc2 = createTypedDoc(schema)
+
+    // Create a node with empty tags in doc1
+    const node = doc1.tree.createNode({
+      id: "node-1",
+      tags: {},
+    })
+
+    // Get the node ID for later lookup
+    const nodeId = loro(node).container.id
+
+    // Sync doc1 -> doc2
+    loro(doc2).doc.import(loro(doc1).doc.export({ mode: "snapshot" }))
+
+    // Peer 2 writes to tags using raw Loro API (since TreeRef doesn't expose get by ID)
+    const tree2 = loro(doc2.tree).container
+    const node2Raw = tree2.getNodeByID(nodeId)
+    expect(node2Raw).toBeDefined()
+    if (node2Raw) {
+      const data2 = node2Raw.data as any
+      const tags2 = data2.get("tags") as any
+      tags2.set("important", true)
+    }
+
+    // Sync doc2 -> doc1
+    loro(doc1).doc.import(
+      loro(doc2).doc.export({ mode: "update", from: loro(doc1).doc.version() }),
+    )
+
+    // Verify the tag is visible in doc1's node
+    expect(node.data.tags.get("important")).toBe(true)
+  })
+})