@vuer-ai/vuer-rtc-server 0.2.0 → 0.2.2

package/tests/journal/compaction-load-bug.test.ts

@@ -0,0 +1,409 @@
+/**
+ * Reproduction test for: Document load fails after compaction
+ *
+ * Bug: Server-side getStateForClient() filters by lamportTime,
+ * but client-side initFromServer() filters by vectorClock comparison.
+ * This mismatch causes messages to be sent but not applied.
+ */
+
+import { describe, it, expect, beforeEach } from '@jest/globals';
+import type { CRDTMessage, Operation } from '@vuer-ai/vuer-rtc';
+import { applyMessage, createEmptyGraph, OperationValidator } from '@vuer-ai/vuer-rtc';
+
+interface JournalEntry {
+  msg: CRDTMessage;
+  deletedAt?: number;
+}
+
+interface Snapshot {
+  graph: any;
+  vectorClock: Record<string, number>;
+  lamportTime: number;
+  journalIndex: number;
+}
+
+interface DocumentState {
+  snapshot: Snapshot;
+  journal: JournalEntry[];
+}
+
+function makeMsg(
+  id: string,
+  sessionId: string,
+  ops: Operation[],
+  lamportTime: number,
+  clock?: Record<string, number>,
+): CRDTMessage {
+  return {
+    id,
+    sessionId,
+    clock: clock ?? { [sessionId]: lamportTime },
+    lamportTime,
+    timestamp: Date.now() / 1000,
+    ops,
+  };
+}
+
+function nodeInsertOp(parentKey: string, nodeKey: string, props: Record<string, unknown> = {}): Operation {
+  return {
+    key: parentKey,
+    otype: 'node.insert',
+    path: 'children',
+    value: {
+      key: nodeKey,
+      tag: 'Mesh',
+      name: nodeKey,
+      ...props,
+    },
+  } as Operation;
+}
+
+function vec3SetOp(nodeKey: string, path: string, value: [number, number, number]): Operation {
+  return {
+    key: nodeKey,
+    otype: 'vector3.set',
+    path,
+    value,
+  } as Operation;
+}
+
+/**
+ * Simulate server-side compaction logic
+ */
+function serverCompact(state: DocumentState): void {
+  let newGraph = state.snapshot.graph;
+  let mergedClock = { ...state.snapshot.vectorClock };
+  let maxLamport = state.snapshot.lamportTime;
+
+  for (const entry of state.journal) {
+    if (!entry.deletedAt) {
+      const realOps = entry.msg.ops.filter((op) => !op.otype.startsWith('meta.'));
+      if (realOps.length > 0) {
+        newGraph = applyMessage(newGraph, { ...entry.msg, ops: realOps });
+      }
+    }
+    for (const [sessionId, time] of Object.entries(entry.msg.clock)) {
+      mergedClock[sessionId] = Math.max(mergedClock[sessionId] || 0, time);
+    }
+    maxLamport = Math.max(maxLamport, entry.msg.lamportTime);
+  }
+
+  state.snapshot = {
+    graph: newGraph,
+    vectorClock: mergedClock,
+    lamportTime: maxLamport,
+    journalIndex: maxLamport,
+  };
+  state.journal = [];
+}
+
+/**
+ * Simulate server-side getStateForClient (CURRENT BUGGY VERSION)
+ */
+function serverGetStateForClient_BUGGY(state: DocumentState): {
+  snapshot: Snapshot;
+  journal: CRDTMessage[];
+} {
+  // Server filters by lamportTime only
+  const postSnapshotJournal = state.journal
+    .filter((e) => e.msg.lamportTime > state.snapshot.lamportTime)
+    .map((e) => e.msg);
+
+  return {
+    snapshot: state.snapshot,
+    journal: postSnapshotJournal,
+  };
+}
+
+/**
+ * Simulate server-side getStateForClient (FIXED VERSION)
+ */
+function serverGetStateForClient_FIXED(state: DocumentState): {
+  snapshot: Snapshot;
+  journal: CRDTMessage[];
+} {
+  // Server filters using vector clock comparison (matches client-side logic)
+  const postSnapshotJournal = state.journal
+    .filter((e) => {
+      // Include message if ANY session in its clock is ahead of snapshot
+      for (const [sessionId, time] of Object.entries(e.msg.clock)) {
+        if (time > (state.snapshot.vectorClock[sessionId] ?? 0)) {
+          return true;
+        }
+      }
+      return false; // All clock components <= snapshot, already applied
+    })
+    .map((e) => e.msg);
+
+  return {
+    snapshot: state.snapshot,
+    journal: postSnapshotJournal,
+  };
+}
+
+/**
+ * Simulate client-side initFromServer filtering (CURRENT VERSION)
+ */
+function clientFilterJournal(snapshot: Snapshot, journal: CRDTMessage[]): CRDTMessage[] {
+  return journal.filter((msg) => {
+    for (const [sid, time] of Object.entries(msg.clock)) {
+      if (time > (snapshot.vectorClock[sid] ?? 0)) return true;
+    }
+    return false; // All components are <= snapshot clock
+  });
+}
+
+describe('Document load fails after compaction (Bug)', () => {
+  it('should reproduce the bug: message sent by server but filtered out by client', () => {
+    // Setup: Create a document with entries from two sessions
+    const state: DocumentState = {
+      snapshot: {
+        graph: createEmptyGraph(),
+        vectorClock: {},
+        lamportTime: 0,
+        journalIndex: 0,
+      },
+      journal: [],
+    };
+
+    // Alice creates scene and cube
+    const m1 = makeMsg('m1', 'alice', [nodeInsertOp('', 'scene')], 1, { alice: 1 });
+    const m2 = makeMsg('m2', 'alice', [nodeInsertOp('scene', 'cube', { position: [0, 0, 0] })], 2, { alice: 2 });
+    state.journal.push({ msg: m1 }, { msg: m2 });
+
+    // Bob moves the cube
+    const m3 = makeMsg('m3', 'bob', [vec3SetOp('cube', 'position', [1, 1, 1])], 3, { alice: 2, bob: 1 });
+    state.journal.push({ msg: m3 });
+
+    // Server compacts
+    serverCompact(state);
+
+    // Check snapshot state after compaction
+    expect(state.snapshot.lamportTime).toBe(3);
+    expect(state.snapshot.vectorClock).toEqual({ alice: 2, bob: 1 });
+    expect(state.journal.length).toBe(0);
+
+    // Alice sends another update AFTER compaction
+    const m4 = makeMsg('m4', 'alice', [vec3SetOp('cube', 'position', [2, 2, 2])], 4, { alice: 3, bob: 1 });
+    state.journal.push({ msg: m4 });
+
+    // Server sends state to new client
+    const transferred = serverGetStateForClient_BUGGY(state);
+
+    // BUG: Server includes m4 because lamportTime 4 > 3
+    expect(transferred.journal.length).toBe(1);
+    expect(transferred.journal[0].id).toBe('m4');
+
+    // But client filters it out because alice: 3 <= snapshot.vectorClock.alice: 3 is FALSE
+    // Wait, actually alice: 3 > 2, so it SHOULD pass...
+
+    // Let me create a different scenario that triggers the bug:
+  });
+
+  it('should reproduce the actual bug: vector clock vs lamport time mismatch', () => {
+    // Setup: Concurrent messages from multiple sessions
+    const state: DocumentState = {
+      snapshot: {
+        graph: createEmptyGraph(),
+        vectorClock: {},
+        lamportTime: 0,
+        journalIndex: 0,
+      },
+      journal: [],
+    };
+
+    // Alice creates scene
+    const m1 = makeMsg('m1', 'alice', [nodeInsertOp('', 'scene')], 1, { alice: 1 });
+    // Bob creates cube (concurrent with alice)
+    const m2 = makeMsg('m2', 'bob', [nodeInsertOp('scene', 'cube')], 2, { bob: 1 });
+    // Alice moves cube (has seen bob's message)
+    const m3 = makeMsg('m3', 'alice', [vec3SetOp('cube', 'position', [1, 1, 1])], 3, { alice: 2, bob: 1 });
+
+    state.journal.push({ msg: m1 }, { msg: m2 }, { msg: m3 });
+
+    // Compact
+    serverCompact(state);
+
+    // Snapshot now has:
+    // lamportTime: 3
+    // vectorClock: { alice: 2, bob: 1 }
+    expect(state.snapshot.lamportTime).toBe(3);
+    expect(state.snapshot.vectorClock).toEqual({ alice: 2, bob: 1 });
+
+    // Bob sends another message with OLD local clock (hasn't seen alice's latest)
+    const m4 = makeMsg('m4', 'bob', [vec3SetOp('cube', 'position', [5, 5, 5])], 4, { alice: 1, bob: 2 });
+    state.journal.push({ msg: m4 });
+
+    // Server sends state
+    const transferred = serverGetStateForClient_BUGGY(state);
+
+    // Server includes m4 because lamportTime 4 > 3 ✓
+    expect(transferred.journal.length).toBe(1);
+    expect(transferred.journal[0].id).toBe('m4');
+
+    // Client filtering:
+    // m4.clock = { alice: 1, bob: 2 }
+    // snapshot.vectorClock = { alice: 2, bob: 1 }
+    // alice: 1 > 2? NO
+    // bob: 2 > 1? YES
+    // Result: Message PASSES client filter
+
+    const clientFiltered = clientFilterJournal(transferred.snapshot, transferred.journal);
+    expect(clientFiltered.length).toBe(1);
+
+    // Actually, this scenario also passes. Let me think of another case...
+  });
+
+  it('should demonstrate the core issue: filter criteria mismatch', () => {
+    // The real bug is more subtle. Let's test with a scenario where:
+    // - Message has lamportTime > snapshot.lamportTime (included by server)
+    // - But message's clock is completely dominated by snapshot clock (excluded by client)
+
+    // This can happen when compaction happens at a different lamport time than
+    // the max vector clock component
+
+    const state: DocumentState = {
+      snapshot: {
+        graph: createEmptyGraph(),
+        vectorClock: {},
+        lamportTime: 0,
+        journalIndex: 0,
+      },
+      journal: [],
+    };
+
+    // Three sessions: alice, bob, carol
+    const m1 = makeMsg('m1', 'alice', [nodeInsertOp('', 'scene')], 1, { alice: 1 });
+    const m2 = makeMsg('m2', 'bob', [nodeInsertOp('scene', 'cube')], 2, { bob: 1 });
+    const m3 = makeMsg('m3', 'carol', [vec3SetOp('cube', 'position', [1, 1, 1])], 3, { carol: 1 });
+
+    state.journal.push({ msg: m1 }, { msg: m2 }, { msg: m3 });
+
+    // Compact - this sets:
+    // lamportTime: 3 (max)
+    // vectorClock: { alice: 1, bob: 1, carol: 1 }
+    serverCompact(state);
+
+    expect(state.snapshot.lamportTime).toBe(3);
+    expect(state.snapshot.vectorClock).toEqual({ alice: 1, bob: 1, carol: 1 });
+
+    // Now alice sends a message that was delayed/concurrent
+    // Alice hasn't seen bob or carol's messages yet
+    const m4 = makeMsg('m4', 'alice', [vec3SetOp('cube', 'position', [2, 2, 2])], 4, { alice: 2 });
+    state.journal.push({ msg: m4 });
+
+    // Server filtering: lamportTime 4 > 3 → INCLUDED
+    const transferred = serverGetStateForClient_BUGGY(state);
+    expect(transferred.journal.length).toBe(1);
+    expect(transferred.journal[0].id).toBe('m4');
+
+    // Client filtering:
+    // m4.clock = { alice: 2 }
+    // snapshot.vectorClock = { alice: 1, bob: 1, carol: 1 }
+    // alice: 2 > 1? YES → INCLUDED
+    const clientFiltered = clientFilterJournal(transferred.snapshot, transferred.journal);
+    expect(clientFiltered.length).toBe(1);
+
+    // Still works... Let me try a different approach
+  });
+
+  it('ACTUAL BUG: journalIndex field confusion', () => {
+    // I think the actual bug might be that snapshot.journalIndex is set to lamportTime,
+    // but should be used differently. Or there's confusion between lamportTime filtering
+    // and journalIndex.
+
+    // Looking at the code again:
+    // In compact():
+    //   state.snapshot.journalIndex = maxLamport;
+    // In getStateForClient():
+    //   .filter((e) => e.msg.lamportTime > state.snapshot.lamportTime)
+    //
+    // This should work correctly IF all messages are ordered by lamportTime
+    // and there are no gaps.
+
+    // But what if there's a gap in lamport times after compaction?
+
+    const state: DocumentState = {
+      snapshot: {
+        graph: createEmptyGraph(),
+        vectorClock: {},
+        lamportTime: 0,
+        journalIndex: 0,
+      },
+      journal: [],
+    };
+
+    // Messages with lamport times 1, 2, 5 (gap at 3, 4)
+    const m1 = makeMsg('m1', 'alice', [nodeInsertOp('', 'scene')], 1, { alice: 1 });
+    const m2 = makeMsg('m2', 'bob', [nodeInsertOp('scene', 'cube')], 2, { bob: 1 });
+    const m5 = makeMsg('m5', 'carol', [vec3SetOp('cube', 'position', [1, 1, 1])], 5, { carol: 1 });
+
+    state.journal.push({ msg: m1 }, { msg: m2 }, { msg: m5 });
+
+    // Compact
+    serverCompact(state);
+
+    // Snapshot: lamportTime = 5
+    expect(state.snapshot.lamportTime).toBe(5);
+
+    // New message with lamportTime 3 (filling the gap, received late)
+    const m3 = makeMsg('m3', 'dave', [vec3SetOp('cube', 'position', [2, 2, 2])], 3, { dave: 1 });
+    state.journal.push({ msg: m3 });
+
+    // Server: lamportTime 3 > 5? NO → EXCLUDED
+    const transferred = serverGetStateForClient_BUGGY(state);
+    expect(transferred.journal.length).toBe(0); // BUG: m3 is excluded!
+
+    // But m3 should be included because it hasn't been applied yet!
+    // This is the bug: after compaction, any message with lamportTime <= snapshot.lamportTime
+    // is assumed to be already applied, even if it arrived late.
+
+    console.log('BUG REPRODUCED: Message m3 with lamportTime 3 is excluded after compaction');
+    console.log('snapshot.lamportTime:', state.snapshot.lamportTime);
+    console.log('m3.lamportTime:', m3.lamportTime);
+    console.log('Server includes m3?:', m3.lamportTime > state.snapshot.lamportTime);
+  });
+
+  it('FIX: Use vector clock comparison to include delayed messages', () => {
+    const state: DocumentState = {
+      snapshot: {
+        graph: createEmptyGraph(),
+        vectorClock: {},
+        lamportTime: 0,
+        journalIndex: 0,
+      },
+      journal: [],
+    };
+
+    // Messages with lamport times 1, 2, 5 (gap at 3, 4)
+    const m1 = makeMsg('m1', 'alice', [nodeInsertOp('', 'scene')], 1, { alice: 1 });
+    const m2 = makeMsg('m2', 'bob', [nodeInsertOp('scene', 'cube')], 2, { bob: 1 });
+    const m5 = makeMsg('m5', 'carol', [vec3SetOp('cube', 'position', [1, 1, 1])], 5, { carol: 1 });
+
+    state.journal.push({ msg: m1 }, { msg: m2 }, { msg: m5 });
+
+    // Compact
+    serverCompact(state);
+
+    // Snapshot: lamportTime = 5, vectorClock = { alice: 1, bob: 1, carol: 1 }
+    expect(state.snapshot.lamportTime).toBe(5);
+    expect(state.snapshot.vectorClock).toEqual({ alice: 1, bob: 1, carol: 1 });
+
+    // New message from dave (new session) with lamportTime 3 (filling the gap)
+    const m3 = makeMsg('m3', 'dave', [vec3SetOp('cube', 'position', [2, 2, 2])], 3, { dave: 1 });
+    state.journal.push({ msg: m3 });
+
+    // Buggy version: EXCLUDES m3 (lamportTime 3 <= 5)
+    const buggyResult = serverGetStateForClient_BUGGY(state);
+    expect(buggyResult.journal.length).toBe(0);
+
+    // Fixed version: INCLUDES m3 (dave: 1 > snapshot.vectorClock.dave: 0)
+    const fixedResult = serverGetStateForClient_FIXED(state);
+    expect(fixedResult.journal.length).toBe(1);
+    expect(fixedResult.journal[0].id).toBe('m3');
+
+    console.log('FIX VERIFIED: Message m3 is now correctly included');
+    console.log('m3.clock:', m3.clock);
+    console.log('snapshot.vectorClock:', state.snapshot.vectorClock);
+  });
+});

package/tests/journal/compaction.test.ts

@@ -410,8 +410,8 @@ describe('JournalService.compact()', () => {
 
     expect(graphAfter.nodes['cube'].position).toEqual(graphBefore.nodes['cube'].position);
     expect(graphAfter.nodes['cube'].opacity).toBe(graphBefore.nodes['cube'].opacity);
-    // sphere should be tombstoned
-    expect(graphAfter.nodes['sphere']?._crdt?.deletedAt).toBeDefined();
+    // sphere should be tombstoned (tombstones are at root level, not on node)
+    expect(graphAfter.tombstones['sphere']).toBeDefined();
   });
 
   it('should update snapshot vector clock after compaction', () => {
@@ -645,4 +645,44 @@ describe('Edge Cases', () => {
       expect(snap.vectorClock[s]).toBe(10);
     }
   });
+
+  it('should handle graphs with circular references (parent pointers) during compaction', () => {
+    const DOC_ID = 'doc-circular';
+    svc.createDocument(DOC_ID);
+
+    // Create a scene graph
+    svc.processMessage(DOC_ID, makeMsg('m1', 'a', [nodeInsertOp('', 'scene', { tag: 'Scene' })], 1));
+    svc.processMessage(DOC_ID, makeMsg('m2', 'a', [nodeInsertOp('scene', 'parent', { tag: 'Group' })], 2));
+    svc.processMessage(DOC_ID, makeMsg('m3', 'a', [nodeInsertOp('parent', 'child', { tag: 'Mesh', position: [1, 2, 3] })], 3));
+
+    const graphBefore = svc.computeGraph(svc.loadDocument(DOC_ID)!);
+
+    // Add circular references (parent pointers) that would cause JSON.stringify to fail
+    // This simulates what happens in real scene graphs with bidirectional links
+    const circularGraph = {
+      ...graphBefore,
+      nodes: {
+        ...graphBefore.nodes,
+        parent: { ...graphBefore.nodes['parent'], parent: graphBefore.nodes['scene'] },
+        child: { ...graphBefore.nodes['child'], parent: graphBefore.nodes['parent'] },
+      },
+    };
+
+    // Manually set state to have circular refs (simulating real-world scenario)
+    const state = svc.loadDocument(DOC_ID)!;
+    state.snapshot.graph = circularGraph as any;
+
+    // Compaction should not throw even with circular references
+    // The safeSerialize function in JournalService strips parent pointers
+    expect(() => svc.compact(DOC_ID)).not.toThrow();
+
+    // Verify graph structure is preserved (minus parent pointers)
+    const stateAfter = svc.loadDocument(DOC_ID)!;
+    expect(stateAfter.snapshot.graph.nodes['scene']).toBeDefined();
+    expect(stateAfter.snapshot.graph.nodes['parent']).toBeDefined();
+    expect(stateAfter.snapshot.graph.nodes['child']).toBeDefined();
+
+    // Verify data integrity (non-circular properties should be preserved)
+    expect((stateAfter.snapshot.graph.nodes['child'] as any).position).toEqual([1, 2, 3]);
+  });
 });