@vuer-ai/vuer-rtc-server 0.2.3 → 0.4.1

package/src/journal/TextJournalService.ts

@@ -0,0 +1,483 @@
+/**
+ * Text Journal Service - Business logic for text document journal operations
+ *
+ * Handles:
+ * - Receiving and validating CRDTMessages for text documents
+ * - Storing messages in journal
+ * - Processing meta.undo/meta.redo operations
+ * - Computing current text state (TextRope)
+ * - Providing snapshot + journal for new clients
+ */
+
+import type { PrismaClient } from '@prisma/client';
+import {
+  type CRDTMessage,
+  type VectorClock,
+  type RawTextRope,
+  TextRope,
+  create,
+  insert,
+  remove as deleteText,
+  compactRope,
+  fromRaw,
+} from '@vuer-ai/vuer-rtc';
+
+/** How often the compaction loop runs (ms). */
+const COMPACTION_INTERVAL_MS = 30_000;
+
+/** Number of journal entries that triggers compaction for a document. */
+const COMPACTION_THRESHOLD_ENTRIES = 500;
+
+export interface TextJournalEntry {
+  msg: CRDTMessage;
+  deletedAt?: number;
+}
+
+export interface TextDocumentState {
+  snapshot: {
+    text: TextRope;
+    vectorClock: VectorClock;
+    lt: number;
+  };
+  journal: TextJournalEntry[];
+}
+
+/**
+ * Safely parse a TextDocument.currentText (Json) into a TextRope.
+ * Text is stored as plain strings and lazy-upgraded to TextRope on edit.
+ */
+function parseTextSnapshot(currentText: unknown, agentId: string): TextRope {
+  if (typeof currentText === 'string') {
+    const rope = create(agentId);
+    if (currentText.length > 0) {
+      // Insert initial text
+      insert(rope, 0, currentText);
+    }
+    return rope;
+  }
+
+  // If it's already a TextRope object, reconstruct it
+  if (currentText && typeof currentText === 'object') {
+    try {
+      // Attempt to hydrate from serialized format
+      const hydrated = fromRaw(currentText as RawTextRope);
+      hydrated.agentId = agentId;
+      return hydrated;
+    } catch {
+      // Fall back to empty rope
+      return create(agentId);
+    }
+  }
+
+  return create(agentId);
+}
+
+export class TextJournalService {
+  private prisma: PrismaClient;
+
+  // In-memory state per document (for fast access)
+  private documentStates: Map<string, TextDocumentState> = new Map();
+
+  /** Handle returned by setInterval for the compaction loop. */
+  private compactionTimer: ReturnType<typeof setInterval> | null = null;
+
+  /** Mutex set to prevent concurrent compact + processMessage races. */
+  private compactionLocks = new Set<string>();
+
+  /** Callback for fetching connected member vector clocks per document. */
+  private memberClockProvider: ((docId: string) => Promise<VectorClock[]>) | null = null;
+
+  constructor(prisma: PrismaClient) {
+    this.prisma = prisma;
+  }
+
+  // ── Compaction loop ──────────────────────────────────────────────
+
+  /**
+   * Start the periodic compaction loop.
+   * Should be called once after the service is created.
+   */
+  startCompactionLoop(): void {
+    if (this.compactionTimer) return; // already running
+    this.compactionTimer = setInterval(() => {
+      this.compactAllLoadedDocuments().catch((err) =>
+        console.error('[text-compaction-loop]', err)
+      );
+    }, COMPACTION_INTERVAL_MS);
+  }
+
+  /**
+   * Stop the periodic compaction loop (for graceful shutdown).
+   */
+  stopCompactionLoop(): void {
+    if (this.compactionTimer) {
+      clearInterval(this.compactionTimer);
+      this.compactionTimer = null;
+    }
+  }
+
+  /**
+   * Store a callback for fetching connected member vector clocks per document.
+   * Used by the compaction loop to compute safe watermarks.
+   */
+  setMemberClockProvider(fn: (docId: string) => Promise<VectorClock[]>): void {
+    this.memberClockProvider = fn;
+  }
+
+  /**
+   * Compute safe compaction point: the component-wise minimum of all
+   * connected clients' vector clocks.
+   */
+  computeCompactionWatermark(memberClocks: VectorClock[]): VectorClock {
+    if (memberClocks.length === 0) return {};
+
+    const result: VectorClock = { ...memberClocks[0] };
+    for (let i = 1; i < memberClocks.length; i++) {
+      const clock = memberClocks[i];
+      for (const client of Object.keys(result)) {
+        result[client] = Math.min(
+          result[client],
+          clock[client] ?? 0,
+        );
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Check if a message's vector clock is dominated by the watermark.
+   */
+  isDominatedByWatermark(msgClock: VectorClock, watermark: VectorClock): boolean {
+    for (const [client, time] of Object.entries(msgClock)) {
+      if ((watermark[client] ?? 0) < time) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Iterate all in-memory document states and compact those whose
+   * journal exceeds COMPACTION_THRESHOLD_ENTRIES.
+   */
+  private async compactAllLoadedDocuments(): Promise<void> {
+    for (const [documentId, state] of this.documentStates) {
+      if (state.journal.length >= COMPACTION_THRESHOLD_ENTRIES) {
+        try {
+          let watermark: VectorClock | undefined;
+          if (this.memberClockProvider) {
+            const clocks = await this.memberClockProvider(documentId);
+            if (clocks.length > 0) {
+              watermark = this.computeCompactionWatermark(clocks);
+            }
+          }
+          await this.compact(documentId, watermark);
+        } catch (err) {
+          console.error(`[text-compaction] failed for ${documentId}:`, err);
+        }
+      }
+    }
+  }
+
+  // ── Document loading ─────────────────────────────────────────────
+
+  /**
+   * Load text document state from database.
+   */
+  async loadDocument(documentId: string): Promise<TextDocumentState | null> {
+    // Check in-memory cache first
+    if (this.documentStates.has(documentId)) {
+      return this.documentStates.get(documentId)!;
+    }
+
+    // Load from database
+    const doc = await this.prisma.textDocument.findUnique({ where: { id: documentId } });
+    if (!doc) return null;
+
+    // Parse text rope
+    const textRope = parseTextSnapshot(doc.currentText, 'system');
+
+    // Load journal entries after the snapshot
+    const batches = await this.prisma.textJournalBatch.findMany({
+      where: { textDocumentId: documentId },
+      orderBy: { lamportTime: 'asc' },
+    });
+
+    const journal: TextJournalEntry[] = batches.flatMap((batch) => {
+      const ops = Array.isArray(batch.operations) ? batch.operations : [];
+      return ops.map((op: any) => ({
+        msg: {
+          id: batch.batchId,
+          client: batch.client,
+          ops: [op],
+          clock: (batch.vectorClock as VectorClock) || {},
+          lt: batch.lamportTime || 0,
+          ts: new Date(batch.persistedAt).getTime() / 1000,
+        },
+        deletedAt: batch.deletedAt ? new Date(batch.deletedAt).getTime() / 1000 : undefined,
+      }));
+    });
+
+    const state: TextDocumentState = {
+      snapshot: {
+        text: textRope,
+        vectorClock: {},
+        lt: 0,
+      },
+      journal,
+    };
+
+    this.documentStates.set(documentId, state);
+    return state;
+  }
+
+  /**
+   * Process incoming message from client
+   */
+  async processMessage(
+    documentId: string,
+    msg: CRDTMessage
+  ): Promise<{ success: boolean; error?: string }> {
+    // Load state
+    const state = await this.loadDocument(documentId);
+    if (!state) {
+      return { success: false, error: 'Text document not found' };
+    }
+
+    // Check for duplicate
+    const exists = await this.prisma.textJournalBatch.findFirst({
+      where: { textDocumentId: documentId, batchId: msg.id },
+    });
+    if (exists) {
+      return { success: true }; // Already processed
+    }
+
+    // Process meta operations (undo/redo)
+    for (const op of msg.ops) {
+      if (op.ot === 'meta.undo') {
+        const targetId = (op as any).targetMsgId;
+        const target = state.journal.find((e) => e.msg.id === targetId);
+        if (target) {
+          target.deletedAt = msg.ts;
+          // Update in DB
+          await this.prisma.textJournalBatch.updateMany({
+            where: { textDocumentId: documentId, batchId: targetId },
+            data: { deletedAt: new Date(msg.ts * 1000) },
+          });
+        }
+      } else if (op.ot === 'meta.redo') {
+        const targetId = (op as any).targetMsgId;
+        const target = state.journal.find((e) => e.msg.id === targetId);
+        if (target) {
+          delete target.deletedAt;
+          // Clear in DB
+          await this.prisma.textJournalBatch.updateMany({
+            where: { textDocumentId: documentId, batchId: targetId },
+            data: { deletedAt: null },
+          });
+        }
+      }
+    }
+
+    // Add to journal
+    state.journal.push({ msg });
+
+    // Persist to database
+    await this.prisma.textJournalBatch.create({
+      data: {
+        textDocumentId: documentId,
+        client: msg.client,
+        batchId: msg.id,
+        operations: msg.ops as any,
+        vectorClock: msg.clock as any,
+        lamportTime: msg.lt,
+        startTime: new Date(msg.ts * 1000),
+        endTime: new Date(msg.ts * 1000),
+      },
+    });
+
+    return { success: true };
+  }
+
+  /**
+   * Get state for new client (snapshot + journal).
+   */
+  async getStateForClient(documentId: string): Promise<{
+    snapshot: { text: string; vectorClock: VectorClock; lt: number };
+    journal: CRDTMessage[];
+  } | null> {
+    const state = await this.loadDocument(documentId);
+    if (!state) return null;
+
+    // Filter journal entries using vector clock comparison
+    const postSnapshotJournal = state.journal
+      .filter((e) => {
+        for (const [client, time] of Object.entries(e.msg.clock)) {
+          if (time > (state.snapshot.vectorClock[client] ?? 0)) {
+            return true;
+          }
+        }
+        return false;
+      })
+      .map((e) => e.msg);
+
+    // Serialize TextRope to plain string via toJSON()
+    const serializedText = state.snapshot.text.toString();
+
+    return {
+      snapshot: {
+        text: serializedText,
+        vectorClock: state.snapshot.vectorClock,
+        lt: state.snapshot.lt,
+      },
+      journal: postSnapshotJournal,
+    };
+  }
+
+  /**
+   * Compact journal (create new snapshot from acknowledged entries).
+   */
+  async compact(documentId: string, watermark?: VectorClock): Promise<void> {
+    // Acquire compaction lock
+    if (this.compactionLocks.has(documentId)) return;
+    this.compactionLocks.add(documentId);
+
+    try {
+      const state = await this.loadDocument(documentId);
+      if (!state || state.journal.length === 0) return;
+
+      // Determine compaction boundary
+      let compactUpToIndex = -1;
+      if (watermark) {
+        for (let i = 0; i < state.journal.length; i++) {
+          if (this.isDominatedByWatermark(state.journal[i].msg.clock, watermark)) {
+            compactUpToIndex = i;
+          } else {
+            break;
+          }
+        }
+      } else {
+        compactUpToIndex = state.journal.length - 1;
+      }
+
+      if (compactUpToIndex < 0) return;
+
+      // Build new snapshot
+      let newText = state.snapshot.text;
+      let mergedClock = { ...state.snapshot.vectorClock };
+      let maxLamport = state.snapshot.lt;
+
+      for (let i = 0; i <= compactUpToIndex; i++) {
+        const entry = state.journal[i];
+        if (!entry.deletedAt) {
+          // Apply text operations to rope
+          for (const op of entry.msg.ops) {
+            if (op.ot.startsWith('text.')) {
+              // Apply text operation
+              if (op.ot === 'text.insert') {
+                insert(newText, (op as any).position, (op as any).value);
+              } else if (op.ot === 'text.delete') {
+                deleteText(newText, (op as any).position, (op as any).length || 1);
+              }
+            }
+          }
+        }
+        // Merge clock and lamport
+        for (const [client, time] of Object.entries(entry.msg.clock)) {
+          mergedClock[client] = Math.max(
+            mergedClock[client] || 0,
+            time,
+          );
+        }
+        maxLamport = Math.max(maxLamport, entry.msg.lt);
+      }
+
+      // Compact TextRope (merge spans, remove tombstones)
+      newText = compactRope(newText);
+
+      // Collect batchIds of compacted entries
+      const compactedBatchIds = state.journal
+        .slice(0, compactUpToIndex + 1)
+        .map((e) => e.msg.id);
+
+      // Update snapshot
+      state.snapshot = {
+        text: newText,
+        vectorClock: mergedClock,
+        lt: maxLamport,
+      };
+
+      // Remove compacted entries from in-memory journal
+      state.journal = state.journal.slice(compactUpToIndex + 1);
+
+      // Persist snapshot (TextRope.toJSON() returns plain string)
+      const serializedText = newText.toJSON();
+      await this.prisma.textDocument.update({
+        where: { id: documentId },
+        data: { currentText: serializedText as any },
+      });
+
+      // Delete compacted batches from DB
+      if (compactedBatchIds.length > 0) {
+        const deletedCount = await this.prisma.textJournalBatch.deleteMany({
+          where: {
+            textDocumentId: documentId,
+            batchId: { in: compactedBatchIds },
+          },
+        });
+        console.log(`[text-compact] Deleted ${deletedCount.count} journal batches for doc ${documentId}`);
+      }
+    } finally {
+      this.compactionLocks.delete(documentId);
+    }
+  }
+
+  /**
+   * Create a new text document
+   */
+  async createDocument(
+    name: string,
+    ownerId: string
+  ): Promise<string> {
+    const doc = await this.prisma.textDocument.create({
+      data: {
+        name,
+        ownerId,
+        currentText: '',
+        version: 0,
+      },
+    });
+
+    // Initialize in-memory state
+    this.documentStates.set(doc.id, {
+      snapshot: {
+        text: create('system'),
+        vectorClock: {},
+        lt: 0,
+      },
+      journal: [],
+    });
+
+    return doc.id;
+  }
+
+  /**
+   * Clear all journal entries for a document (keep document record).
+   */
+  async clearDocument(documentId: string): Promise<void> {
+    await this.prisma.textJournalBatch.deleteMany({
+      where: { textDocumentId: documentId },
+    });
+
+    // Reset in-memory state
+    const state = this.documentStates.get(documentId);
+    if (state) {
+      state.journal = [];
+      state.snapshot = {
+        text: create('system'),
+        vectorClock: {},
+        lt: 0,
+      };
+    }
+  }
+}

package/src/journal/index.ts

@@ -4,7 +4,15 @@
 
 export { JournalRepository } from './JournalRepository.js';
 export {
-  JournalService,
+  GraphJournalService,
   type JournalEntry as ServerJournalEntry,
   type DocumentState,
-} from './JournalService.js';
+} from './GraphJournalService.js';
+export {
+  TextJournalService,
+  type TextJournalEntry as ServerTextJournalEntry,
+  type TextDocumentState as ServerTextDocumentState,
+} from './TextJournalService.js';
+
+// Backward compatibility export
+export { GraphJournalService as JournalService } from './GraphJournalService.js';

package/src/journal/rle-demo.ts

@@ -17,7 +17,7 @@ import {
  */
 function createMessage(
   id: string,
-  sessionId: string,
+  client: string,
   lamportTime: number,
   timestamp: number,
   key: string = 'cube-1',
@@ -25,14 +25,14 @@ function createMessage(
 ): CRDTMessage {
   return {
     id,
-    sessionId,
-    clock: { [sessionId]: lamportTime },
-    lamportTime,
-    timestamp,
+    client,
+    clock: { [client]: lamportTime },
+    lt: lamportTime,
+    ts: timestamp,
     ops: [
       {
         key,
-        otype: 'vector3.set',
+        ot: 'vector3.set',
         path: 'position',
         value,
       },
@@ -130,14 +130,14 @@ function main() {
       const user = i < 15 ? 'user-1' : 'user-2';
       const msg: CRDTMessage = {
         id: `msg-${i}`,
-        sessionId: user,
+        client: user,
         clock: { [user]: i },
-        lamportTime: i,
-        timestamp: 1000 + i,
+        lt: i,
+        ts: 1000 + i,
         ops: [
           {
             key: `node-${i % 5}`,
-            otype: 'node.insert',
+            ot: 'node.insert',
             path: 'children',
             value: {
               key: `child-${i}`,
@@ -150,7 +150,7 @@ function main() {
           },
           {
             key: `node-${i % 5}`,
-            otype: 'vector3.set',
+            ot: 'vector3.set',
             path: 'color',
             value: [Math.random(), Math.random(), Math.random()],
           },