@vuer-ai/vuer-rtc-server 0.2.0 → 0.2.2

package/src/journal/CoalescingService.ts

@@ -0,0 +1,626 @@
+/**
+ * CoalescingService - Merges compatible operations in journal batches
+ *
+ * Reduces journal size by:
+ * - Merging consecutive text.insert operations into multi-char inserts
+ * - Merging set operations on same (key, path) within time threshold
+ * - Merging additive operations (vector3.add, number.add)
+ */
+
+import type { PrismaClient } from '@prisma/client';
+import type { CRDTMessage, Operation } from '@vuer-ai/vuer-rtc';
+
+// Default time threshold for set operation coalescing (ms)
+const DEFAULT_SET_THRESHOLD_MS = 5000;
+const DEFAULT_SAMPLING_INTERVAL_MS = 100;
+
+/**
+ * Coalescing strategy for set operations
+ */
+export type SetCoalescingStrategy =
+  | 'last-write'      // Keep only the last write within threshold (current behavior)
+  | 'throttle';       // Sample at regular intervals
+
+/**
+ * Per-property coalescing configuration
+ */
+export interface PropertyCoalescingConfig {
+  /** Time window for coalescing (ms) */
+  thresholdMs?: number;
+  /** Coalescing strategy */
+  strategy?: SetCoalescingStrategy;
+  /** Sampling interval for throttle strategy (ms) */
+  samplingIntervalMs?: number;
+  /** Delta threshold - only coalesce if value change is below this (for numeric values) */
+  deltaThreshold?: number;
+}
+
+export interface CoalescingConfig {
+  /** Default time threshold for set operations (ms) */
+  setThresholdMs?: number;
+  /** Default coalescing strategy for set operations */
+  setStrategy?: SetCoalescingStrategy;
+  /** Default sampling interval for throttle strategy (ms) */
+  samplingIntervalMs?: number;
+  /** Per-property overrides (key format: "key.path" or "*.path" for all keys) */
+  propertyConfig?: Record<string, PropertyCoalescingConfig>;
+
+  // Feature toggles
+  enableTextCoalesce?: boolean;
+  enableSetCoalesce?: boolean;
+  enableVectorCoalesce?: boolean;
+}
+
+export interface CoalescingResult {
+  ok: boolean;
+  before: { journalBatches: number; operations: number };
+  after: { journalBatches: number; operations: number };
+  reduction: { journalBatches: number; operations: number };
+}
+
+/**
+ * Type for position-based text insert
+ */
+interface PositionTextInsertOp {
+  otype: 'text.insert';
+  key: string;
+  path: string;
+  position: number;
+  value: string;
+}
+
+/**
+ * Type for position-based text delete
+ */
+interface PositionTextDeleteOp {
+  otype: 'text.delete';
+  key: string;
+  path: string;
+  position: number;
+  length: number;
+}
+
+/**
+ * Operation with embedded timestamp for coalescing
+ */
+interface TimestampedOp {
+  op: Operation;
+  timestamp: number; // ms
+}
+
+/**
+ * Check if an operation is a position-based text insert
+ */
+function isPositionTextInsertOp(op: Operation): op is PositionTextInsertOp {
+  return op.otype === 'text.insert' &&
+    typeof (op as any).position === 'number' &&
+    typeof (op as any).value === 'string';
+}
+
+/**
+ * Check if an operation is a position-based text delete
+ */
+function isPositionTextDeleteOp(op: Operation): op is PositionTextDeleteOp {
+  return op.otype === 'text.delete' &&
+    typeof (op as any).position === 'number' &&
+    typeof (op as any).length === 'number';
+}
+
+/**
+ * Check if an operation is a set operation
+ */
+function isSetOp(op: Operation): boolean {
+  return op.otype.endsWith('.set');
+}
+
+/**
+ * Check if an operation is an additive operation
+ * Note: number.multiply is NOT included because multiplying multiple
+ * factors together gives a different result than applying them sequentially
+ */
+function isAdditiveOp(op: Operation): boolean {
+  return op.otype === 'vector3.add' ||
+         op.otype === 'number.add';
+}
+
+/**
+ * Coalesce consecutive text inserts and deletes (works with timestamped ops)
+ *
+ * Insert coalescing: Consecutive inserts at adjacent positions
+ *   insert(10, "a"), insert(11, "b"), insert(12, "c") → insert(10, "abc")
+ *
+ * Delete coalescing:
+ *   - Forward (Delete key): Same position
+ *     del(10, 1), del(10, 1), del(10, 1) → del(10, 3)
+ *   - Backward (Backspace): Position moves left by length
+ *     del(10, 1), del(9, 1), del(8, 1) → del(8, 3)
+ */
+function coalesceTextOps(tOps: TimestampedOp[]): TimestampedOp[] {
+  if (tOps.length === 0) return tOps;
+
+  const result: TimestampedOp[] = [];
+  let pendingInsert: { op: PositionTextInsertOp; timestamp: number } | null = null;
+  let pendingDelete: { op: PositionTextDeleteOp; timestamp: number } | null = null;
+
+  for (const tOp of tOps) {
+    const op = tOp.op;
+
+    if (isPositionTextInsertOp(op)) {
+      // Flush any pending delete
+      if (pendingDelete !== null) {
+        result.push({ op: pendingDelete.op as unknown as Operation, timestamp: pendingDelete.timestamp });
+        pendingDelete = null;
+      }
+
+      // Coalesce inserts
+      if (pendingInsert === null) {
+        pendingInsert = {
+          op: {
+            otype: 'text.insert',
+            key: op.key,
+            path: op.path,
+            position: op.position,
+            value: op.value,
+          },
+          timestamp: tOp.timestamp,
+        };
+      } else if (
+        pendingInsert.op.key === op.key &&
+        pendingInsert.op.path === op.path &&
+        pendingInsert.op.position + pendingInsert.op.value.length === op.position
+      ) {
+        // Consecutive insert - merge
+        pendingInsert = {
+          op: {
+            otype: 'text.insert',
+            key: pendingInsert.op.key,
+            path: pendingInsert.op.path,
+            position: pendingInsert.op.position,
+            value: pendingInsert.op.value + op.value,
+          },
+          timestamp: tOp.timestamp,
+        };
+      } else {
+        // Non-consecutive - flush and start new
+        result.push({ op: pendingInsert.op as unknown as Operation, timestamp: pendingInsert.timestamp });
+        pendingInsert = {
+          op: {
+            otype: 'text.insert',
+            key: op.key,
+            path: op.path,
+            position: op.position,
+            value: op.value,
+          },
+          timestamp: tOp.timestamp,
+        };
+      }
+    } else if (isPositionTextDeleteOp(op)) {
+      // Flush any pending insert
+      if (pendingInsert !== null) {
+        result.push({ op: pendingInsert.op as unknown as Operation, timestamp: pendingInsert.timestamp });
+        pendingInsert = null;
+      }
+
+      // Coalesce deletes
+      if (pendingDelete === null) {
+        pendingDelete = {
+          op: {
+            otype: 'text.delete',
+            key: op.key,
+            path: op.path,
+            position: op.position,
+            length: op.length,
+          },
+          timestamp: tOp.timestamp,
+        };
+      } else if (
+        pendingDelete.op.key === op.key &&
+        pendingDelete.op.path === op.path
+      ) {
+        // Check if this is forward deletion (same position) or backward deletion (position moves left)
+        const isForwardDelete = pendingDelete.op.position === op.position;
+        const isBackwardDelete = op.position + op.length === pendingDelete.op.position;
+
+        if (isForwardDelete) {
+          // Forward delete: accumulate lengths at same position
+          pendingDelete = {
+            op: {
+              otype: 'text.delete',
+              key: pendingDelete.op.key,
+              path: pendingDelete.op.path,
+              position: pendingDelete.op.position,
+              length: pendingDelete.op.length + op.length,
+            },
+            timestamp: tOp.timestamp,
+          };
+        } else if (isBackwardDelete) {
+          // Backward delete: position moves left, accumulate lengths
+          pendingDelete = {
+            op: {
+              otype: 'text.delete',
+              key: pendingDelete.op.key,
+              path: pendingDelete.op.path,
+              position: op.position, // use new (leftmost) position
+              length: pendingDelete.op.length + op.length,
+            },
+            timestamp: tOp.timestamp,
+          };
+        } else {
+          // Non-consecutive - flush and start new
+          result.push({ op: pendingDelete.op as unknown as Operation, timestamp: pendingDelete.timestamp });
+          pendingDelete = {
+            op: {
+              otype: 'text.delete',
+              key: op.key,
+              path: op.path,
+              position: op.position,
+              length: op.length,
+            },
+            timestamp: tOp.timestamp,
+          };
+        }
+      } else {
+        // Different key/path - flush and start new
+        result.push({ op: pendingDelete.op as unknown as Operation, timestamp: pendingDelete.timestamp });
+        pendingDelete = {
+          op: {
+            otype: 'text.delete',
+            key: op.key,
+            path: op.path,
+            position: op.position,
+            length: op.length,
+          },
+          timestamp: tOp.timestamp,
+        };
+      }
+    } else {
+      // Other operation type - flush pending
+      if (pendingInsert !== null) {
+        result.push({ op: pendingInsert.op as unknown as Operation, timestamp: pendingInsert.timestamp });
+        pendingInsert = null;
+      }
+      if (pendingDelete !== null) {
+        result.push({ op: pendingDelete.op as unknown as Operation, timestamp: pendingDelete.timestamp });
+        pendingDelete = null;
+      }
+      result.push(tOp);
+    }
+  }
+
+  // Flush any remaining pending ops
+  if (pendingInsert !== null) {
+    result.push({ op: pendingInsert.op as unknown as Operation, timestamp: pendingInsert.timestamp });
+  }
+  if (pendingDelete !== null) {
+    result.push({ op: pendingDelete.op as unknown as Operation, timestamp: pendingDelete.timestamp });
+  }
+
+  return result;
+}
+
+/**
+ * Coalesce set operations with configurable strategy
+ * Uses embedded timestamps in TimestampedOp for correct tracking
+ *
+ * Strategies:
+ * - 'last-write': Keep only the last write within threshold (default)
+ * - 'throttle': Sample at regular intervals (samplingIntervalMs)
+ */
+function coalesceSetOps(
+  tOps: TimestampedOp[],
+  config: CoalescingConfig
+): TimestampedOp[] {
+  if (tOps.length === 0) return tOps;
+
+  const thresholdMs = config.setThresholdMs ?? DEFAULT_SET_THRESHOLD_MS;
+  const strategy = config.setStrategy ?? 'last-write';
+  const samplingIntervalMs = config.samplingIntervalMs ?? DEFAULT_SAMPLING_INTERVAL_MS;
+
+  if (strategy === 'last-write') {
+    return coalesceSetOpsLastWrite(tOps, thresholdMs);
+  } else if (strategy === 'throttle') {
+    return coalesceSetOpsThrottle(tOps, thresholdMs, samplingIntervalMs);
+  }
+
+  return tOps;
+}
+
+/**
+ * Last-write strategy: Keep only the last write within threshold
+ */
+function coalesceSetOpsLastWrite(tOps: TimestampedOp[], thresholdMs: number): TimestampedOp[] {
+  const result: TimestampedOp[] = [];
+  const lastSetByKey = new Map<string, { tOp: TimestampedOp; resultIndex: number }>();
+
+  for (const tOp of tOps) {
+    const op = tOp.op;
+    const timestamp = tOp.timestamp;
+
+    if (isSetOp(op)) {
+      const key = `${(op as any).key}:${(op as any).path}`;
+      const existing = lastSetByKey.get(key);
+
+      if (existing && (timestamp - existing.tOp.timestamp) <= thresholdMs) {
+        // Within threshold - replace existing with new
+        result[existing.resultIndex] = tOp;
+        lastSetByKey.set(key, { tOp, resultIndex: existing.resultIndex });
+      } else {
+        // New set or outside threshold
+        const resultIndex = result.length;
+        result.push(tOp);
+        lastSetByKey.set(key, { tOp, resultIndex });
+      }
+    } else {
+      result.push(tOp);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Throttle strategy: Sample at regular intervals
+ * Keep one sample per interval, dropping intermediate updates
+ */
+function coalesceSetOpsThrottle(
+  tOps: TimestampedOp[],
+  thresholdMs: number,
+  samplingIntervalMs: number
+): TimestampedOp[] {
+  const result: TimestampedOp[] = [];
+  const lastSampleByKey = new Map<string, { tOp: TimestampedOp; resultIndex: number; intervalStart: number }>();
+
+  for (const tOp of tOps) {
+    const op = tOp.op;
+    const timestamp = tOp.timestamp;
+
+    if (isSetOp(op)) {
+      const key = `${(op as any).key}:${(op as any).path}`;
+      const existing = lastSampleByKey.get(key);
+
+      if (!existing) {
+        // First sample for this key
+        const resultIndex = result.length;
+        result.push(tOp);
+        lastSampleByKey.set(key, {
+          tOp,
+          resultIndex,
+          intervalStart: Math.floor(timestamp / samplingIntervalMs) * samplingIntervalMs,
+        });
+      } else {
+        const currentInterval = Math.floor(timestamp / samplingIntervalMs) * samplingIntervalMs;
+        const timeSinceFirstSample = timestamp - existing.intervalStart;
+
+        if (currentInterval === existing.intervalStart) {
+          // Same interval - replace with latest
+          result[existing.resultIndex] = tOp;
+          lastSampleByKey.set(key, { ...existing, tOp });
+        } else if (timeSinceFirstSample <= thresholdMs) {
+          // New interval, still within threshold - add new sample
+          const resultIndex = result.length;
+          result.push(tOp);
+          lastSampleByKey.set(key, { tOp, resultIndex, intervalStart: currentInterval });
+        } else {
+          // Outside threshold - reset
+          const resultIndex = result.length;
+          result.push(tOp);
+          lastSampleByKey.set(key, { tOp, resultIndex, intervalStart: currentInterval });
+        }
+      }
+    } else {
+      result.push(tOp);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Coalesce additive operations (works with timestamped ops)
+ */
+function coalesceAdditiveOps(tOps: TimestampedOp[]): TimestampedOp[] {
+  if (tOps.length === 0) return tOps;
+
+  const result: TimestampedOp[] = [];
+  const pendingByKey = new Map<string, TimestampedOp>();
+
+  for (const tOp of tOps) {
+    const op = tOp.op;
+    if (isAdditiveOp(op)) {
+      const key = `${(op as any).key}:${(op as any).path}:${op.otype}`;
+      const existing = pendingByKey.get(key);
+
+      if (existing && existing.op.otype === op.otype) {
+        // Merge values
+        if (op.otype === 'vector3.add') {
+          const va = (existing.op as any).value as [number, number, number];
+          const vb = (op as any).value as [number, number, number];
+          (existing.op as any).value = [va[0] + vb[0], va[1] + vb[1], va[2] + vb[2]];
+          existing.timestamp = tOp.timestamp; // update to latest timestamp
+        } else if (op.otype === 'number.add') {
+          (existing.op as any).value = (existing.op as any).value + (op as any).value;
+          existing.timestamp = tOp.timestamp;
+        }
+      } else {
+        if (existing) {
+          result.push(existing);
+        }
+        pendingByKey.set(key, { op: { ...op }, timestamp: tOp.timestamp });
+      }
+    } else {
+      // Flush all pending additive ops
+      for (const pending of pendingByKey.values()) {
+        result.push(pending);
+      }
+      pendingByKey.clear();
+      result.push(tOp);
+    }
+  }
+
+  // Flush remaining
+  for (const pending of pendingByKey.values()) {
+    result.push(pending);
+  }
+
+  return result;
+}
+
+/**
+ * Coalesce operations in a message array
+ *
+ * IMPORTANT: Preserves interleaved order to maintain causality.
+ * Groups messages into consecutive runs from the same session, then coalesces
+ * each run separately. This ensures Lamport's "happened-before" relation is preserved.
+ *
+ * Example:
+ *   Input:  [alice-msg1, alice-msg2, bob-msg3, alice-msg4]
+ *   Output: [alice-run1, bob-run2, alice-run3]
+ *
+ * This prevents the bug where all operations from a session get the last message's
+ * lamportTime, which breaks LWW conflict resolution.
+ */
+function coalesceOperations(
+  messages: CRDTMessage[],
+  config: CoalescingConfig
+): CRDTMessage[] {
+  if (messages.length === 0) return [];
+
+  // Group messages into consecutive runs from the same session
+  const runs: CRDTMessage[][] = [];
+  let currentRun: CRDTMessage[] = [];
+  let lastSessionId: string | null = null;
+
+  for (const msg of messages) {
+    if (msg.sessionId !== lastSessionId && currentRun.length > 0) {
+      // Session switch - emit current run and start new one
+      runs.push(currentRun);
+      currentRun = [];
+    }
+    currentRun.push(msg);
+    lastSessionId = msg.sessionId;
+  }
+  if (currentRun.length > 0) {
+    runs.push(currentRun);
+  }
+
+  // Coalesce each run separately
+  const coalescedMessages: CRDTMessage[] = [];
+
+  for (const run of runs) {
+    // Flatten all ops from this run with embedded timestamps
+    let tOps: TimestampedOp[] = [];
+    for (const msg of run) {
+      const timestamp = msg.timestamp * 1000; // convert to ms
+      for (const op of msg.ops) {
+        tOps.push({ op, timestamp });
+      }
+    }
+
+    // Apply coalescing in order
+    if (config.enableTextCoalesce !== false) {
+      tOps = coalesceTextOps(tOps);
+    }
+    if (config.enableSetCoalesce !== false) {
+      tOps = coalesceSetOps(tOps, config);
+    }
+    if (config.enableVectorCoalesce !== false) {
+      tOps = coalesceAdditiveOps(tOps);
+    }
+
+    // Extract final ops
+    const allOps = tOps.map(t => t.op);
+
+    if (allOps.length === 0) continue;
+
+    // Rebuild message for this run - use the last message's metadata
+    const lastMsg = run[run.length - 1];
+    coalescedMessages.push({
+      ...lastMsg,
+      ops: allOps,
+    });
+  }
+
+  return coalescedMessages;
+}
+
+export class CoalescingService {
+  constructor(private prisma: PrismaClient) {}
+
+  /**
+   * Coalesce operations in a document's journal
+   */
+  async coalesce(docId: string, config: CoalescingConfig = {}): Promise<CoalescingResult> {
+    // Get all journal batches
+    const batches = await this.prisma.journalBatch.findMany({
+      where: { documentId: docId },
+      orderBy: { lamportTime: 'asc' },
+    });
+
+    const beforeOpsCount = batches.reduce((sum, b) => {
+      const ops = Array.isArray(b.operations) ? b.operations : [];
+      return sum + ops.length;
+    }, 0);
+
+    // Convert batches to messages
+    const messages: CRDTMessage[] = batches.map(b => ({
+      id: b.id,
+      sessionId: b.sessionId ?? 'unknown',
+      clock: (b.vectorClock as Record<string, number>) ?? {},
+      lamportTime: b.lamportTime ?? 0,
+      timestamp: b.persistedAt.getTime() / 1000,
+      ops: (Array.isArray(b.operations) ? b.operations : []) as unknown as Operation[],
+    }));
+
+    // Coalesce
+    const coalesced = coalesceOperations(messages, config);
+
+    // Count ops after
+    const afterOpsCount = coalesced.reduce((sum, m) => sum + m.ops.length, 0);
+
+    // If no reduction, skip update
+    if (afterOpsCount >= beforeOpsCount) {
+      return {
+        ok: true,
+        before: { journalBatches: batches.length, operations: beforeOpsCount },
+        after: { journalBatches: batches.length, operations: beforeOpsCount },
+        reduction: { journalBatches: 0, operations: 0 },
+      };
+    }
+
+    // Delete old batches and create new ones (one per session)
+    const now = new Date();
+    await this.prisma.$transaction(async (tx) => {
+      // Delete old batches
+      await tx.journalBatch.deleteMany({ where: { documentId: docId } });
+
+      // Create coalesced batches - one per session
+      for (let i = 0; i < coalesced.length; i++) {
+        const msg = coalesced[i];
+        if (msg.ops.length > 0) {
+          await tx.journalBatch.create({
+            data: {
+              documentId: docId,
+              batchId: `coalesced-${msg.sessionId}-${Date.now()}-${i}`,
+              lamportTime: msg.lamportTime,
+              sessionId: msg.sessionId,
+              vectorClock: msg.clock,
+              operations: msg.ops as unknown as any[],
+              startTime: now,
+              endTime: now,
+            },
+          });
+        }
+      }
+    });
+
+    return {
+      ok: true,
+      before: { journalBatches: batches.length, operations: beforeOpsCount },
+      after: { journalBatches: coalesced.length, operations: afterOpsCount },
+      reduction: {
+        journalBatches: batches.length - coalesced.length,
+        operations: beforeOpsCount - afterOpsCount,
+      },
+    };
+  }
+}