@trestleinc/replicate 1.1.2 → 1.2.0-preview.1

package/src/client/services/awareness.ts

@@ -0,0 +1,373 @@
+import * as Y from "yjs";
+import { Awareness } from "y-protocols/awareness";
+import type { ConvexClient } from "convex/browser";
+import type { FunctionReference } from "convex/server";
+
+const DEFAULT_HEARTBEAT_INTERVAL = 10000;
+const DEFAULT_THROTTLE_MS = 50;
+
+interface AwarenessApi {
+  mark: FunctionReference<"mutation">;
+  sessions: FunctionReference<"query">;
+  cursors: FunctionReference<"query">;
+  leave: FunctionReference<"mutation">;
+}
+
+export interface ConvexAwarenessConfig {
+  convexClient: ConvexClient;
+  api: AwarenessApi;
+  document: string;
+  client: string;
+  ydoc: Y.Doc;
+  interval?: number;
+  syncReady?: Promise<void>;
+}
+
+export interface ConvexAwarenessProvider {
+  awareness: Awareness;
+  document: Y.Doc;
+  destroy: () => void;
+}
+
+/**
+ * Creates a Yjs Awareness instance backed by Convex for transport.
+ * This provider syncs awareness state (cursors, user info) via Convex
+ * mutations and queries instead of WebSocket.
+ *
+ * Compatible with TipTap's CollaborationCursor and BlockNote's collaboration.
+ */
+export function createAwarenessProvider(
+  config: ConvexAwarenessConfig,
+): ConvexAwarenessProvider {
+  const {
+    convexClient,
+    api,
+    document,
+    client,
+    ydoc,
+    interval = DEFAULT_HEARTBEAT_INTERVAL,
+    syncReady,
+  } = config;
+
+  const awareness = new Awareness(ydoc);
+  let destroyed = false;
+  let visible = true;
+  let heartbeatTimer: ReturnType<typeof setInterval> | null = null;
+  let pendingUpdate: ReturnType<typeof setTimeout> | null = null;
+  let unsubscribeCursors: (() => void) | undefined;
+  let unsubscribeVisibility: (() => void) | undefined;
+  let unsubscribePageHide: (() => void) | undefined;
+
+  // Track remote client IDs we know about
+  const remoteClientIds = new Map<string, number>();
+
+  const getVector = (): ArrayBuffer | undefined => {
+    return Y.encodeStateVector(ydoc).buffer as ArrayBuffer;
+  };
+
+  /**
+   * Extract cursor from awareness state for Convex storage.
+   * y-prosemirror stores cursor as RelativePosition class instances.
+   * We serialize to plain JSON objects for Convex storage.
+   */
+  const extractCursorFromState = (state: Record<string, unknown> | null): {
+    anchor: unknown;
+    head: unknown;
+  } | undefined => {
+    if (!state) return undefined;
+
+    const cursor = state.cursor as {
+      anchor?: unknown;
+      head?: unknown;
+    } | undefined | null;
+
+    if (cursor?.anchor === undefined || cursor.head === undefined) {
+      return undefined;
+    }
+
+    try {
+      const serialized = {
+        anchor: JSON.parse(JSON.stringify(cursor.anchor)),
+        head: JSON.parse(JSON.stringify(cursor.head)),
+      };
+      return serialized;
+    }
+    catch {
+      return undefined;
+    }
+  };
+
+  /**
+   * Extract user profile from awareness state.
+   */
+  const extractUserFromState = (state: Record<string, unknown> | null): {
+    user?: string;
+    profile?: { name?: string; color?: string; avatar?: string };
+  } => {
+    if (!state) return {};
+
+    const user = state.user as
+      | { name?: string; color?: string; [key: string]: unknown }
+      | undefined;
+    if (user) {
+      return {
+        profile: {
+          name: user.name,
+          color: user.color,
+          avatar: user.avatar as string | undefined,
+        },
+      };
+    }
+
+    return {};
+  };
+
+  const sendToServer = () => {
+    if (destroyed || !visible) return;
+
+    const localState = awareness.getLocalState();
+    const cursor = extractCursorFromState(localState);
+    const { user, profile } = extractUserFromState(localState);
+    const vector = getVector();
+
+    convexClient.mutation(api.mark, {
+      document,
+      client,
+      cursor,
+      user,
+      profile,
+      interval,
+      vector,
+    });
+  };
+
+  /**
+   * Throttled version of sendToServer for frequent updates.
+   */
+  const throttledSend = () => {
+    if (pendingUpdate) return;
+
+    pendingUpdate = setTimeout(() => {
+      pendingUpdate = null;
+      sendToServer();
+    }, DEFAULT_THROTTLE_MS);
+  };
+
+  /**
+   * Handle local awareness changes.
+   */
+  const onLocalAwarenessUpdate = (
+    changes: { added: number[]; updated: number[]; removed: number[] },
+    origin: unknown,
+  ) => {
+    // Only send if the change is local (not from applying remote state)
+    if (origin === "remote") return;
+
+    // Check if our client was updated
+    const localClientId = awareness.clientID;
+    if (
+      changes.added.includes(localClientId)
+      || changes.updated.includes(localClientId)
+    ) {
+      throttledSend();
+    }
+  };
+
+  const subscribeToPresence = () => {
+    unsubscribeCursors = convexClient.onUpdate(
+      api.sessions,
+      { document, connected: true, exclude: client },
+      (remotes: {
+        client: string;
+        document: string;
+        user?: string;
+        profile?: { name?: string; color?: string; avatar?: string };
+        cursor?: { anchor: unknown; head: unknown; field?: string };
+      }[]) => {
+        if (destroyed) return;
+
+        const validRemotes = remotes.filter(r => r.document === document);
+
+        const currentRemotes = new Set<string>();
+
+        for (const remote of validRemotes) {
+          currentRemotes.add(remote.client);
+
+          let remoteClientId = remoteClientIds.get(remote.client);
+          if (!remoteClientId) {
+            remoteClientId = hashStringToNumber(remote.client);
+            remoteClientIds.set(remote.client, remoteClientId);
+          }
+
+          const remoteState: Record<string, unknown> = {
+            user: {
+              name: remote.profile?.name ?? remote.user ?? "Anonymous",
+              color: remote.profile?.color ?? getColorForClient(remote.client),
+              avatar: remote.profile?.avatar,
+              clientId: remote.client,
+            },
+          };
+
+          if (remote.cursor) {
+            remoteState.cursor = remote.cursor;
+          }
+
+          awareness.states.set(remoteClientId, remoteState);
+        }
+
+        for (const [clientStr, clientId] of remoteClientIds) {
+          if (!currentRemotes.has(clientStr)) {
+            awareness.states.delete(clientId);
+            remoteClientIds.delete(clientStr);
+          }
+        }
+
+        awareness.emit("update", [
+          { added: [], updated: Array.from(remoteClientIds.values()), removed: [] },
+          "remote",
+        ]);
+      },
+    );
+  };
+
+  const setupVisibilityHandler = () => {
+    if (typeof globalThis.document === "undefined") return;
+
+    const handler = () => {
+      const wasVisible = visible;
+      visible = globalThis.document.visibilityState === "visible";
+
+      if (wasVisible && !visible) {
+        convexClient.mutation(api.mark, {
+          document,
+          client,
+          cursor: undefined,
+          interval,
+          vector: getVector(),
+        });
+      }
+      else if (!wasVisible && visible) {
+        sendToServer();
+      }
+    };
+
+    globalThis.document.addEventListener("visibilitychange", handler);
+    unsubscribeVisibility = () => {
+      globalThis.document.removeEventListener("visibilitychange", handler);
+    };
+  };
+
+  const setupPageHideHandler = () => {
+    if (typeof globalThis.window === "undefined") return;
+
+    const handler = (e: PageTransitionEvent) => {
+      if (e.persisted) return;
+      if (destroyed) return;
+
+      convexClient.mutation(api.leave, { document, client });
+    };
+
+    globalThis.window.addEventListener("pagehide", handler);
+    unsubscribePageHide = () => {
+      globalThis.window.removeEventListener("pagehide", handler);
+    };
+  };
+
+  /**
+   * Start periodic heartbeat to keep presence alive.
+   */
+  const startHeartbeat = () => {
+    sendToServer();
+    heartbeatTimer = setInterval(sendToServer, interval);
+  };
+
+  /**
+   * Stop heartbeat.
+   */
+  const stopHeartbeat = () => {
+    if (heartbeatTimer) {
+      clearInterval(heartbeatTimer);
+      heartbeatTimer = null;
+    }
+  };
+
+  awareness.on("update", onLocalAwarenessUpdate);
+  subscribeToPresence();
+  setupVisibilityHandler();
+  setupPageHideHandler();
+
+  let startTimeout: ReturnType<typeof setTimeout> | null = null;
+
+  const initHeartbeat = async () => {
+    if (syncReady) {
+      await syncReady;
+    }
+    if (!destroyed) {
+      startHeartbeat();
+    }
+  };
+
+  startTimeout = setTimeout(() => {
+    initHeartbeat();
+  }, 0);
+
+  return {
+    awareness,
+    document: ydoc,
+
+    destroy: () => {
+      if (destroyed) return;
+      destroyed = true;
+
+      if (startTimeout) {
+        clearTimeout(startTimeout);
+        startTimeout = null;
+      }
+      if (pendingUpdate) {
+        clearTimeout(pendingUpdate);
+        pendingUpdate = null;
+      }
+      stopHeartbeat();
+      awareness.off("update", onLocalAwarenessUpdate);
+      unsubscribeCursors?.();
+      unsubscribeVisibility?.();
+      unsubscribePageHide?.();
+
+      for (const clientId of remoteClientIds.values()) {
+        awareness.states.delete(clientId);
+      }
+      remoteClientIds.clear();
+      awareness.emit("update", [{ added: [], updated: [], removed: [] }, "remote"]);
+
+      convexClient.mutation(api.leave, { document, client });
+
+      awareness.destroy();
+    },
+  };
+}
+
+/**
+ * Hash a string to a positive number for use as clientId.
+ */
+function hashStringToNumber(str: string): number {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash = ((hash << 5) - hash) + char;
+    hash = hash & hash; // Convert to 32-bit integer
+  }
+  return Math.abs(hash);
+}
+
+/**
+ * Generate a color for a client based on their ID.
+ */
+const DEFAULT_COLORS = [
+  "#F87171", "#FB923C", "#FBBF24", "#A3E635",
+  "#34D399", "#22D3EE", "#60A5FA", "#A78BFA", "#F472B6",
+];
+
+function getColorForClient(clientId: string): string {
+  const hash = hashStringToNumber(clientId);
+  return DEFAULT_COLORS[hash % DEFAULT_COLORS.length];
+}

package/src/client/services/context.ts

@@ -0,0 +1,114 @@
+import { createMutex } from "lib0/mutex";
+import type { ConvexClient } from "convex/browser";
+import type { FunctionReference } from "convex/server";
+import type { Collection } from "@tanstack/db";
+import type { Persistence } from "$/client/persistence/types";
+import type { SubdocManager } from "$/client/subdocs";
+
+interface ConvexCollectionApi {
+  stream: FunctionReference<"query">;
+  insert: FunctionReference<"mutation">;
+  update: FunctionReference<"mutation">;
+  remove: FunctionReference<"mutation">;
+  recovery: FunctionReference<"query">;
+  mark: FunctionReference<"mutation">;
+  compact: FunctionReference<"mutation">;
+  material?: FunctionReference<"query">;
+  sessions?: FunctionReference<"query">;
+  cursors?: FunctionReference<"query">;
+  leave?: FunctionReference<"mutation">;
+}
+
+export interface ProseState {
+  applyingFromServer: Map<string, boolean>;
+  debounceTimers: Map<string, ReturnType<typeof setTimeout>>;
+  lastSyncedVectors: Map<string, Uint8Array>;
+  pendingState: Map<string, boolean>;
+  pendingListeners: Map<string, Set<(pending: boolean) => void>>;
+  fragmentObservers: Map<string, () => void>;
+  failedSyncQueue: Map<string, boolean>;
+}
+
+export interface CollectionContext {
+  collection: string;
+  subdocs: SubdocManager;
+  client: ConvexClient;
+  api: ConvexCollectionApi;
+  persistence: Persistence;
+  fields: Set<string>;
+  mutex: ReturnType<typeof createMutex>;
+  debounce: number;
+  prose: ProseState;
+  cleanup?: () => void;
+  clientId?: string;
+  ref?: Collection<any>;
+  vector?: Uint8Array;
+  synced?: Promise<void>;
+  resolve?: () => void;
+}
+
+const contexts = new Map<string, CollectionContext>();
+
+export function getContext(collection: string): CollectionContext {
+  const ctx = contexts.get(collection);
+  if (!ctx) throw new Error(`Collection ${collection} not initialized`);
+  return ctx;
+}
+
+export function hasContext(collection: string): boolean {
+  return contexts.has(collection);
+}
+
+type InitContextConfig = Omit<
+  CollectionContext,
+  | "mutex"
+  | "prose"
+  | "cleanup"
+  | "clientId"
+  | "ref"
+  | "vector"
+>;
+
+function createProseState(): ProseState {
+  return {
+    applyingFromServer: new Map(),
+    debounceTimers: new Map(),
+    lastSyncedVectors: new Map(),
+    pendingState: new Map(),
+    pendingListeners: new Map(),
+    fragmentObservers: new Map(),
+    failedSyncQueue: new Map(),
+  };
+}
+
+export function initContext(config: InitContextConfig): CollectionContext {
+  let resolver: () => void;
+  const synced = new Promise<void>((r) => {
+    resolver = r;
+  });
+
+  const ctx: CollectionContext = {
+    ...config,
+    mutex: createMutex(),
+    prose: createProseState(),
+    synced,
+    resolve: resolver!,
+  };
+  contexts.set(config.collection, ctx);
+  return ctx;
+}
+
+export function deleteContext(collection: string): void {
+  contexts.delete(collection);
+}
+
+type UpdateableFields = "clientId" | "ref" | "vector" | "cleanup";
+
+export function updateContext(
+  collection: string,
+  updates: Partial<Pick<CollectionContext, UpdateableFields>>,
+): CollectionContext {
+  const ctx = getContext(collection);
+  Object.assign(ctx, updates);
+  return ctx;
+}

package/src/client/services/seq.ts

@@ -0,0 +1,78 @@
+import { Effect, Context, Layer } from "effect";
+import { IDBError, IDBWriteError } from "$/client/errors";
+import type { KeyValueStore } from "$/client/persistence/types";
+
+export type Seq = number;
+
+export class SeqService extends Context.Tag("SeqService")<
+  SeqService,
+  {
+    readonly load: (collection: string) => Effect.Effect<Seq, IDBError>;
+    readonly save: (collection: string, seq: Seq) => Effect.Effect<void, IDBWriteError>;
+    readonly clear: (collection: string) => Effect.Effect<void, IDBError>;
+  }
+>() {}
+
+export function createSeqLayer(kv: KeyValueStore) {
+  return Layer.succeed(
+    SeqService,
+    SeqService.of({
+      load: (collection: string) =>
+        Effect.gen(function* (_) {
+          const key = `cursor:${collection}`;
+          const stored = yield* _(
+            Effect.tryPromise({
+              try: () => kv.get<Seq>(key),
+              catch: cause => new IDBError({ operation: "get", key, cause }),
+            }),
+          );
+
+          if (stored !== undefined) {
+            yield* _(
+              Effect.logDebug("Loaded seq from storage", {
+                collection,
+                seq: stored,
+              }),
+            );
+            return stored;
+          }
+
+          yield* _(
+            Effect.logDebug("No stored seq, using default", {
+              collection,
+            }),
+          );
+          return 0;
+        }),
+
+      save: (collection: string, seq: Seq) =>
+        Effect.gen(function* (_) {
+          const key = `cursor:${collection}`;
+          yield* _(
+            Effect.tryPromise({
+              try: () => kv.set(key, seq),
+              catch: cause => new IDBWriteError({ key, value: seq, cause }),
+            }),
+          );
+          yield* _(
+            Effect.logDebug("Seq saved", {
+              collection,
+              seq,
+            }),
+          );
+        }),
+
+      clear: (collection: string) =>
+        Effect.gen(function* (_) {
+          const key = `cursor:${collection}`;
+          yield* _(
+            Effect.tryPromise({
+              try: () => kv.del(key),
+              catch: cause => new IDBError({ operation: "delete", key, cause }),
+            }),
+          );
+          yield* _(Effect.logDebug("Seq cleared", { collection }));
+        }),
+    }),
+  );
+}

package/src/client/services/session.ts

@@ -0,0 +1,20 @@
+function generateClientId(): number {
+  return Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+}
+
+export function getClientId(collection: string): string {
+  const key = `replicate:clientId:${collection}`;
+
+  if (typeof localStorage === "undefined") {
+    return String(generateClientId());
+  }
+
+  const stored = localStorage.getItem(key);
+  if (stored) {
+    return stored;
+  }
+
+  const clientId = String(generateClientId());
+  localStorage.setItem(key, clientId);
+  return clientId;
+}

package/src/client/services/sync.ts

@@ -0,0 +1,122 @@
+import { Effect, Context, Layer, Schedule, Duration } from "effect";
+import type { ConvexClient } from "convex/browser";
+import type { FunctionReference } from "convex/server";
+import { NetworkError } from "$/client/errors";
+
+interface SyncApi {
+  stream: FunctionReference<"query">;
+  recovery: FunctionReference<"query">;
+  compact: FunctionReference<"mutation">;
+  mark: FunctionReference<"mutation">;
+}
+
+export interface SyncConfig {
+  collection: string;
+  convexClient: ConvexClient;
+  api: SyncApi;
+}
+
+export interface CompactResult {
+  success: boolean;
+  removed: number;
+  retained: number;
+  size: number;
+}
+
+export interface RecoveryResult {
+  diff?: ArrayBuffer;
+  vector: ArrayBuffer;
+  cursor: number;
+}
+
+export interface StreamChange {
+  document: string;
+  bytes: ArrayBuffer;
+  seq: number;
+  type: string;
+}
+
+export interface StreamResponse {
+  changes: StreamChange[];
+  cursor: number;
+  more: boolean;
+  compact?: string;
+}
+
+const retryPolicy = Schedule.exponential("1 second").pipe(
+  Schedule.jittered,
+  Schedule.compose(Schedule.elapsed),
+  Schedule.whileOutput(duration => Duration.lessThan(duration, Duration.seconds(30))),
+);
+
+export class Sync extends Context.Tag("Sync")<
+  Sync,
+  {
+    readonly subscribe: (
+      cursor: number,
+      limit: number,
+      onUpdate: (response: StreamResponse) => void,
+    ) => Effect.Effect<() => void, NetworkError>;
+    readonly recover: (vector: ArrayBuffer) => Effect.Effect<RecoveryResult, NetworkError>;
+    readonly compact: (document: string) => Effect.Effect<CompactResult, NetworkError>;
+    readonly mark: (
+      document: string,
+      client: string,
+      seq: number,
+    ) => Effect.Effect<void, NetworkError>;
+  }
+>() {}
+
+export function createSyncLayer(config: SyncConfig) {
+  const { convexClient, api } = config;
+
+  return Layer.succeed(
+    Sync,
+    Sync.of({
+      subscribe: (cursor, limit, onUpdate) =>
+        Effect.gen(function* () {
+          const unsubscribe = yield* Effect.try({
+            try: () =>
+              convexClient.onUpdate(
+                api.stream,
+                { cursor, limit },
+                (response: StreamResponse) => {
+                  onUpdate(response);
+                },
+              ),
+            catch: cause => new NetworkError({ operation: "subscribe", cause, retryable: true }),
+          });
+
+          return unsubscribe;
+        }),
+
+      recover: vector =>
+        Effect.gen(function* () {
+          const response = yield* Effect.tryPromise({
+            try: () => convexClient.query(api.recovery, { vector }),
+            catch: cause => new NetworkError({ operation: "recovery", cause, retryable: true }),
+          });
+
+          return response as RecoveryResult;
+        }).pipe(Effect.retry(retryPolicy)),
+
+      compact: document =>
+        Effect.gen(function* () {
+          const result = yield* Effect.tryPromise({
+            try: () => convexClient.mutation(api.compact, { document }),
+            catch: cause => new NetworkError({ operation: "compact", cause, retryable: true }),
+          });
+
+          return result as CompactResult;
+        }).pipe(Effect.retry(retryPolicy)),
+
+      mark: (document, client, seq) =>
+        Effect.gen(function* () {
+          yield* Effect.tryPromise({
+            try: () => convexClient.mutation(api.mark, { document, client, seq }),
+            catch: cause => new NetworkError({ operation: "mark", cause, retryable: true }),
+          });
+        }),
+    }),
+  );
+}