@trestleinc/replicate 0.1.0

package/src/component/public.ts

@@ -0,0 +1,212 @@
+import { v } from 'convex/values';
+import { mutation, query } from './_generated/server';
+
+/**
+ * Insert a new document with CRDT bytes (Yjs format).
+ * Appends delta to event log (event sourcing pattern).
+ *
+ * @param collectionName - Collection identifier
+ * @param documentId - Unique document identifier
+ * @param crdtBytes - ArrayBuffer containing Yjs CRDT bytes (delta)
+ * @param version - CRDT version number
+ */
+export const insertDocument = mutation({
+  args: {
+    collectionName: v.string(),
+    documentId: v.string(),
+    crdtBytes: v.bytes(),
+    version: v.number(),
+  },
+  returns: v.object({
+    success: v.boolean(),
+  }),
+  handler: async (ctx, args) => {
+    // Append delta to event log (no duplicate check - event sourcing!)
+    await ctx.db.insert('documents', {
+      collectionName: args.collectionName,
+      documentId: args.documentId,
+      crdtBytes: args.crdtBytes,
+      version: args.version,
+      timestamp: Date.now(),
+      operationType: 'insert',
+    });
+
+    return { success: true };
+  },
+});
+
+/**
+ * Update an existing document with new CRDT bytes (Yjs format).
+ * Appends delta to event log (event sourcing pattern).
+ *
+ * @param collectionName - Collection identifier
+ * @param documentId - Unique document identifier
+ * @param crdtBytes - ArrayBuffer containing Yjs CRDT bytes (delta)
+ * @param version - CRDT version number
+ */
+export const updateDocument = mutation({
+  args: {
+    collectionName: v.string(),
+    documentId: v.string(),
+    crdtBytes: v.bytes(),
+    version: v.number(),
+  },
+  returns: v.object({
+    success: v.boolean(),
+  }),
+  handler: async (ctx, args) => {
+    // Append delta to event log (no check - event sourcing!)
+    await ctx.db.insert('documents', {
+      collectionName: args.collectionName,
+      documentId: args.documentId,
+      crdtBytes: args.crdtBytes,
+      version: args.version,
+      timestamp: Date.now(),
+      operationType: 'update',
+    });
+
+    return { success: true };
+  },
+});
+
+/**
+ * Delete a document from CRDT storage.
+ * Appends deletion delta to event log (preserves history).
+ *
+ * @param collectionName - Collection identifier
+ * @param documentId - Unique document identifier
+ * @param crdtBytes - ArrayBuffer containing Yjs deletion delta
+ * @param version - CRDT version number
+ */
+export const deleteDocument = mutation({
+  args: {
+    collectionName: v.string(),
+    documentId: v.string(),
+    crdtBytes: v.bytes(),
+    version: v.number(),
+  },
+  returns: v.object({
+    success: v.boolean(),
+  }),
+  handler: async (ctx, args) => {
+    // Append deletion delta to event log (preserve history!)
+    await ctx.db.insert('documents', {
+      collectionName: args.collectionName,
+      documentId: args.documentId,
+      crdtBytes: args.crdtBytes,
+      version: args.version,
+      timestamp: Date.now(),
+      operationType: 'delete',
+    });
+
+    return { success: true };
+  },
+});
+
+/**
+ * Get complete event history for a document.
+ * Returns all CRDT deltas in chronological order.
+ *
+ * Used for:
+ * - Future recovery features (client-side)
+ * - Audit trails
+ * - Debugging
+ *
+ * @param collectionName - Collection identifier
+ * @param documentId - Unique document identifier
+ */
+export const getDocumentHistory = query({
+  args: {
+    collectionName: v.string(),
+    documentId: v.string(),
+  },
+  returns: v.array(
+    v.object({
+      crdtBytes: v.bytes(),
+      version: v.number(),
+      timestamp: v.number(),
+      operationType: v.string(),
+    })
+  ),
+  handler: async (ctx, args) => {
+    // Fetch ALL deltas for this document in chronological order
+    const deltas = await ctx.db
+      .query('documents')
+      .withIndex('by_collection_document_version', (q) =>
+        q.eq('collectionName', args.collectionName).eq('documentId', args.documentId)
+      )
+      .order('asc')
+      .collect();
+
+    return deltas.map((d) => ({
+      crdtBytes: d.crdtBytes,
+      version: d.version,
+      timestamp: d.timestamp,
+      operationType: d.operationType,
+    }));
+  },
+});
+
+/**
+ * Stream CRDT changes for incremental replication.
+ * Returns Yjs CRDT bytes for documents modified since the checkpoint.
+ * Can be used for both polling (awaitReplication) and subscriptions (live updates).
+ *
+ * @param collectionName - Collection identifier
+ * @param checkpoint - Last replication checkpoint
+ * @param limit - Maximum number of changes to return (default: 100)
+ */
+export const stream = query({
+  args: {
+    collectionName: v.string(),
+    checkpoint: v.object({
+      lastModified: v.number(),
+    }),
+    limit: v.optional(v.number()),
+  },
+  returns: v.object({
+    changes: v.array(
+      v.object({
+        documentId: v.string(),
+        crdtBytes: v.bytes(),
+        version: v.number(),
+        timestamp: v.number(),
+      })
+    ),
+    checkpoint: v.object({
+      lastModified: v.number(),
+    }),
+    hasMore: v.boolean(),
+  }),
+  handler: async (ctx, args) => {
+    const limit = args.limit ?? 100;
+
+    const documents = await ctx.db
+      .query('documents')
+      .withIndex('by_timestamp', (q) =>
+        q.eq('collectionName', args.collectionName).gt('timestamp', args.checkpoint.lastModified)
+      )
+      .order('asc')
+      .take(limit);
+
+    const changes = documents.map((doc) => ({
+      documentId: doc.documentId,
+      crdtBytes: doc.crdtBytes,
+      version: doc.version,
+      timestamp: doc.timestamp,
+    }));
+
+    const newCheckpoint = {
+      lastModified:
+        documents.length > 0
+          ? (documents[documents.length - 1]?.timestamp ?? args.checkpoint.lastModified)
+          : args.checkpoint.lastModified,
+    };
+
+    return {
+      changes,
+      checkpoint: newCheckpoint,
+      hasMore: documents.length === limit,
+    };
+  },
+});

package/src/component/schema.ts

@@ -0,0 +1,16 @@
+import { defineSchema, defineTable } from 'convex/server';
+import { v } from 'convex/values';
+
+export default defineSchema({
+  documents: defineTable({
+    collectionName: v.string(),
+    documentId: v.string(),
+    crdtBytes: v.bytes(),
+    version: v.number(),
+    timestamp: v.number(),
+    operationType: v.string(), // 'insert' | 'update' | 'delete'
+  })
+    .index('by_collection', ['collectionName'])
+    .index('by_collection_document_version', ['collectionName', 'documentId', 'version'])
+    .index('by_timestamp', ['collectionName', 'timestamp']),
+});

package/src/server/index.ts

@@ -0,0 +1,26 @@
+/**
+ * Server-side utilities for Convex backend.
+ * Import this in your Convex functions (convex/*.ts files).
+ *
+ * @example
+ * ```typescript
+ * // convex/tasks.ts
+ * import {
+ *   insertDocumentHelper,
+ *   updateDocumentHelper,
+ *   deleteDocumentHelper,
+ *   streamHelper,
+ * } from '@trestleinc/replicate/server';
+ * ```
+ */
+
+// Replication helpers for mutations/queries
+export {
+  insertDocumentHelper,
+  updateDocumentHelper,
+  deleteDocumentHelper,
+  streamHelper,
+} from './replication.js';
+
+// Schema utilities
+export { replicatedTable, type ReplicationFields } from './schema.js';

package/src/server/replication.ts

@@ -0,0 +1,244 @@
+import type { GenericDataModel } from 'convex/server';
+
+function cleanDocument(doc: unknown): unknown {
+  return Object.fromEntries(
+    Object.entries(doc as Record<string, unknown>).filter(
+      ([_, value]) => value !== undefined && value !== null
+    )
+  );
+}
+
+/**
+ * Insert a document into both the CRDT component and the main application table.
+ *
+ * DUAL-STORAGE ARCHITECTURE:
+ * This helper implements a dual-storage pattern where documents are stored in two places:
+ *
+ * 1. Component Storage (CRDT Layer):
+ *    - Stores CRDT bytes (from Yjs) for offline-first conflict resolution
+ *    - Handles concurrent updates with automatic merging
+ *    - Provides the source of truth for offline changes
+ *
+ * 2. Main Application Table:
+ *    - Stores materialized documents for efficient querying
+ *    - Used by server-side Convex functions that need to query/join data
+ *    - Optimized for reactive subscriptions and complex queries
+ *
+ * WHY BOTH?
+ * - Component: Handles conflict resolution and offline replication (CRDT bytes)
+ * - Main table: Enables efficient server-side queries (materialized docs)
+ * - Similar to event sourcing: component = event log, main table = read model
+ *
+ * @param ctx - Convex mutation context
+ * @param components - Generated components from Convex
+ * @param tableName - Name of the main application table
+ * @param args - Document data with id, crdtBytes, materializedDoc, and version
+ * @returns Success indicator
+ */
+export async function insertDocumentHelper<_DataModel extends GenericDataModel>(
+  ctx: unknown,
+  components: unknown,
+  tableName: string,
+  args: { id: string; crdtBytes: ArrayBuffer; materializedDoc: unknown; version: number }
+): Promise<{
+  success: boolean;
+  metadata: {
+    documentId: string;
+    timestamp: number;
+    version: number;
+    collectionName: string;
+  };
+}> {
+  // Use consistent timestamp for both writes to enable sync matching
+  const timestamp = Date.now();
+
+  // Write CRDT bytes to component
+  await (ctx as any).runMutation((components as any).replicate.public.insertDocument, {
+    collectionName: tableName,
+    documentId: args.id,
+    crdtBytes: args.crdtBytes,
+    version: args.version,
+  });
+
+  // Write materialized doc to main table
+  const db = (ctx as any).db;
+  const cleanDoc = cleanDocument(args.materializedDoc) as Record<string, unknown>;
+
+  await db.insert(tableName, {
+    id: args.id,
+    ...cleanDoc,
+    version: args.version,
+    timestamp,
+  });
+
+  // Return metadata for replication matching
+  return {
+    success: true,
+    metadata: {
+      documentId: args.id,
+      timestamp,
+      version: args.version,
+      collectionName: tableName,
+    },
+  };
+}
+
+/**
+ * Update a document in both the CRDT component and the main application table.
+ *
+ * @param ctx - Convex mutation context
+ * @param components - Generated components from Convex
+ * @param tableName - Name of the main application table
+ * @param args - Document data with id, crdtBytes, materializedDoc, and version
+ * @returns Success indicator
+ */
+export async function updateDocumentHelper<_DataModel extends GenericDataModel>(
+  ctx: unknown,
+  components: unknown,
+  tableName: string,
+  args: { id: string; crdtBytes: ArrayBuffer; materializedDoc: unknown; version: number }
+): Promise<{
+  success: boolean;
+  metadata: {
+    documentId: string;
+    timestamp: number;
+    version: number;
+    collectionName: string;
+  };
+}> {
+  // Use consistent timestamp for both writes to enable sync matching
+  const timestamp = Date.now();
+
+  // Write CRDT bytes to component
+  await (ctx as any).runMutation((components as any).replicate.public.updateDocument, {
+    collectionName: tableName,
+    documentId: args.id,
+    crdtBytes: args.crdtBytes,
+    version: args.version,
+  });
+
+  // Update materialized doc in main table
+  const db = (ctx as any).db;
+  const existing = await db
+    .query(tableName)
+    .withIndex('by_user_id', (q: unknown) => (q as any).eq('id', args.id))
+    .first();
+
+  if (!existing) {
+    throw new Error(`Document ${args.id} not found in table ${tableName}`);
+  }
+
+  const cleanDoc = cleanDocument(args.materializedDoc) as Record<string, unknown>;
+
+  await db.patch(existing._id, {
+    ...cleanDoc,
+    version: args.version,
+    timestamp,
+  });
+
+  // Return metadata for replication matching
+  return {
+    success: true,
+    metadata: {
+      documentId: args.id,
+      timestamp,
+      version: args.version,
+      collectionName: tableName,
+    },
+  };
+}
+
+/**
+ * HARD delete a document from main table, APPEND deletion delta to component.
+ *
+ * NEW BEHAVIOR (v0.3.0):
+ * - Appends deletion delta to component event log (preserves history)
+ * - Physically removes document from main table (hard delete)
+ * - CRDT history preserved for future recovery features
+ *
+ * @param ctx - Convex mutation context
+ * @param components - Generated components from Convex
+ * @param tableName - Name of the main application table
+ * @param args - Document data with id, crdtBytes (deletion delta), and version
+ * @returns Success indicator with metadata
+ */
+export async function deleteDocumentHelper<_DataModel extends GenericDataModel>(
+  ctx: unknown,
+  components: unknown,
+  tableName: string,
+  args: { id: string; crdtBytes: ArrayBuffer; version: number }
+): Promise<{
+  success: boolean;
+  metadata: {
+    documentId: string;
+    timestamp: number;
+    version: number;
+    collectionName: string;
+  };
+}> {
+  const timestamp = Date.now();
+
+  // 1. Append deletion delta to component (event log)
+  await (ctx as any).runMutation((components as any).replicate.public.deleteDocument, {
+    collectionName: tableName,
+    documentId: args.id,
+    crdtBytes: args.crdtBytes,
+    version: args.version,
+  });
+
+  // 2. HARD DELETE from main table (physical removal)
+  const db = (ctx as any).db;
+  const existing = await db
+    .query(tableName)
+    .withIndex('by_user_id', (q: unknown) => (q as any).eq('id', args.id))
+    .first();
+
+  if (existing) {
+    await db.delete(existing._id);
+  }
+
+  return {
+    success: true,
+    metadata: {
+      documentId: args.id,
+      timestamp,
+      version: args.version,
+      collectionName: tableName,
+    },
+  };
+}
+
+/**
+ * Stream document changes from the CRDT component storage.
+ *
+ * This reads CRDT bytes from the component (not the main table) to enable
+ * true Y.applyUpdate() conflict resolution on the client.
+ * Can be used for both polling (awaitReplication) and subscriptions (live updates).
+ *
+ * @param ctx - Convex query context
+ * @param components - Generated components from Convex
+ * @param tableName - Name of the collection
+ * @param args - Checkpoint and limit for pagination
+ * @returns Array of changes with CRDT bytes
+ */
+export async function streamHelper<_DataModel extends GenericDataModel>(
+  ctx: unknown,
+  components: unknown,
+  tableName: string,
+  args: { checkpoint: { lastModified: number }; limit?: number }
+): Promise<{
+  changes: Array<{
+    documentId: string;
+    crdtBytes: ArrayBuffer;
+    version: number;
+    timestamp: number;
+  }>;
+  checkpoint: { lastModified: number };
+  hasMore: boolean;
+}> {
+  return (ctx as any).runQuery((components as any).replicate.public.stream, {
+    collectionName: tableName,
+    checkpoint: args.checkpoint,
+    limit: args.limit,
+  });
+}

package/src/server/schema.ts

@@ -0,0 +1,97 @@
+/**
+ * Schema utilities for defining replicated tables.
+ * Automatically adds replication metadata fields so users don't have to.
+ *
+ * @example
+ * ```typescript
+ * // convex/schema.ts
+ * import { defineSchema } from 'convex/server';
+ * import { v } from 'convex/values';
+ * import { replicatedTable } from '@trestleinc/replicate/server';
+ *
+ * export default defineSchema({
+ *   tasks: replicatedTable(
+ *     {
+ *       id: v.string(),
+ *       text: v.string(),
+ *       isCompleted: v.boolean(),
+ *     },
+ *     (table) => table
+ *       .index('by_id', ['id'])
+ *       .index('by_timestamp', ['timestamp'])
+ *   ),
+ * });
+ * ```
+ */
+
+import { defineTable } from 'convex/server';
+import { v } from 'convex/values';
+
+/**
+ * Internal replication metadata fields added to every replicated table.
+ * These are managed automatically by the replication layer.
+ */
+export type ReplicationFields = {
+  /** Version number for conflict resolution */
+  version: number;
+  /** Last modification timestamp (Unix ms) */
+  timestamp: number;
+};
+
+/**
+ * Wraps a table definition to automatically add replication metadata fields.
+ *
+ * Users define their business logic fields, and we inject:
+ * - `version` - For conflict resolution and CRDT versioning
+ * - `timestamp` - For incremental sync and change tracking
+ *
+ * Enables:
+ * - Dual-storage architecture (CRDT component + main table)
+ * - Conflict-free replication across clients
+ * - Hard delete support with CRDT history preservation
+ * - Event sourcing via component storage
+ *
+ * @param userFields - User's business logic fields (id, text, etc.)
+ * @param applyIndexes - Optional callback to add indexes to the table
+ * @returns TableDefinition with replication fields injected
+ *
+ * @example
+ * ```typescript
+ * // Simple table with hard delete support
+ * tasks: replicatedTable({
+ *   id: v.string(),
+ *   text: v.string(),
+ * })
+ *
+ * // With indexes
+ * tasks: replicatedTable(
+ *   {
+ *     id: v.string(),
+ *     text: v.string(),
+ *   },
+ *   (table) => table
+ *     .index('by_id', ['id'])
+ *     .index('by_timestamp', ['timestamp'])
+ * )
+ * ```
+ */
+export function replicatedTable(
+  userFields: Record<string, any>,
+  applyIndexes?: (table: any) => any
+): any {
+  // Create table with user fields + replication metadata
+  const tableWithMetadata = defineTable({
+    ...userFields,
+
+    // Injected replication fields (hidden from user's mental model)
+    version: v.number(),
+    timestamp: v.number(),
+  });
+
+  // Apply user-defined indexes if provided
+  if (applyIndexes) {
+    return applyIndexes(tableWithMetadata);
+  }
+
+  return tableWithMetadata;
+}

package/src/server/ssr.ts

@@ -0,0 +1,106 @@
+/**
+ * Server-Side Rendering (SSR) Utilities
+ *
+ * This module provides utilities for loading collection data during
+ * server-side rendering. Use `loadCollection` with an explicit config
+ * object for clarity and type safety.
+ *
+ * @module ssr
+ * @example
+ * ```typescript
+ * import { loadCollection } from '@convex-replicate/core/ssr';
+ * import { api } from '../convex/_generated/api';
+ *
+ * const tasks = await loadCollection<Task>(httpClient, {
+ *   api: api.tasks,
+ *   collection: 'tasks',
+ *   limit: 100,
+ * });
+ * ```
+ */
+
+import type { ConvexHttpClient } from 'convex/browser';
+import type { FunctionReference } from 'convex/server';
+
+/**
+ * API module shape expected by loadCollection.
+ *
+ * This should match the generated API module for your collection
+ * (e.g., api.tasks, api.users, etc.)
+ */
+export type CollectionAPI = {
+  stream: FunctionReference<'query', 'public' | 'internal'>;
+};
+
+/**
+ * Configuration for loading collection data during SSR.
+ */
+export interface LoadCollectionConfig {
+  /** The API module for the collection (e.g., api.tasks) */
+  api: CollectionAPI;
+  /** The collection name (should match the API module name) */
+  collection: string;
+  /** Maximum number of items to load (default: 100) */
+  limit?: number;
+}
+
+/**
+ * Load collection data for server-side rendering.
+ *
+ * **IMPORTANT**: This function is currently limited because `stream` only returns
+ * CRDT bytes, not materialized documents. For most SSR use cases, it's recommended to
+ * create a separate query that reads from your main table instead.
+ *
+ * @deprecated Consider creating a dedicated SSR query instead. See example below.
+ *
+ * @param httpClient - Convex HTTP client for server-side queries
+ * @param config - Configuration object with api, collection, and options
+ * @returns Promise resolving to array of items from the collection
+ *
+ * @example
+ * **Recommended SSR Pattern:**
+ * ```typescript
+ * // convex/tasks.ts
+ * export const list = query({
+ *   handler: async (ctx) => {
+ *     return await ctx.db
+ *       .query('tasks')
+ *       .filter((q) => q.neq(q.field('deleted'), true))
+ *       .collect();
+ *   },
+ * });
+ *
+ * // In your route loader
+ * import { ConvexHttpClient } from 'convex/browser';
+ * import { api } from '../convex/_generated/api';
+ *
+ * const httpClient = new ConvexHttpClient(import.meta.env.VITE_CONVEX_URL);
+ * const tasks = await httpClient.query(api.tasks.list);
+ * ```
+ */
+export async function loadCollection<TItem extends { id: string }>(
+  httpClient: ConvexHttpClient,
+  config: LoadCollectionConfig
+): Promise<ReadonlyArray<TItem>> {
+  // NOTE: This implementation is limited because stream only returns CRDT bytes,
+  // not materialized documents. The code below attempts to construct items but
+  // `change.document` does not exist in the actual stream response.
+  //
+  // For production use, create a dedicated query that reads from your main table.
+
+  const result = await httpClient.query(config.api.stream as any, {
+    collectionName: config.collection,
+    checkpoint: { lastModified: 0 },
+    limit: config.limit ?? 100,
+  });
+
+  const items: TItem[] = [];
+  for (const change of result.changes) {
+    // FIXME: change.document doesn't exist - stream only returns crdtBytes
+    // This code is here for backwards compatibility but won't work correctly
+    const item = { id: change.documentId, ...change.document } as TItem;
+    items.push(item);
+  }
+
+  return items;
+}