@trestleinc/replicate 0.1.0

package/dist/index.js

@@ -0,0 +1,375 @@
+import { componentsGeneric } from "convex/server";
+import { NonRetriableError, startOfflineExecutor } from "@tanstack/offline-transactions";
+import { getLogger } from "@logtape/logtape";
+import * as __WEBPACK_EXTERNAL_MODULE_yjs__ from "yjs";
+componentsGeneric();
+class ReplicateStorage {
+    component;
+    collectionName;
+    constructor(component, collectionName){
+        this.component = component;
+        this.collectionName = collectionName;
+    }
+    async insertDocument(ctx, documentId, crdtBytes, version) {
+        return ctx.runMutation(this.component.public.insertDocument, {
+            collectionName: this.collectionName,
+            documentId,
+            crdtBytes,
+            version
+        });
+    }
+    async updateDocument(ctx, documentId, crdtBytes, version) {
+        return ctx.runMutation(this.component.public.updateDocument, {
+            collectionName: this.collectionName,
+            documentId,
+            crdtBytes,
+            version
+        });
+    }
+    async deleteDocument(ctx, documentId, crdtBytes, version) {
+        return ctx.runMutation(this.component.public.deleteDocument, {
+            collectionName: this.collectionName,
+            documentId,
+            crdtBytes,
+            version
+        });
+    }
+    async stream(ctx, checkpoint, limit) {
+        return ctx.runQuery(this.component.public.stream, {
+            collectionName: this.collectionName,
+            checkpoint,
+            limit
+        });
+    }
+}
+function logger_getLogger(category) {
+    return getLogger([
+        'convex-replicate',
+        ...category
+    ]);
+}
+const logger = logger_getLogger([
+    'convex-replicate',
+    'collection'
+]);
+function convexCollectionOptions({ getKey, initialData, convexClient, api, collectionName }) {
+    const ydoc = new __WEBPACK_EXTERNAL_MODULE_yjs__.Doc({
+        guid: collectionName
+    });
+    const ymap = ydoc.getMap(collectionName);
+    let pendingUpdate = null;
+    ydoc.on('update', (update, origin)=>{
+        pendingUpdate = update;
+        logger.debug('Yjs update event fired', {
+            collectionName,
+            updateSize: update.length,
+            origin
+        });
+    });
+    return {
+        id: collectionName,
+        getKey,
+        _convexClient: convexClient,
+        _collectionName: collectionName,
+        onInsert: async ({ transaction })=>{
+            logger.debug('onInsert handler called', {
+                collectionName,
+                mutationCount: transaction.mutations.length
+            });
+            try {
+                ydoc.transact(()=>{
+                    transaction.mutations.forEach((mut)=>{
+                        const itemYMap = new __WEBPACK_EXTERNAL_MODULE_yjs__.Map();
+                        Object.entries(mut.modified).forEach(([k, v])=>{
+                            itemYMap.set(k, v);
+                        });
+                        ymap.set(String(mut.key), itemYMap);
+                    });
+                }, 'insert');
+                if (pendingUpdate) {
+                    logger.debug('Sending insert delta to Convex', {
+                        collectionName,
+                        documentId: String(transaction.mutations[0].key),
+                        deltaSize: pendingUpdate.length
+                    });
+                    await convexClient.mutation(api.insertDocument, {
+                        collectionName,
+                        documentId: String(transaction.mutations[0].key),
+                        crdtBytes: pendingUpdate.buffer,
+                        materializedDoc: transaction.mutations[0].modified,
+                        version: Date.now()
+                    });
+                    pendingUpdate = null;
+                    logger.info('Insert persisted to Convex', {
+                        collectionName,
+                        documentId: String(transaction.mutations[0].key)
+                    });
+                }
+            } catch (error) {
+                logger.error('Insert failed', {
+                    collectionName,
+                    error: error?.message,
+                    status: error?.status
+                });
+                if (error?.status === 401 || error?.status === 403) throw new NonRetriableError('Authentication failed');
+                if (error?.status === 422) throw new NonRetriableError('Validation error');
+                throw error;
+            }
+        },
+        onUpdate: async ({ transaction })=>{
+            logger.debug('onUpdate handler called', {
+                collectionName,
+                mutationCount: transaction.mutations.length
+            });
+            try {
+                ydoc.transact(()=>{
+                    transaction.mutations.forEach((mut)=>{
+                        const itemYMap = ymap.get(String(mut.key));
+                        if (itemYMap) Object.entries(mut.modified || {}).forEach(([k, v])=>{
+                            itemYMap.set(k, v);
+                        });
+                        else {
+                            const newYMap = new __WEBPACK_EXTERNAL_MODULE_yjs__.Map();
+                            Object.entries(mut.modified).forEach(([k, v])=>{
+                                newYMap.set(k, v);
+                            });
+                            ymap.set(String(mut.key), newYMap);
+                        }
+                    });
+                }, 'update');
+                if (pendingUpdate) {
+                    logger.debug('Sending update delta to Convex', {
+                        collectionName,
+                        documentId: String(transaction.mutations[0].key),
+                        deltaSize: pendingUpdate.length
+                    });
+                    await convexClient.mutation(api.updateDocument, {
+                        collectionName,
+                        documentId: String(transaction.mutations[0].key),
+                        crdtBytes: pendingUpdate.buffer,
+                        materializedDoc: transaction.mutations[0].modified,
+                        version: Date.now()
+                    });
+                    pendingUpdate = null;
+                    logger.info('Update persisted to Convex', {
+                        collectionName,
+                        documentId: String(transaction.mutations[0].key)
+                    });
+                }
+            } catch (error) {
+                logger.error('Update failed', {
+                    collectionName,
+                    error: error?.message,
+                    status: error?.status
+                });
+                if (error?.status === 401 || error?.status === 403) throw new NonRetriableError('Authentication failed');
+                if (error?.status === 422) throw new NonRetriableError('Validation error');
+                throw error;
+            }
+        },
+        onDelete: async ({ transaction })=>{
+            logger.debug('onDelete handler called', {
+                collectionName,
+                mutationCount: transaction.mutations.length
+            });
+            try {
+                ydoc.transact(()=>{
+                    transaction.mutations.forEach((mut)=>{
+                        ymap.delete(String(mut.key));
+                    });
+                }, 'delete');
+                if (pendingUpdate) {
+                    logger.debug('Sending delete delta to Convex', {
+                        collectionName,
+                        documentId: String(transaction.mutations[0].key),
+                        deltaSize: pendingUpdate.length
+                    });
+                    await convexClient.mutation(api.deleteDocument, {
+                        collectionName,
+                        documentId: String(transaction.mutations[0].key),
+                        crdtBytes: pendingUpdate.buffer,
+                        version: Date.now()
+                    });
+                    pendingUpdate = null;
+                    logger.info('Delete persisted to Convex', {
+                        collectionName,
+                        documentId: String(transaction.mutations[0].key)
+                    });
+                }
+            } catch (error) {
+                logger.error('Delete failed', {
+                    collectionName,
+                    error: error?.message,
+                    status: error?.status
+                });
+                if (error?.status === 401 || error?.status === 403) throw new NonRetriableError('Authentication failed');
+                if (error?.status === 422) throw new NonRetriableError('Validation error');
+                throw error;
+            }
+        },
+        sync: {
+            sync: (params)=>{
+                const { begin, write, commit, markReady } = params;
+                if (initialData && initialData.length > 0) {
+                    ydoc.transact(()=>{
+                        for (const item of initialData){
+                            const key = getKey(item);
+                            const itemYMap = new __WEBPACK_EXTERNAL_MODULE_yjs__.Map();
+                            Object.entries(item).forEach(([k, v])=>{
+                                itemYMap.set(k, v);
+                            });
+                            ymap.set(String(key), itemYMap);
+                        }
+                    }, 'ssr-init');
+                    begin();
+                    for (const item of initialData)write({
+                        type: 'insert',
+                        value: item
+                    });
+                    commit();
+                    logger.debug('Initialized with SSR data', {
+                        collectionName,
+                        count: initialData.length
+                    });
+                }
+                logger.debug("Setting up Convex subscription", {
+                    collectionName
+                });
+                let previousItems = new Map();
+                const subscription = convexClient.onUpdate(api.stream, {}, async (items)=>{
+                    try {
+                        logger.debug("Subscription update received", {
+                            collectionName,
+                            itemCount: items.length
+                        });
+                        const currentItems = new Map();
+                        for (const item of items){
+                            const key = getKey(item);
+                            currentItems.set(key, item);
+                        }
+                        const deletedItems = [];
+                        for (const [prevId, prevItem] of previousItems)if (!currentItems.has(prevId)) deletedItems.push(prevItem);
+                        if (deletedItems.length > 0) logger.info('Detected remote hard deletes', {
+                            collectionName,
+                            deletedCount: deletedItems.length,
+                            deletedIds: deletedItems.map((item)=>getKey(item))
+                        });
+                        begin();
+                        for (const deletedItem of deletedItems){
+                            const deletedId = getKey(deletedItem);
+                            ydoc.transact(()=>{
+                                ymap.delete(String(deletedId));
+                            }, 'remote-delete');
+                            write({
+                                type: 'delete',
+                                value: deletedItem
+                            });
+                        }
+                        ydoc.transact(()=>{
+                            for (const item of items){
+                                const key = getKey(item);
+                                const itemYMap = new __WEBPACK_EXTERNAL_MODULE_yjs__.Map();
+                                Object.entries(item).forEach(([k, v])=>{
+                                    itemYMap.set(k, v);
+                                });
+                                ymap.set(String(key), itemYMap);
+                            }
+                        }, "subscription-sync");
+                        for (const item of items){
+                            const key = getKey(item);
+                            params.collection.has(key) ? write({
+                                type: 'update',
+                                value: item
+                            }) : write({
+                                type: 'insert',
+                                value: item
+                            });
+                        }
+                        commit();
+                        previousItems = currentItems;
+                        logger.debug('Successfully synced items to collection', {
+                            count: items.length,
+                            deletedCount: deletedItems.length
+                        });
+                    } catch (error) {
+                        logger.error("Failed to sync items from subscription", {
+                            error: error.message,
+                            errorName: error.name,
+                            stack: error?.stack,
+                            collectionName,
+                            itemCount: items.length
+                        });
+                        throw error;
+                    }
+                });
+                markReady();
+                return ()=>{
+                    logger.debug("Cleaning up Convex subscription", {
+                        collectionName
+                    });
+                    subscription();
+                };
+            }
+        }
+    };
+}
+function createConvexCollection(rawCollection) {
+    const config = rawCollection.config;
+    const convexClient = config._convexClient;
+    const collectionName = config._collectionName;
+    if (!convexClient || !collectionName) throw new Error("createConvexCollection requires a collection created with convexCollectionOptions. Make sure you pass convexClient and collectionName to convexCollectionOptions.");
+    logger.info('Creating Convex collection with offline support', {
+        collectionName
+    });
+    const offline = startOfflineExecutor({
+        collections: {
+            [collectionName]: rawCollection
+        },
+        mutationFns: {},
+        beforeRetry: (transactions)=>{
+            const cutoff = Date.now() - 86400000;
+            const filtered = transactions.filter((tx)=>{
+                const isRecent = tx.createdAt.getTime() > cutoff;
+                const notExhausted = tx.retryCount < 10;
+                return isRecent && notExhausted;
+            });
+            if (filtered.length < transactions.length) logger.warn('Filtered stale transactions', {
+                collectionName,
+                before: transactions.length,
+                after: filtered.length
+            });
+            return filtered;
+        },
+        onLeadershipChange: (isLeader)=>{
+            logger.info(isLeader ? 'Offline mode active' : 'Online-only mode', {
+                collectionName
+            });
+        },
+        onStorageFailure: (diagnostic)=>{
+            logger.warn('Storage failed - online-only mode', {
+                collectionName,
+                code: diagnostic.code,
+                message: diagnostic.message
+            });
+        }
+    });
+    if (convexClient.connectionState) {
+        const connectionState = convexClient.connectionState();
+        logger.debug('Initial connection state', {
+            collectionName,
+            isConnected: connectionState.isWebSocketConnected
+        });
+    }
+    if ('undefined' != typeof window) window.addEventListener('online', ()=>{
+        logger.info('Network online - notifying offline executor', {
+            collectionName
+        });
+        offline.notifyOnline();
+    });
+    logger.info('Offline support initialized', {
+        collectionName,
+        mode: offline.mode
+    });
+    return rawCollection;
+}
+export { NonRetriableError, ReplicateStorage, __WEBPACK_EXTERNAL_MODULE_yjs__ as Y, convexCollectionOptions, createConvexCollection };

package/dist/server/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Server-side utilities for Convex backend.
+ * Import this in your Convex functions (convex/*.ts files).
+ *
+ * @example
+ * ```typescript
+ * // convex/tasks.ts
+ * import {
+ *   insertDocumentHelper,
+ *   updateDocumentHelper,
+ *   deleteDocumentHelper,
+ *   streamHelper,
+ * } from '@trestleinc/replicate/server';
+ * ```
+ */
+export { insertDocumentHelper, updateDocumentHelper, deleteDocumentHelper, streamHelper, } from './replication.js';
+export { replicatedTable, type ReplicationFields } from './schema.js';

package/dist/server/replication.d.ts

@@ -0,0 +1,122 @@
+import type { GenericDataModel } from 'convex/server';
+/**
+ * Insert a document into both the CRDT component and the main application table.
+ *
+ * DUAL-STORAGE ARCHITECTURE:
+ * This helper implements a dual-storage pattern where documents are stored in two places:
+ *
+ * 1. Component Storage (CRDT Layer):
+ *    - Stores CRDT bytes (from Yjs) for offline-first conflict resolution
+ *    - Handles concurrent updates with automatic merging
+ *    - Provides the source of truth for offline changes
+ *
+ * 2. Main Application Table:
+ *    - Stores materialized documents for efficient querying
+ *    - Used by server-side Convex functions that need to query/join data
+ *    - Optimized for reactive subscriptions and complex queries
+ *
+ * WHY BOTH?
+ * - Component: Handles conflict resolution and offline replication (CRDT bytes)
+ * - Main table: Enables efficient server-side queries (materialized docs)
+ * - Similar to event sourcing: component = event log, main table = read model
+ *
+ * @param ctx - Convex mutation context
+ * @param components - Generated components from Convex
+ * @param tableName - Name of the main application table
+ * @param args - Document data with id, crdtBytes, materializedDoc, and version
+ * @returns Success indicator
+ */
+export declare function insertDocumentHelper<_DataModel extends GenericDataModel>(ctx: unknown, components: unknown, tableName: string, args: {
+    id: string;
+    crdtBytes: ArrayBuffer;
+    materializedDoc: unknown;
+    version: number;
+}): Promise<{
+    success: boolean;
+    metadata: {
+        documentId: string;
+        timestamp: number;
+        version: number;
+        collectionName: string;
+    };
+}>;
+/**
+ * Update a document in both the CRDT component and the main application table.
+ *
+ * @param ctx - Convex mutation context
+ * @param components - Generated components from Convex
+ * @param tableName - Name of the main application table
+ * @param args - Document data with id, crdtBytes, materializedDoc, and version
+ * @returns Success indicator
+ */
+export declare function updateDocumentHelper<_DataModel extends GenericDataModel>(ctx: unknown, components: unknown, tableName: string, args: {
+    id: string;
+    crdtBytes: ArrayBuffer;
+    materializedDoc: unknown;
+    version: number;
+}): Promise<{
+    success: boolean;
+    metadata: {
+        documentId: string;
+        timestamp: number;
+        version: number;
+        collectionName: string;
+    };
+}>;
+/**
+ * HARD delete a document from main table, APPEND deletion delta to component.
+ *
+ * NEW BEHAVIOR (v0.3.0):
+ * - Appends deletion delta to component event log (preserves history)
+ * - Physically removes document from main table (hard delete)
+ * - CRDT history preserved for future recovery features
+ *
+ * @param ctx - Convex mutation context
+ * @param components - Generated components from Convex
+ * @param tableName - Name of the main application table
+ * @param args - Document data with id, crdtBytes (deletion delta), and version
+ * @returns Success indicator with metadata
+ */
+export declare function deleteDocumentHelper<_DataModel extends GenericDataModel>(ctx: unknown, components: unknown, tableName: string, args: {
+    id: string;
+    crdtBytes: ArrayBuffer;
+    version: number;
+}): Promise<{
+    success: boolean;
+    metadata: {
+        documentId: string;
+        timestamp: number;
+        version: number;
+        collectionName: string;
+    };
+}>;
+/**
+ * Stream document changes from the CRDT component storage.
+ *
+ * This reads CRDT bytes from the component (not the main table) to enable
+ * true Y.applyUpdate() conflict resolution on the client.
+ * Can be used for both polling (awaitReplication) and subscriptions (live updates).
+ *
+ * @param ctx - Convex query context
+ * @param components - Generated components from Convex
+ * @param tableName - Name of the collection
+ * @param args - Checkpoint and limit for pagination
+ * @returns Array of changes with CRDT bytes
+ */
+export declare function streamHelper<_DataModel extends GenericDataModel>(ctx: unknown, components: unknown, tableName: string, args: {
+    checkpoint: {
+        lastModified: number;
+    };
+    limit?: number;
+}): Promise<{
+    changes: Array<{
+        documentId: string;
+        crdtBytes: ArrayBuffer;
+        version: number;
+        timestamp: number;
+    }>;
+    checkpoint: {
+        lastModified: number;
+    };
+    hasMore: boolean;
+}>;

package/dist/server/schema.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Schema utilities for defining replicated tables.
+ * Automatically adds replication metadata fields so users don't have to.
+ *
+ * @example
+ * ```typescript
+ * // convex/schema.ts
+ * import { defineSchema } from 'convex/server';
+ * import { v } from 'convex/values';
+ * import { replicatedTable } from '@trestleinc/replicate/server';
+ *
+ * export default defineSchema({
+ *   tasks: replicatedTable(
+ *     {
+ *       id: v.string(),
+ *       text: v.string(),
+ *       isCompleted: v.boolean(),
+ *     },
+ *     (table) => table
+ *       .index('by_id', ['id'])
+ *       .index('by_timestamp', ['timestamp'])
+ *   ),
+ * });
+ * ```
+ */
+/**
+ * Internal replication metadata fields added to every replicated table.
+ * These are managed automatically by the replication layer.
+ */
+export type ReplicationFields = {
+    /** Version number for conflict resolution */
+    version: number;
+    /** Last modification timestamp (Unix ms) */
+    timestamp: number;
+};
+/**
+ * Wraps a table definition to automatically add replication metadata fields.
+ *
+ * Users define their business logic fields, and we inject:
+ * - `version` - For conflict resolution and CRDT versioning
+ * - `timestamp` - For incremental sync and change tracking
+ *
+ * Enables:
+ * - Dual-storage architecture (CRDT component + main table)
+ * - Conflict-free replication across clients
+ * - Hard delete support with CRDT history preservation
+ * - Event sourcing via component storage
+ *
+ * @param userFields - User's business logic fields (id, text, etc.)
+ * @param applyIndexes - Optional callback to add indexes to the table
+ * @returns TableDefinition with replication fields injected
+ *
+ * @example
+ * ```typescript
+ * // Simple table with hard delete support
+ * tasks: replicatedTable({
+ *   id: v.string(),
+ *   text: v.string(),
+ * })
+ *
+ * // With indexes
+ * tasks: replicatedTable(
+ *   {
+ *     id: v.string(),
+ *     text: v.string(),
+ *   },
+ *   (table) => table
+ *     .index('by_id', ['id'])
+ *     .index('by_timestamp', ['timestamp'])
+ * )
+ * ```
+ */
+export declare function replicatedTable(userFields: Record<string, any>, applyIndexes?: (table: any) => any): any;

package/dist/server/ssr.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Server-Side Rendering (SSR) Utilities
+ *
+ * This module provides utilities for loading collection data during
+ * server-side rendering. Use `loadCollection` with an explicit config
+ * object for clarity and type safety.
+ *
+ * @module ssr
+ * @example
+ * ```typescript
+ * import { loadCollection } from '@convex-replicate/core/ssr';
+ * import { api } from '../convex/_generated/api';
+ *
+ * const tasks = await loadCollection<Task>(httpClient, {
+ *   api: api.tasks,
+ *   collection: 'tasks',
+ *   limit: 100,
+ * });
+ * ```
+ */
+import type { ConvexHttpClient } from 'convex/browser';
+import type { FunctionReference } from 'convex/server';
+/**
+ * API module shape expected by loadCollection.
+ *
+ * This should match the generated API module for your collection
+ * (e.g., api.tasks, api.users, etc.)
+ */
+export type CollectionAPI = {
+    stream: FunctionReference<'query', 'public' | 'internal'>;
+};
+/**
+ * Configuration for loading collection data during SSR.
+ */
+export interface LoadCollectionConfig {
+    /** The API module for the collection (e.g., api.tasks) */
+    api: CollectionAPI;
+    /** The collection name (should match the API module name) */
+    collection: string;
+    /** Maximum number of items to load (default: 100) */
+    limit?: number;
+}
+/**
+ * Load collection data for server-side rendering.
+ *
+ * **IMPORTANT**: This function is currently limited because `stream` only returns
+ * CRDT bytes, not materialized documents. For most SSR use cases, it's recommended to
+ * create a separate query that reads from your main table instead.
+ *
+ * @deprecated Consider creating a dedicated SSR query instead. See example below.
+ *
+ * @param httpClient - Convex HTTP client for server-side queries
+ * @param config - Configuration object with api, collection, and options
+ * @returns Promise resolving to array of items from the collection
+ *
+ * @example
+ * **Recommended SSR Pattern:**
+ * ```typescript
+ * // convex/tasks.ts
+ * export const list = query({
+ *   handler: async (ctx) => {
+ *     return await ctx.db
+ *       .query('tasks')
+ *       .filter((q) => q.neq(q.field('deleted'), true))
+ *       .collect();
+ *   },
+ * });
+ *
+ * // In your route loader
+ * import { ConvexHttpClient } from 'convex/browser';
+ * import { api } from '../convex/_generated/api';
+ *
+ * const httpClient = new ConvexHttpClient(import.meta.env.VITE_CONVEX_URL);
+ * const tasks = await httpClient.query(api.tasks.list);
+ * ```
+ */
+export declare function loadCollection<TItem extends {
+    id: string;
+}>(httpClient: ConvexHttpClient, config: LoadCollectionConfig): Promise<ReadonlyArray<TItem>>;