@trestleinc/replicate 1.1.1 → 1.1.2-preview.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +395 -146
  2. package/dist/client/index.d.ts +311 -19
  3. package/dist/client/index.js +4027 -0
  4. package/dist/component/_generated/api.d.ts +13 -17
  5. package/dist/component/_generated/api.js +24 -4
  6. package/dist/component/_generated/component.d.ts +79 -77
  7. package/dist/component/_generated/component.js +1 -0
  8. package/dist/component/_generated/dataModel.d.ts +12 -15
  9. package/dist/component/_generated/dataModel.js +1 -0
  10. package/dist/component/_generated/server.d.ts +19 -22
  11. package/dist/component/_generated/server.js +65 -1
  12. package/dist/component/_virtual/rolldown_runtime.js +18 -0
  13. package/dist/component/convex.config.d.ts +6 -2
  14. package/dist/component/convex.config.js +7 -3
  15. package/dist/component/logger.d.ts +10 -6
  16. package/dist/component/logger.js +25 -28
  17. package/dist/component/public.d.ts +70 -61
  18. package/dist/component/public.js +311 -295
  19. package/dist/component/schema.d.ts +53 -45
  20. package/dist/component/schema.js +26 -32
  21. package/dist/component/shared/types.d.ts +9 -0
  22. package/dist/component/shared/types.js +15 -0
  23. package/dist/server/index.d.ts +134 -13
  24. package/dist/server/index.js +368 -0
  25. package/dist/shared/index.d.ts +27 -3
  26. package/dist/shared/index.js +1 -2
  27. package/package.json +34 -29
  28. package/src/client/collection.ts +339 -306
  29. package/src/client/errors.ts +9 -9
  30. package/src/client/index.ts +13 -32
  31. package/src/client/logger.ts +2 -2
  32. package/src/client/merge.ts +37 -34
  33. package/src/client/persistence/custom.ts +84 -0
  34. package/src/client/persistence/index.ts +9 -46
  35. package/src/client/persistence/indexeddb.ts +111 -84
  36. package/src/client/persistence/memory.ts +3 -3
  37. package/src/client/persistence/sqlite/browser.ts +168 -0
  38. package/src/client/persistence/sqlite/native.ts +29 -0
  39. package/src/client/persistence/sqlite/schema.ts +124 -0
  40. package/src/client/persistence/types.ts +32 -28
  41. package/src/client/prose-schema.ts +55 -0
  42. package/src/client/prose.ts +28 -25
  43. package/src/client/replicate.ts +5 -5
  44. package/src/client/services/cursor.ts +109 -0
  45. package/src/component/_generated/component.ts +31 -29
  46. package/src/component/convex.config.ts +2 -2
  47. package/src/component/logger.ts +7 -7
  48. package/src/component/public.ts +225 -237
  49. package/src/component/schema.ts +18 -15
  50. package/src/server/builder.ts +20 -7
  51. package/src/server/index.ts +3 -5
  52. package/src/server/schema.ts +5 -5
  53. package/src/server/storage.ts +113 -59
  54. package/src/shared/index.ts +5 -5
  55. package/src/shared/types.ts +51 -14
  56. package/dist/client/collection.d.ts +0 -96
  57. package/dist/client/errors.d.ts +0 -59
  58. package/dist/client/logger.d.ts +0 -2
  59. package/dist/client/merge.d.ts +0 -77
  60. package/dist/client/persistence/adapters/index.d.ts +0 -8
  61. package/dist/client/persistence/adapters/opsqlite.d.ts +0 -46
  62. package/dist/client/persistence/adapters/sqljs.d.ts +0 -83
  63. package/dist/client/persistence/index.d.ts +0 -49
  64. package/dist/client/persistence/indexeddb.d.ts +0 -17
  65. package/dist/client/persistence/memory.d.ts +0 -16
  66. package/dist/client/persistence/sqlite-browser.d.ts +0 -51
  67. package/dist/client/persistence/sqlite-level.d.ts +0 -63
  68. package/dist/client/persistence/sqlite-rn.d.ts +0 -36
  69. package/dist/client/persistence/sqlite.d.ts +0 -47
  70. package/dist/client/persistence/types.d.ts +0 -42
  71. package/dist/client/prose.d.ts +0 -56
  72. package/dist/client/replicate.d.ts +0 -40
  73. package/dist/client/services/checkpoint.d.ts +0 -18
  74. package/dist/client/services/reconciliation.d.ts +0 -24
  75. package/dist/index.js +0 -1618
  76. package/dist/server/builder.d.ts +0 -94
  77. package/dist/server/schema.d.ts +0 -27
  78. package/dist/server/storage.d.ts +0 -80
  79. package/dist/server.js +0 -281
  80. package/dist/shared/types.d.ts +0 -50
  81. package/dist/shared/types.js +0 -6
  82. package/dist/shared.js +0 -6
  83. package/src/client/persistence/adapters/index.ts +0 -8
  84. package/src/client/persistence/adapters/opsqlite.ts +0 -54
  85. package/src/client/persistence/adapters/sqljs.ts +0 -128
  86. package/src/client/persistence/sqlite-browser.ts +0 -107
  87. package/src/client/persistence/sqlite-level.ts +0 -407
  88. package/src/client/persistence/sqlite-rn.ts +0 -44
  89. package/src/client/persistence/sqlite.ts +0 -160
  90. package/src/client/services/checkpoint.ts +0 -86
  91. package/src/client/services/reconciliation.ts +0 -108
package/README.md CHANGED
@@ -14,7 +14,7 @@ sequenceDiagram
14
14
  participant UI as React Component
15
15
  participant Collection as TanStack DB Collection
16
16
  participant Yjs as Yjs CRDT
17
- participant Storage as Local Storage<br/>(IndexedDB/SQLite)
17
+ participant Storage as Local Storage<br/>(SQLite)
18
18
  participant Convex as Convex Backend
19
19
  participant Table as Main Table
20
20
 
@@ -41,7 +41,7 @@ graph TB
41
41
  subgraph Client
42
42
  TDB[TanStack DB]
43
43
  Yjs[Yjs CRDT]
44
- Local[(IndexedDB/SQLite)]
44
+ Local[(SQLite)]
45
45
  TDB <--> Yjs
46
46
  Yjs <--> Local
47
47
  end
@@ -111,7 +111,7 @@ export default defineSchema({
111
111
  isCompleted: v.boolean(),
112
112
  },
113
113
  (t) => t
114
- .index('by_user_id', ['id']) // Required for document lookups
114
+ .index('by_doc_id', ['id']) // Required for document lookups
115
115
  .index('by_timestamp', ['timestamp']) // Required for incremental sync
116
116
  ),
117
117
  });
@@ -122,7 +122,7 @@ export default defineSchema({
122
122
  - You only define your business logic fields
123
123
 
124
124
  **Required indexes:**
125
- - `by_user_id` on `['id']` - Enables fast document lookups during updates
125
+ - `by_doc_id` on `['id']` - Enables fast document lookups during updates
126
126
  - `by_timestamp` on `['timestamp']` - Enables efficient incremental synchronization
127
127
 
128
128
  ### Step 3: Create Replication Functions
@@ -144,73 +144,93 @@ export const {
144
144
  insert,
145
145
  update,
146
146
  remove,
147
+ mark, // Peer sync progress tracking
148
+ compact, // Manual compaction trigger
147
149
  } = r<Task>({
148
150
  collection: 'tasks',
149
- compaction: { threshold: 5_000_000 }, // Optional: size threshold for auto-compaction (default: 5MB)
151
+ compaction: {
152
+ sizeThreshold: "5mb", // Type-safe size: "100kb", "5mb", "1gb"
153
+ peerTimeout: "24h", // Type-safe duration: "30m", "24h", "7d"
154
+ },
150
155
  });
151
156
  ```
152
157
 
153
158
  **What `replicate()` generates:**
154
159
 
155
- - `stream` - Real-time CRDT stream query (checkpoint-based subscriptions)
160
+ - `stream` - Real-time CRDT stream query (cursor-based subscriptions with `seq` numbers)
156
161
  - `material` - SSR-friendly query (for server-side rendering)
157
162
  - `recovery` - State vector sync query (for startup reconciliation)
158
163
  - `insert` - Dual-storage insert mutation (auto-compacts when threshold exceeded)
159
164
  - `update` - Dual-storage update mutation (auto-compacts when threshold exceeded)
160
165
  - `remove` - Dual-storage delete mutation (auto-compacts when threshold exceeded)
166
+ - `mark` - Report sync progress to server (peer tracking for safe compaction)
167
+ - `compact` - Manual compaction trigger (peer-aware, respects active peer sync state)
161
168
 
162
- ### Step 4: Create a Custom Hook
169
+ ### Step 4: Define Your Collection
163
170
 
164
- Create a hook that wraps TanStack DB with Convex collection options:
171
+ Create a collection definition using `collection.create()`. This is SSR-safe because persistence and config are deferred until `init()` is called in the browser:
165
172
 
166
173
  ```typescript
167
- // src/useTasks.ts
168
- import { createCollection } from '@tanstack/react-db';
169
- import { convexCollectionOptions } from '@trestleinc/replicate/client';
170
- import { api } from '../convex/_generated/api';
171
- import { convexClient } from './router';
172
- import { useMemo } from 'react';
174
+ // src/collections/tasks.ts
175
+ import { collection, persistence } from '@trestleinc/replicate/client';
176
+ import { ConvexClient } from 'convex/browser';
177
+ import { api } from '../../convex/_generated/api';
178
+ import initSqlJs from 'sql.js';
179
+ import { z } from 'zod';
173
180
 
174
- export interface Task {
175
- id: string;
176
- text: string;
177
- isCompleted: boolean;
178
- }
181
+ // Define your Zod schema (required)
182
+ const taskSchema = z.object({
183
+ id: z.string(),
184
+ text: z.string(),
185
+ isCompleted: z.boolean(),
186
+ });
179
187
 
180
- // Module-level singleton to prevent multiple collection instances
181
- // This ensures only one sync process runs, even across component remounts
182
- let tasksCollection: ReturnType<typeof createCollection<Task>>;
183
-
184
- export function useTasks(
185
- initialData?: { documents: Task[], checkpoint?: any, count?: number, crdtBytes?: Uint8Array }
186
- ) {
187
- return useMemo(() => {
188
- if (!tasksCollection) {
189
- tasksCollection = createCollection(
190
- convexCollectionOptions<Task>({
191
- convexClient,
192
- api: api.tasks,
193
- collection: 'tasks',
194
- getKey: (task) => task.id,
195
- material: initialData,
196
- })
197
- );
198
- }
199
- return tasksCollection;
200
- }, [initialData]);
201
- }
188
+ export type Task = z.infer<typeof taskSchema>;
189
+
190
+ // Create lazy-initialized collection (SSR-safe)
191
+ export const tasks = collection.create({
192
+ // Async factory - only called in browser during init()
193
+ persistence: async () => {
194
+ const SQL = await initSqlJs({ locateFile: (f) => `/${f}` });
195
+ return persistence.sqlite.browser(SQL, 'tasks');
196
+ },
197
+ // Sync factory - only called in browser during init()
198
+ config: () => ({
199
+ schema: taskSchema,
200
+ convexClient: new ConvexClient(import.meta.env.VITE_CONVEX_URL),
201
+ api: api.tasks,
202
+ getKey: (task) => task.id,
203
+ }),
204
+ });
202
205
  ```
203
206
 
204
- ### Step 5: Use in Components
207
+ **Key points:**
208
+ - `collection.create()` returns a lazy collection that's safe to import during SSR
209
+ - `persistence` and `config` are factory functions, not values - they're only called during `init()`
210
+ - `schema` is required (Zod schema for type inference and prose field detection)
211
+ - Collection name is auto-extracted from `api.tasks` function path
212
+
213
+ ### Step 5: Initialize and Use in Components
214
+
215
+ Initialize the collection once in your app's entry point (browser only), then use it in components:
216
+
217
+ ```typescript
218
+ // src/routes/__root.tsx (or app entry point)
219
+ import { tasks } from '../collections/tasks';
220
+
221
+ // Initialize once during app startup (browser only)
222
+ // For SSR frameworks, do this in a client-side effect or loader
223
+ await tasks.init();
224
+ ```
205
225
 
206
226
  ```typescript
207
- // src/routes/index.tsx
227
+ // src/components/TaskList.tsx
208
228
  import { useLiveQuery } from '@tanstack/react-db';
209
- import { useTasks } from '../useTasks';
229
+ import { tasks, type Task } from '../collections/tasks';
210
230
 
211
231
  export function TaskList() {
212
- const collection = useTasks();
213
- const { data: tasks, isLoading, isError } = useLiveQuery(collection);
232
+ const collection = tasks.get();
233
+ const { data: taskList, isLoading, isError } = useLiveQuery(collection);
214
234
 
215
235
  const handleCreate = () => {
216
236
  collection.insert({
@@ -243,7 +263,7 @@ export function TaskList() {
243
263
  <div>
244
264
  <button onClick={handleCreate}>Add Task</button>
245
265
 
246
- {tasks.map((task) => (
266
+ {taskList.map((task) => (
247
267
  <div key={task.id}>
248
268
  <input
249
269
  type="checkbox"
@@ -259,9 +279,14 @@ export function TaskList() {
259
279
  }
260
280
  ```
261
281
 
282
+ **Lifecycle:**
283
+ 1. `collection.create()` - Define collection (module-level, SSR-safe)
284
+ 2. `await tasks.init()` - Initialize persistence and config (browser only, call once)
285
+ 3. `tasks.get()` - Get the TanStack DB collection instance (after init)
286
+
262
287
  ### Step 6: Server-Side Rendering (Recommended)
263
288
 
264
- For frameworks that support SSR (TanStack Start, Next.js, Remix, SvelteKit), preloading data on the server is the recommended approach for instant page loads and better SEO.
289
+ For frameworks that support SSR (TanStack Start, Next.js, Remix, SvelteKit), preloading data on the server enables instant page loads.
265
290
 
266
291
  **Why SSR is recommended:**
267
292
  - **Instant page loads** - No loading spinners on first render
@@ -269,55 +294,121 @@ For frameworks that support SSR (TanStack Start, Next.js, Remix, SvelteKit), pre
269
294
  - **Reduced client work** - Data already available on hydration
270
295
  - **Seamless transition** - Real-time sync takes over after hydration
271
296
 
272
- **Step 1: Use the `material` query from replicate()**
273
-
274
- The `material` query is automatically generated by `replicate()` and returns all documents for SSR hydration.
297
+ **Step 1: Prefetch material on the server**
275
298
 
276
- **Step 2: Load data in your route loader**
299
+ Use `ConvexHttpClient` to fetch data during SSR. The `material` query is generated by `replicate()`:
277
300
 
278
301
  ```typescript
279
- // src/routes/index.tsx
280
- import { createFileRoute } from '@tanstack/react-router';
302
+ // TanStack Start: src/routes/__root.tsx
303
+ import { createRootRoute } from '@tanstack/react-router';
281
304
  import { ConvexHttpClient } from 'convex/browser';
282
305
  import { api } from '../convex/_generated/api';
283
- import type { Task } from '../useTasks';
284
306
 
285
307
  const httpClient = new ConvexHttpClient(import.meta.env.VITE_CONVEX_URL);
286
308
 
287
- export const Route = createFileRoute('/')({
309
+ export const Route = createRootRoute({
288
310
  loader: async () => {
289
- const tasks = await httpClient.query(api.tasks.material);
290
- return { tasks };
311
+ const tasksMaterial = await httpClient.query(api.tasks.material);
312
+ return { tasksMaterial };
291
313
  },
292
314
  });
315
+ ```
316
+
317
+ ```typescript
318
+ // SvelteKit: src/routes/+layout.server.ts
319
+ import { ConvexHttpClient } from 'convex/browser';
320
+ import { api } from '../convex/_generated/api';
321
+ import { PUBLIC_CONVEX_URL } from '$env/static/public';
322
+
323
+ const httpClient = new ConvexHttpClient(PUBLIC_CONVEX_URL);
324
+
325
+ export async function load() {
326
+ const tasksMaterial = await httpClient.query(api.tasks.material);
327
+ return { tasksMaterial };
328
+ }
329
+ ```
330
+
331
+ **Step 2: Pass material to init() on the client**
332
+
333
+ ```typescript
334
+ // TanStack Start: src/routes/__root.tsx (client component)
335
+ import { tasks } from '../collections/tasks';
293
336
 
294
- function TasksPage() {
295
- const { tasks: initialTasks } = Route.useLoaderData();
337
+ function RootComponent() {
338
+ const { tasksMaterial } = Route.useLoaderData();
296
339
 
297
- // Pass initialData to your hook - no loading state on first render!
298
- const collection = useTasks(initialTasks);
299
- const { data: tasks } = useLiveQuery(collection);
340
+ useEffect(() => {
341
+ // Initialize with SSR data - no loading state!
342
+ tasks.init(tasksMaterial);
343
+ }, []);
300
344
 
301
- return <TaskList tasks={tasks} />;
345
+ return <Outlet />;
302
346
  }
303
347
  ```
304
348
 
305
- **Note:** If your framework doesn't support SSR, the collection works just fine without `initialData` - it will fetch data on mount and show a loading state.
349
+ ```svelte
350
+ <!-- SvelteKit: src/routes/+layout.svelte -->
351
+ <script lang="ts">
352
+ import { tasks } from '../collections/tasks';
353
+ import { onMount } from 'svelte';
354
+
355
+ export let data; // From +layout.server.ts
356
+
357
+ onMount(async () => {
358
+ await tasks.init(data.tasksMaterial);
359
+ });
360
+ </script>
361
+ ```
362
+
363
+ **Note:** If your framework doesn't support SSR, just call `await tasks.init()` without arguments - it will fetch data on mount and show a loading state.
306
364
 
307
365
  ## Sync Protocol
308
366
 
309
- Replicate uses two complementary sync mechanisms:
367
+ Replicate uses cursor-based sync with peer tracking for safe compaction.
368
+
369
+ ### `stream` - Cursor-Based Real-Time Sync
370
+
371
+ The primary sync mechanism uses monotonically increasing sequence numbers (`seq`):
372
+
373
+ 1. Client subscribes with last known `cursor` (seq number)
374
+ 2. Server returns all changes with `seq > cursor`
375
+ 3. Client applies changes and updates local cursor
376
+ 4. Client calls `mark` to report sync progress to server
377
+ 5. Subscription stays open for live updates
378
+
379
+ This approach enables:
380
+ - **Safe compaction**: Server knows which deltas each peer has synced
381
+ - **Peer tracking**: Active peers are tracked via `mark` calls
382
+ - **No data loss**: Compaction only removes deltas all active peers have received
383
+
384
+ ### `mark` - Peer Sync Tracking
310
385
 
311
- ### `stream` - Real-time Checkpoint Sync
386
+ Clients report their sync progress to the server:
312
387
 
313
- The primary sync mechanism for real-time updates. Uses checkpoint-based incremental sync:
388
+ ```typescript
389
+ // Called automatically after applying changes
390
+ await convexClient.mutation(api.tasks.mark, {
391
+ peerId: "client-uuid",
392
+ syncedSeq: 42, // Last processed seq number
393
+ });
394
+ ```
395
+
396
+ The server tracks:
397
+ - Which peers are actively syncing
398
+ - Each peer's last synced `seq` number
399
+ - Peer timeout for cleanup (configurable via `peerTimeout`)
400
+
401
+ ### `compact` - Peer-Aware Compaction
314
402
 
315
- 1. Client subscribes with last known checkpoint (timestamp)
316
- 2. Server returns all deltas since that checkpoint
317
- 3. Client applies deltas and updates checkpoint
318
- 4. Subscription stays open for live updates
403
+ Compaction is safe because it respects peer sync state:
319
404
 
320
- This is efficient for ongoing sync but requires the server to have deltas going back to the client's checkpoint.
405
+ 1. Server checks minimum `syncedSeq` across all active peers
406
+ 2. Only deletes deltas where `seq < minSyncedSeq`
407
+ 3. Ensures no active peer loses data they haven't synced
408
+
409
+ **Compaction triggers:**
410
+ - **Automatic**: When document deltas exceed `sizeThreshold`
411
+ - **Manual**: Via `compact` mutation
321
412
 
322
413
  ### `recovery` - State Vector Sync
323
414
 
@@ -332,12 +423,7 @@ Used on startup to reconcile client and server state using Yjs state vectors:
332
423
  **When recovery is used:**
333
424
  - App startup (before stream subscription begins)
334
425
  - After extended offline periods
335
- - When checkpoint-based sync can't satisfy the request (deltas compacted)
336
-
337
- **Why both?**
338
- - `stream` is optimized for real-time (small checkpoint, fast subscription)
339
- - `recovery` handles cold starts and large gaps efficiently (state vectors)
340
- - Together they ensure clients always sync correctly regardless of history
426
+ - When cursor-based sync can't satisfy the request (deltas compacted)
341
427
 
342
428
  ## Delete Pattern: Hard Delete with Event History
343
429
 
@@ -398,6 +484,8 @@ export const {
398
484
  insert,
399
485
  update,
400
486
  remove,
487
+ mark,
488
+ compact,
401
489
  } = r<Task>({
402
490
  collection: 'tasks',
403
491
 
@@ -416,6 +504,16 @@ export const {
416
504
  const userId = await ctx.auth.getUserIdentity();
417
505
  if (!userId) throw new Error('Unauthorized');
418
506
  },
507
+ evalMark: async (ctx, peerId) => {
508
+ // Validate peer identity
509
+ const userId = await ctx.auth.getUserIdentity();
510
+ if (!userId) throw new Error('Unauthorized');
511
+ },
512
+ evalCompact: async (ctx, documentId) => {
513
+ // Restrict compaction to admin users
514
+ const userId = await ctx.auth.getUserIdentity();
515
+ if (!userId) throw new Error('Unauthorized');
516
+ },
419
517
 
420
518
  // Lifecycle callbacks (on* hooks run AFTER execution)
421
519
  onStream: async (ctx, result) => { /* after stream query */ },
@@ -456,48 +554,82 @@ const binding = await collection.utils.prose(notebookId, 'content');
456
554
 
457
555
  ### Persistence Providers
458
556
 
459
- Choose the right storage backend for your platform:
557
+ Choose the right storage backend for your platform. Persistence is configured in the `persistence` factory of `collection.create()`:
460
558
 
461
559
  ```typescript
462
- import { persistence, adapters } from '@trestleinc/replicate/client';
463
-
464
- // Browser: IndexedDB (default, no config needed)
465
- convexCollectionOptions<Task>({
466
- // ... other options
467
- persistence: persistence.indexeddb(),
468
- });
560
+ import { collection, persistence } from '@trestleinc/replicate/client';
469
561
 
470
562
  // Browser SQLite: Uses sql.js WASM with OPFS persistence
471
- // You initialize sql.js and pass the SQL object
472
- import initSqlJs from 'sql.js';
473
- const SQL = await initSqlJs({ locateFile: (file) => `/${file}` });
474
- convexCollectionOptions<Task>({
475
- // ... other options
476
- persistence: await persistence.sqlite.browser(SQL, 'my-app-db'),
563
+ export const tasks = collection.create({
564
+ persistence: async () => {
565
+ const initSqlJs = (await import('sql.js')).default;
566
+ const SQL = await initSqlJs({ locateFile: (f) => `/${f}` });
567
+ return persistence.sqlite.browser(SQL, 'my-app-db');
568
+ },
569
+ config: () => ({ /* ... */ }),
477
570
  });
478
571
 
479
572
  // React Native SQLite: Uses op-sqlite (native SQLite)
480
- import { open } from '@op-engineering/op-sqlite';
481
- const db = open({ name: 'my-app-db' });
482
- convexCollectionOptions<Task>({
483
- // ... other options
484
- persistence: await persistence.sqlite.native(db, 'my-app-db'),
573
+ export const tasks = collection.create({
574
+ persistence: async () => {
575
+ const { open } = await import('@op-engineering/op-sqlite');
576
+ const db = open({ name: 'my-app-db' });
577
+ return persistence.sqlite.native(db, 'my-app-db');
578
+ },
579
+ config: () => ({ /* ... */ }),
485
580
  });
486
581
 
487
582
  // Testing: In-memory (no persistence)
488
- convexCollectionOptions<Task>({
489
- // ... other options
490
- persistence: persistence.memory(),
583
+ export const tasks = collection.create({
584
+ persistence: async () => persistence.memory(),
585
+ config: () => ({ /* ... */ }),
491
586
  });
492
- ```
493
587
 
494
- **IndexedDB** (default) - Uses y-indexeddb for Y.Doc persistence and browser-level for metadata. Browser only.
588
+ // Custom backend: Implement StorageAdapter interface
589
+ export const tasks = collection.create({
590
+ persistence: async () => persistence.custom(new MyCustomAdapter()),
591
+ config: () => ({ /* ... */ }),
592
+ });
593
+ ```
495
594
 
496
595
  **SQLite Browser** - Uses sql.js (SQLite compiled to WASM) with OPFS persistence. You initialize sql.js yourself and pass the SQL object.
497
596
 
498
597
  **SQLite Native** - Uses op-sqlite for React Native. You create the database and pass it.
499
598
 
500
- **Memory** - No persistence, useful for testing without IndexedDB side effects.
599
+ **Memory** - No persistence, useful for testing.
600
+
601
+ **Custom** - Implement `StorageAdapter` for any storage backend.
602
+
603
+ ### Custom Storage Backends
604
+
605
+ Implement `StorageAdapter` for custom storage (Chrome extensions, localStorage, cloud storage):
606
+
607
+ ```typescript
608
+ import { persistence, type StorageAdapter } from '@trestleinc/replicate/client';
609
+
610
+ class ChromeStorageAdapter implements StorageAdapter {
611
+ async get(key: string): Promise<Uint8Array | undefined> {
612
+ const result = await chrome.storage.local.get(key);
613
+ return result[key] ? new Uint8Array(result[key]) : undefined;
614
+ }
615
+
616
+ async set(key: string, value: Uint8Array): Promise<void> {
617
+ await chrome.storage.local.set({ [key]: Array.from(value) });
618
+ }
619
+
620
+ async delete(key: string): Promise<void> {
621
+ await chrome.storage.local.remove(key);
622
+ }
623
+
624
+ async keys(prefix: string): Promise<string[]> {
625
+ const all = await chrome.storage.local.get(null);
626
+ return Object.keys(all).filter(k => k.startsWith(prefix));
627
+ }
628
+ }
629
+
630
+ // Use custom adapter
631
+ const chromePersistence = persistence.custom(new ChromeStorageAdapter());
632
+ ```
501
633
 
502
634
  ### Logging Configuration
503
635
 
@@ -523,42 +655,90 @@ await configure({
523
655
 
524
656
  ### Client-Side (`@trestleinc/replicate/client`)
525
657
 
526
- #### `convexCollectionOptions<T>(config)`
658
+ #### `collection.create({ persistence, config })`
527
659
 
528
- Creates collection options for TanStack DB with Yjs CRDT integration.
660
+ Creates a lazy-initialized collection with deferred persistence and config resolution. Both `persistence` and `config` are factory functions that are only called when `init()` is invoked (browser-only).
661
+
662
+ **Parameters:**
663
+ - `persistence` - Async factory function that returns a `Persistence` instance
664
+ - `config` - Sync factory function that returns the collection config (ConvexClient, schema, api, etc.)
665
+
666
+ **Returns:** `LazyCollection` with `init(material?)` and `get()` methods
667
+
668
+ **Example:**
669
+ ```typescript
670
+ import { collection, persistence } from '@trestleinc/replicate/client';
671
+ import { ConvexClient } from 'convex/browser';
672
+ import initSqlJs from 'sql.js';
673
+
674
+ export const tasks = collection.create({
675
+ persistence: async () => {
676
+ const SQL = await initSqlJs({ locateFile: (f) => `/${f}` });
677
+ return persistence.sqlite.browser(SQL, 'tasks');
678
+ },
679
+ config: () => ({
680
+ schema: taskSchema,
681
+ convexClient: new ConvexClient(import.meta.env.VITE_CONVEX_URL),
682
+ api: api.tasks,
683
+ getKey: (task) => task.id,
684
+ }),
685
+ });
686
+
687
+ // In your app initialization (browser only):
688
+ // Pass SSR-prefetched material for instant hydration
689
+ await tasks.init(material);
690
+ const collection = tasks.get();
691
+ ```
692
+
693
+ **SSR Prefetch (server-side):**
694
+ ```typescript
695
+ // SvelteKit: +layout.server.ts
696
+ import { ConvexHttpClient } from 'convex/browser';
697
+ const httpClient = new ConvexHttpClient(PUBLIC_CONVEX_URL);
698
+
699
+ export async function load() {
700
+ const material = await httpClient.query(api.tasks.material);
701
+ return { material };
702
+ }
703
+ ```
704
+
705
+ #### Collection Config Options
706
+
707
+ The `config` factory in `collection.create()` accepts these options:
529
708
 
530
- **Config:**
531
709
  ```typescript
532
- interface ConvexCollectionOptionsConfig<T> {
533
- convexClient: ConvexClient;
534
- api: {
535
- stream: FunctionReference; // Real-time subscription endpoint
710
+ interface CollectionConfig<T> {
711
+ schema: ZodObject; // Required: Zod schema for type inference
712
+ getKey: (item: T) => string | number; // Extract unique key from item
713
+ convexClient: ConvexClient; // Convex client instance
714
+ api: { // API from replicate()
715
+ stream: FunctionReference; // Real-time subscription
536
716
  insert: FunctionReference; // Insert mutation
537
717
  update: FunctionReference; // Update mutation
538
718
  remove: FunctionReference; // Delete mutation
719
+ recovery: FunctionReference; // State vector sync
720
+ mark: FunctionReference; // Peer sync tracking
721
+ compact: FunctionReference; // Manual compaction
722
+ material?: FunctionReference; // SSR hydration query
539
723
  };
540
- collection: string;
541
- getKey: (item: T) => string | number;
542
- persistence?: Persistence; // Optional: defaults to indexeddbPersistence()
543
- material?: Materialized<T>; // SSR hydration data
544
- prose?: Array<keyof T>; // Optional: prose fields for rich text
545
724
  undoCaptureTimeout?: number; // Undo stack merge window (default: 500ms)
546
725
  }
547
726
  ```
548
727
 
549
- **Returns:** Collection options for `createCollection()`
550
-
551
728
  **Example:**
552
729
  ```typescript
553
- const collection = createCollection(
554
- convexCollectionOptions<Task>({
555
- convexClient,
556
- api: api.tasks,
557
- collection: 'tasks',
730
+ export const tasks = collection.create({
731
+ persistence: async () => {
732
+ const SQL = await initSqlJs({ locateFile: (f) => `/${f}` });
733
+ return persistence.sqlite.browser(SQL, 'tasks');
734
+ },
735
+ config: () => ({
736
+ schema: taskSchema,
558
737
  getKey: (task) => task.id,
559
- material: initialData,
560
- })
561
- );
738
+ convexClient: new ConvexClient(import.meta.env.VITE_CONVEX_URL),
739
+ api: api.tasks,
740
+ }),
741
+ });
562
742
  ```
563
743
 
564
744
  #### `prose.extract(proseJson)`
@@ -580,35 +760,54 @@ const plainText = prose.extract(task.content);
580
760
  #### Persistence Providers
581
761
 
582
762
  ```typescript
583
- import { persistence, adapters } from '@trestleinc/replicate/client';
763
+ import { persistence, type StorageAdapter } from '@trestleinc/replicate/client';
584
764
 
585
- // Persistence providers
586
- persistence.indexeddb() // Browser: IndexedDB (default)
765
+ // Persistence providers (use in collection.create persistence factory)
587
766
  persistence.sqlite.browser(SQL, name) // Browser: sql.js WASM + OPFS
588
767
  persistence.sqlite.native(db, name) // React Native: op-sqlite
589
- persistence.memory() // Testing: in-memory (no persistence)
590
-
591
- // SQLite adapters (for advanced use)
592
- adapters.sqljs // SqlJsAdapter class for browser
593
- adapters.opsqlite // OPSqliteAdapter class for React Native
768
+ persistence.memory() // Testing: in-memory (no persistence)
769
+ persistence.custom(adapter) // Custom: your StorageAdapter implementation
594
770
  ```
595
771
 
596
- **`persistence.indexeddb()`** - Browser-only, uses y-indexeddb + browser-level.
597
-
598
772
  **`persistence.sqlite.browser(SQL, name)`** - Browser SQLite using sql.js WASM. You initialize sql.js and pass the SQL object.
599
773
 
600
774
  **`persistence.sqlite.native(db, name)`** - React Native SQLite using op-sqlite. You create the database and pass it.
601
775
 
602
776
  **`persistence.memory()`** - In-memory, no persistence. Useful for testing.
603
777
 
778
+ **`persistence.custom(adapter)`** - Custom storage backend. Pass your `StorageAdapter` implementation.
779
+
780
+ #### `StorageAdapter` Interface
781
+
782
+ Implement for custom storage backends:
783
+
784
+ ```typescript
785
+ interface StorageAdapter {
786
+ /** Get value by key, returns undefined if not found */
787
+ get(key: string): Promise<Uint8Array | undefined>;
788
+
789
+ /** Set value by key */
790
+ set(key: string, value: Uint8Array): Promise<void>;
791
+
792
+ /** Delete value by key */
793
+ delete(key: string): Promise<void>;
794
+
795
+ /** List all keys matching prefix */
796
+ keys(prefix: string): Promise<string[]>;
797
+
798
+ /** Optional: cleanup when persistence is destroyed */
799
+ close?(): void;
800
+ }
801
+ ```
802
+
604
803
  #### Error Classes
605
804
 
606
805
  ```typescript
607
806
  import { errors } from '@trestleinc/replicate/client';
608
807
 
609
808
  errors.Network // Network-related failures
610
- errors.IDB // IndexedDB read errors
611
- errors.IDBWrite // IndexedDB write errors
809
+ errors.IDB // Storage read errors
810
+ errors.IDBWrite // Storage write errors
612
811
  errors.Reconciliation // Phantom document cleanup errors
613
812
  errors.Prose // Rich text field errors
614
813
  errors.CollectionNotReady// Collection not initialized
@@ -644,9 +843,10 @@ Configuration for the bound replicate function.
644
843
  interface ReplicateConfig<T> {
645
844
  collection: string; // Collection name (e.g., 'tasks')
646
845
 
647
- // Optional: Auto-compaction settings
846
+ // Optional: Compaction settings with type-safe values
648
847
  compaction?: {
649
- threshold?: number; // Size threshold in bytes (default: 5MB / 5_000_000)
848
+ sizeThreshold?: Size; // Size threshold: "100kb", "5mb", "1gb" (default: "5mb")
849
+ peerTimeout?: Duration; // Peer timeout: "30m", "24h", "7d" (default: "24h")
650
850
  };
651
851
 
652
852
  // Optional: Hooks for permissions and lifecycle
@@ -655,6 +855,8 @@ interface ReplicateConfig<T> {
655
855
  evalRead?: (ctx, collection) => Promise<void>;
656
856
  evalWrite?: (ctx, doc) => Promise<void>;
657
857
  evalRemove?: (ctx, documentId) => Promise<void>;
858
+ evalMark?: (ctx, peerId) => Promise<void>;
859
+ evalCompact?: (ctx, documentId) => Promise<void>;
658
860
 
659
861
  // Lifecycle callbacks (run after operation)
660
862
  onStream?: (ctx, result) => Promise<void>;
@@ -668,13 +870,19 @@ interface ReplicateConfig<T> {
668
870
  }
669
871
  ```
670
872
 
873
+ **Type-safe values:**
874
+ - `Size`: `"100kb"`, `"5mb"`, `"1gb"`, etc.
875
+ - `Duration`: `"30m"`, `"24h"`, `"7d"`, etc.
876
+
671
877
  **Returns:** Object with generated functions:
672
- - `stream` - Real-time CRDT stream query (checkpoint-based)
878
+ - `stream` - Real-time CRDT stream query (cursor-based with `seq` numbers)
673
879
  - `material` - SSR-friendly query for hydration
674
880
  - `recovery` - State vector sync query (for startup reconciliation)
675
881
  - `insert` - Dual-storage insert mutation (auto-compacts when threshold exceeded)
676
882
  - `update` - Dual-storage update mutation (auto-compacts when threshold exceeded)
677
883
  - `remove` - Dual-storage delete mutation (auto-compacts when threshold exceeded)
884
+ - `mark` - Peer sync tracking mutation (reports `syncedSeq` to server)
885
+ - `compact` - Manual compaction mutation (peer-aware, safe for active clients)
678
886
 
679
887
  #### `schema.table(userFields, applyIndexes?)`
680
888
 
@@ -696,7 +904,7 @@ tasks: schema.table(
696
904
  text: v.string(),
697
905
  },
698
906
  (t) => t
699
- .index('by_user_id', ['id'])
907
+ .index('by_doc_id', ['id'])
700
908
  .index('by_timestamp', ['timestamp'])
701
909
  )
702
910
  ```
@@ -712,27 +920,68 @@ Validator for ProseMirror-compatible JSON fields.
712
920
  content: schema.prose() // Validates ProseMirror JSON structure
713
921
  ```
714
922
 
923
+ ### Shared Types (`@trestleinc/replicate/shared`)
924
+
925
+ ```typescript
926
+ import type { ProseValue } from '@trestleinc/replicate/shared';
927
+
928
+ // ProseValue - branded type for prose fields in Zod schemas
929
+ // Use the prose() helper from client to create fields of this type
930
+ ```
931
+
932
+ ## React Native
933
+
934
+ React Native doesn't include the Web Crypto API by default. Install these polyfills:
935
+
936
+ ```bash
937
+ npm install react-native-get-random-values react-native-random-uuid
938
+ ```
939
+
940
+ Import them at the **very top** of your app's entry point (before any other imports):
941
+
942
+ ```javascript
943
+ // index.js or app/_layout.tsx - MUST be first!
944
+ import "react-native-get-random-values";
945
+ import "react-native-random-uuid";
946
+
947
+ // Then your other imports...
948
+ ```
949
+
950
+ This provides:
951
+ - `crypto.getRandomValues()` - Required by Yjs for CRDT operations
952
+ - `crypto.randomUUID()` - Used for generating document and peer IDs
953
+
954
+ See [`examples/expo/`](./examples/expo/) for a complete React Native example using Expo.
955
+
715
956
  ## Examples
716
957
 
717
958
  ### Interval - Linear-style Issue Tracker
718
959
 
719
960
  A full-featured offline-first issue tracker built with Replicate, demonstrating real-world usage patterns.
720
961
 
721
- 🔗 **Live Demo:** [interval.robelest.com](https://interval.robelest.com)
962
+ **Live Demo:** [interval.robelest.com](https://interval.robelest.com)
722
963
 
723
- 📦 **Source Code:** [github.com/robelest/interval](https://github.com/robelest/interval)
964
+ **Source Code:** Available in three framework variants:
965
+ - [`examples/tanstack-start/`](./examples/tanstack-start/) - TanStack Start (React, web)
966
+ - [`examples/sveltekit/`](./examples/sveltekit/) - SvelteKit (Svelte, web)
967
+ - [`examples/expo/`](./examples/expo/) - Expo (React Native, mobile)
724
968
 
725
- **Features demonstrated:**
969
+ **Web features demonstrated:**
726
970
  - Offline-first with SQLite persistence (sql.js + OPFS)
727
971
  - Rich text editing with TipTap + Yjs collaboration
728
972
  - PWA with custom service worker
729
973
  - Real-time sync across devices
730
974
  - Search with client-side text extraction (`prose.extract()`)
731
975
 
976
+ **Mobile features demonstrated (Expo):**
977
+ - Native SQLite persistence (op-sqlite)
978
+ - Plain TextInput prose binding via `useProseField` hook
979
+ - Crypto polyfills for React Native
980
+
732
981
  ## Development
733
982
 
734
983
  ```bash
735
- bun run build # Build with Rslib (includes ESLint + TypeScript checking)
984
+ bun run build # Build with tsdown (includes ESLint + TypeScript checking)
736
985
  bun run dev # Watch mode
737
986
  bun run clean # Remove build artifacts
738
987
  ```