@trestleinc/replicate 1.1.1 → 1.1.2-preview.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +395 -146
- package/dist/client/index.d.ts +311 -19
- package/dist/client/index.js +4027 -0
- package/dist/component/_generated/api.d.ts +13 -17
- package/dist/component/_generated/api.js +24 -4
- package/dist/component/_generated/component.d.ts +79 -77
- package/dist/component/_generated/component.js +1 -0
- package/dist/component/_generated/dataModel.d.ts +12 -15
- package/dist/component/_generated/dataModel.js +1 -0
- package/dist/component/_generated/server.d.ts +19 -22
- package/dist/component/_generated/server.js +65 -1
- package/dist/component/_virtual/rolldown_runtime.js +18 -0
- package/dist/component/convex.config.d.ts +6 -2
- package/dist/component/convex.config.js +7 -3
- package/dist/component/logger.d.ts +10 -6
- package/dist/component/logger.js +25 -28
- package/dist/component/public.d.ts +70 -61
- package/dist/component/public.js +311 -295
- package/dist/component/schema.d.ts +53 -45
- package/dist/component/schema.js +26 -32
- package/dist/component/shared/types.d.ts +9 -0
- package/dist/component/shared/types.js +15 -0
- package/dist/server/index.d.ts +134 -13
- package/dist/server/index.js +368 -0
- package/dist/shared/index.d.ts +27 -3
- package/dist/shared/index.js +1 -2
- package/package.json +34 -29
- package/src/client/collection.ts +339 -306
- package/src/client/errors.ts +9 -9
- package/src/client/index.ts +13 -32
- package/src/client/logger.ts +2 -2
- package/src/client/merge.ts +37 -34
- package/src/client/persistence/custom.ts +84 -0
- package/src/client/persistence/index.ts +9 -46
- package/src/client/persistence/indexeddb.ts +111 -84
- package/src/client/persistence/memory.ts +3 -3
- package/src/client/persistence/sqlite/browser.ts +168 -0
- package/src/client/persistence/sqlite/native.ts +29 -0
- package/src/client/persistence/sqlite/schema.ts +124 -0
- package/src/client/persistence/types.ts +32 -28
- package/src/client/prose-schema.ts +55 -0
- package/src/client/prose.ts +28 -25
- package/src/client/replicate.ts +5 -5
- package/src/client/services/cursor.ts +109 -0
- package/src/component/_generated/component.ts +31 -29
- package/src/component/convex.config.ts +2 -2
- package/src/component/logger.ts +7 -7
- package/src/component/public.ts +225 -237
- package/src/component/schema.ts +18 -15
- package/src/server/builder.ts +20 -7
- package/src/server/index.ts +3 -5
- package/src/server/schema.ts +5 -5
- package/src/server/storage.ts +113 -59
- package/src/shared/index.ts +5 -5
- package/src/shared/types.ts +51 -14
- package/dist/client/collection.d.ts +0 -96
- package/dist/client/errors.d.ts +0 -59
- package/dist/client/logger.d.ts +0 -2
- package/dist/client/merge.d.ts +0 -77
- package/dist/client/persistence/adapters/index.d.ts +0 -8
- package/dist/client/persistence/adapters/opsqlite.d.ts +0 -46
- package/dist/client/persistence/adapters/sqljs.d.ts +0 -83
- package/dist/client/persistence/index.d.ts +0 -49
- package/dist/client/persistence/indexeddb.d.ts +0 -17
- package/dist/client/persistence/memory.d.ts +0 -16
- package/dist/client/persistence/sqlite-browser.d.ts +0 -51
- package/dist/client/persistence/sqlite-level.d.ts +0 -63
- package/dist/client/persistence/sqlite-rn.d.ts +0 -36
- package/dist/client/persistence/sqlite.d.ts +0 -47
- package/dist/client/persistence/types.d.ts +0 -42
- package/dist/client/prose.d.ts +0 -56
- package/dist/client/replicate.d.ts +0 -40
- package/dist/client/services/checkpoint.d.ts +0 -18
- package/dist/client/services/reconciliation.d.ts +0 -24
- package/dist/index.js +0 -1618
- package/dist/server/builder.d.ts +0 -94
- package/dist/server/schema.d.ts +0 -27
- package/dist/server/storage.d.ts +0 -80
- package/dist/server.js +0 -281
- package/dist/shared/types.d.ts +0 -50
- package/dist/shared/types.js +0 -6
- package/dist/shared.js +0 -6
- package/src/client/persistence/adapters/index.ts +0 -8
- package/src/client/persistence/adapters/opsqlite.ts +0 -54
- package/src/client/persistence/adapters/sqljs.ts +0 -128
- package/src/client/persistence/sqlite-browser.ts +0 -107
- package/src/client/persistence/sqlite-level.ts +0 -407
- package/src/client/persistence/sqlite-rn.ts +0 -44
- package/src/client/persistence/sqlite.ts +0 -160
- package/src/client/services/checkpoint.ts +0 -86
- package/src/client/services/reconciliation.ts +0 -108
package/README.md
CHANGED
|
@@ -14,7 +14,7 @@ sequenceDiagram
|
|
|
14
14
|
participant UI as React Component
|
|
15
15
|
participant Collection as TanStack DB Collection
|
|
16
16
|
participant Yjs as Yjs CRDT
|
|
17
|
-
participant Storage as Local Storage<br/>(
|
|
17
|
+
participant Storage as Local Storage<br/>(SQLite)
|
|
18
18
|
participant Convex as Convex Backend
|
|
19
19
|
participant Table as Main Table
|
|
20
20
|
|
|
@@ -41,7 +41,7 @@ graph TB
|
|
|
41
41
|
subgraph Client
|
|
42
42
|
TDB[TanStack DB]
|
|
43
43
|
Yjs[Yjs CRDT]
|
|
44
|
-
Local[(
|
|
44
|
+
Local[(SQLite)]
|
|
45
45
|
TDB <--> Yjs
|
|
46
46
|
Yjs <--> Local
|
|
47
47
|
end
|
|
@@ -111,7 +111,7 @@ export default defineSchema({
|
|
|
111
111
|
isCompleted: v.boolean(),
|
|
112
112
|
},
|
|
113
113
|
(t) => t
|
|
114
|
-
.index('
|
|
114
|
+
.index('by_doc_id', ['id']) // Required for document lookups
|
|
115
115
|
.index('by_timestamp', ['timestamp']) // Required for incremental sync
|
|
116
116
|
),
|
|
117
117
|
});
|
|
@@ -122,7 +122,7 @@ export default defineSchema({
|
|
|
122
122
|
- You only define your business logic fields
|
|
123
123
|
|
|
124
124
|
**Required indexes:**
|
|
125
|
-
- `
|
|
125
|
+
- `by_doc_id` on `['id']` - Enables fast document lookups during updates
|
|
126
126
|
- `by_timestamp` on `['timestamp']` - Enables efficient incremental synchronization
|
|
127
127
|
|
|
128
128
|
### Step 3: Create Replication Functions
|
|
@@ -144,73 +144,93 @@ export const {
|
|
|
144
144
|
insert,
|
|
145
145
|
update,
|
|
146
146
|
remove,
|
|
147
|
+
mark, // Peer sync progress tracking
|
|
148
|
+
compact, // Manual compaction trigger
|
|
147
149
|
} = r<Task>({
|
|
148
150
|
collection: 'tasks',
|
|
149
|
-
compaction: {
|
|
151
|
+
compaction: {
|
|
152
|
+
sizeThreshold: "5mb", // Type-safe size: "100kb", "5mb", "1gb"
|
|
153
|
+
peerTimeout: "24h", // Type-safe duration: "30m", "24h", "7d"
|
|
154
|
+
},
|
|
150
155
|
});
|
|
151
156
|
```
|
|
152
157
|
|
|
153
158
|
**What `replicate()` generates:**
|
|
154
159
|
|
|
155
|
-
- `stream` - Real-time CRDT stream query (
|
|
160
|
+
- `stream` - Real-time CRDT stream query (cursor-based subscriptions with `seq` numbers)
|
|
156
161
|
- `material` - SSR-friendly query (for server-side rendering)
|
|
157
162
|
- `recovery` - State vector sync query (for startup reconciliation)
|
|
158
163
|
- `insert` - Dual-storage insert mutation (auto-compacts when threshold exceeded)
|
|
159
164
|
- `update` - Dual-storage update mutation (auto-compacts when threshold exceeded)
|
|
160
165
|
- `remove` - Dual-storage delete mutation (auto-compacts when threshold exceeded)
|
|
166
|
+
- `mark` - Report sync progress to server (peer tracking for safe compaction)
|
|
167
|
+
- `compact` - Manual compaction trigger (peer-aware, respects active peer sync state)
|
|
161
168
|
|
|
162
|
-
### Step 4:
|
|
169
|
+
### Step 4: Define Your Collection
|
|
163
170
|
|
|
164
|
-
Create a
|
|
171
|
+
Create a collection definition using `collection.create()`. This is SSR-safe because persistence and config are deferred until `init()` is called in the browser:
|
|
165
172
|
|
|
166
173
|
```typescript
|
|
167
|
-
// src/
|
|
168
|
-
import {
|
|
169
|
-
import {
|
|
170
|
-
import { api } from '
|
|
171
|
-
import
|
|
172
|
-
import {
|
|
174
|
+
// src/collections/tasks.ts
|
|
175
|
+
import { collection, persistence } from '@trestleinc/replicate/client';
|
|
176
|
+
import { ConvexClient } from 'convex/browser';
|
|
177
|
+
import { api } from '../../convex/_generated/api';
|
|
178
|
+
import initSqlJs from 'sql.js';
|
|
179
|
+
import { z } from 'zod';
|
|
173
180
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
181
|
+
// Define your Zod schema (required)
|
|
182
|
+
const taskSchema = z.object({
|
|
183
|
+
id: z.string(),
|
|
184
|
+
text: z.string(),
|
|
185
|
+
isCompleted: z.boolean(),
|
|
186
|
+
});
|
|
179
187
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
) {
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
);
|
|
198
|
-
}
|
|
199
|
-
return tasksCollection;
|
|
200
|
-
}, [initialData]);
|
|
201
|
-
}
|
|
188
|
+
export type Task = z.infer<typeof taskSchema>;
|
|
189
|
+
|
|
190
|
+
// Create lazy-initialized collection (SSR-safe)
|
|
191
|
+
export const tasks = collection.create({
|
|
192
|
+
// Async factory - only called in browser during init()
|
|
193
|
+
persistence: async () => {
|
|
194
|
+
const SQL = await initSqlJs({ locateFile: (f) => `/${f}` });
|
|
195
|
+
return persistence.sqlite.browser(SQL, 'tasks');
|
|
196
|
+
},
|
|
197
|
+
// Sync factory - only called in browser during init()
|
|
198
|
+
config: () => ({
|
|
199
|
+
schema: taskSchema,
|
|
200
|
+
convexClient: new ConvexClient(import.meta.env.VITE_CONVEX_URL),
|
|
201
|
+
api: api.tasks,
|
|
202
|
+
getKey: (task) => task.id,
|
|
203
|
+
}),
|
|
204
|
+
});
|
|
202
205
|
```
|
|
203
206
|
|
|
204
|
-
|
|
207
|
+
**Key points:**
|
|
208
|
+
- `collection.create()` returns a lazy collection that's safe to import during SSR
|
|
209
|
+
- `persistence` and `config` are factory functions, not values - they're only called during `init()`
|
|
210
|
+
- `schema` is required (Zod schema for type inference and prose field detection)
|
|
211
|
+
- Collection name is auto-extracted from `api.tasks` function path
|
|
212
|
+
|
|
213
|
+
### Step 5: Initialize and Use in Components
|
|
214
|
+
|
|
215
|
+
Initialize the collection once in your app's entry point (browser only), then use it in components:
|
|
216
|
+
|
|
217
|
+
```typescript
|
|
218
|
+
// src/routes/__root.tsx (or app entry point)
|
|
219
|
+
import { tasks } from '../collections/tasks';
|
|
220
|
+
|
|
221
|
+
// Initialize once during app startup (browser only)
|
|
222
|
+
// For SSR frameworks, do this in a client-side effect or loader
|
|
223
|
+
await tasks.init();
|
|
224
|
+
```
|
|
205
225
|
|
|
206
226
|
```typescript
|
|
207
|
-
// src/
|
|
227
|
+
// src/components/TaskList.tsx
|
|
208
228
|
import { useLiveQuery } from '@tanstack/react-db';
|
|
209
|
-
import {
|
|
229
|
+
import { tasks, type Task } from '../collections/tasks';
|
|
210
230
|
|
|
211
231
|
export function TaskList() {
|
|
212
|
-
const collection =
|
|
213
|
-
const { data:
|
|
232
|
+
const collection = tasks.get();
|
|
233
|
+
const { data: taskList, isLoading, isError } = useLiveQuery(collection);
|
|
214
234
|
|
|
215
235
|
const handleCreate = () => {
|
|
216
236
|
collection.insert({
|
|
@@ -243,7 +263,7 @@ export function TaskList() {
|
|
|
243
263
|
<div>
|
|
244
264
|
<button onClick={handleCreate}>Add Task</button>
|
|
245
265
|
|
|
246
|
-
{
|
|
266
|
+
{taskList.map((task) => (
|
|
247
267
|
<div key={task.id}>
|
|
248
268
|
<input
|
|
249
269
|
type="checkbox"
|
|
@@ -259,9 +279,14 @@ export function TaskList() {
|
|
|
259
279
|
}
|
|
260
280
|
```
|
|
261
281
|
|
|
282
|
+
**Lifecycle:**
|
|
283
|
+
1. `collection.create()` - Define collection (module-level, SSR-safe)
|
|
284
|
+
2. `await tasks.init()` - Initialize persistence and config (browser only, call once)
|
|
285
|
+
3. `tasks.get()` - Get the TanStack DB collection instance (after init)
|
|
286
|
+
|
|
262
287
|
### Step 6: Server-Side Rendering (Recommended)
|
|
263
288
|
|
|
264
|
-
For frameworks that support SSR (TanStack Start, Next.js, Remix, SvelteKit), preloading data on the server
|
|
289
|
+
For frameworks that support SSR (TanStack Start, Next.js, Remix, SvelteKit), preloading data on the server enables instant page loads.
|
|
265
290
|
|
|
266
291
|
**Why SSR is recommended:**
|
|
267
292
|
- **Instant page loads** - No loading spinners on first render
|
|
@@ -269,55 +294,121 @@ For frameworks that support SSR (TanStack Start, Next.js, Remix, SvelteKit), pre
|
|
|
269
294
|
- **Reduced client work** - Data already available on hydration
|
|
270
295
|
- **Seamless transition** - Real-time sync takes over after hydration
|
|
271
296
|
|
|
272
|
-
**Step 1:
|
|
273
|
-
|
|
274
|
-
The `material` query is automatically generated by `replicate()` and returns all documents for SSR hydration.
|
|
297
|
+
**Step 1: Prefetch material on the server**
|
|
275
298
|
|
|
276
|
-
|
|
299
|
+
Use `ConvexHttpClient` to fetch data during SSR. The `material` query is generated by `replicate()`:
|
|
277
300
|
|
|
278
301
|
```typescript
|
|
279
|
-
// src/routes/
|
|
280
|
-
import {
|
|
302
|
+
// TanStack Start: src/routes/__root.tsx
|
|
303
|
+
import { createRootRoute } from '@tanstack/react-router';
|
|
281
304
|
import { ConvexHttpClient } from 'convex/browser';
|
|
282
305
|
import { api } from '../convex/_generated/api';
|
|
283
|
-
import type { Task } from '../useTasks';
|
|
284
306
|
|
|
285
307
|
const httpClient = new ConvexHttpClient(import.meta.env.VITE_CONVEX_URL);
|
|
286
308
|
|
|
287
|
-
export const Route =
|
|
309
|
+
export const Route = createRootRoute({
|
|
288
310
|
loader: async () => {
|
|
289
|
-
const
|
|
290
|
-
return {
|
|
311
|
+
const tasksMaterial = await httpClient.query(api.tasks.material);
|
|
312
|
+
return { tasksMaterial };
|
|
291
313
|
},
|
|
292
314
|
});
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
```typescript
|
|
318
|
+
// SvelteKit: src/routes/+layout.server.ts
|
|
319
|
+
import { ConvexHttpClient } from 'convex/browser';
|
|
320
|
+
import { api } from '../convex/_generated/api';
|
|
321
|
+
import { PUBLIC_CONVEX_URL } from '$env/static/public';
|
|
322
|
+
|
|
323
|
+
const httpClient = new ConvexHttpClient(PUBLIC_CONVEX_URL);
|
|
324
|
+
|
|
325
|
+
export async function load() {
|
|
326
|
+
const tasksMaterial = await httpClient.query(api.tasks.material);
|
|
327
|
+
return { tasksMaterial };
|
|
328
|
+
}
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
**Step 2: Pass material to init() on the client**
|
|
332
|
+
|
|
333
|
+
```typescript
|
|
334
|
+
// TanStack Start: src/routes/__root.tsx (client component)
|
|
335
|
+
import { tasks } from '../collections/tasks';
|
|
293
336
|
|
|
294
|
-
function
|
|
295
|
-
const {
|
|
337
|
+
function RootComponent() {
|
|
338
|
+
const { tasksMaterial } = Route.useLoaderData();
|
|
296
339
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
340
|
+
useEffect(() => {
|
|
341
|
+
// Initialize with SSR data - no loading state!
|
|
342
|
+
tasks.init(tasksMaterial);
|
|
343
|
+
}, []);
|
|
300
344
|
|
|
301
|
-
return <
|
|
345
|
+
return <Outlet />;
|
|
302
346
|
}
|
|
303
347
|
```
|
|
304
348
|
|
|
305
|
-
|
|
349
|
+
```svelte
|
|
350
|
+
<!-- SvelteKit: src/routes/+layout.svelte -->
|
|
351
|
+
<script lang="ts">
|
|
352
|
+
import { tasks } from '../collections/tasks';
|
|
353
|
+
import { onMount } from 'svelte';
|
|
354
|
+
|
|
355
|
+
export let data; // From +layout.server.ts
|
|
356
|
+
|
|
357
|
+
onMount(async () => {
|
|
358
|
+
await tasks.init(data.tasksMaterial);
|
|
359
|
+
});
|
|
360
|
+
</script>
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
**Note:** If your framework doesn't support SSR, just call `await tasks.init()` without arguments - it will fetch data on mount and show a loading state.
|
|
306
364
|
|
|
307
365
|
## Sync Protocol
|
|
308
366
|
|
|
309
|
-
Replicate uses
|
|
367
|
+
Replicate uses cursor-based sync with peer tracking for safe compaction.
|
|
368
|
+
|
|
369
|
+
### `stream` - Cursor-Based Real-Time Sync
|
|
370
|
+
|
|
371
|
+
The primary sync mechanism uses monotonically increasing sequence numbers (`seq`):
|
|
372
|
+
|
|
373
|
+
1. Client subscribes with last known `cursor` (seq number)
|
|
374
|
+
2. Server returns all changes with `seq > cursor`
|
|
375
|
+
3. Client applies changes and updates local cursor
|
|
376
|
+
4. Client calls `mark` to report sync progress to server
|
|
377
|
+
5. Subscription stays open for live updates
|
|
378
|
+
|
|
379
|
+
This approach enables:
|
|
380
|
+
- **Safe compaction**: Server knows which deltas each peer has synced
|
|
381
|
+
- **Peer tracking**: Active peers are tracked via `mark` calls
|
|
382
|
+
- **No data loss**: Compaction only removes deltas all active peers have received
|
|
383
|
+
|
|
384
|
+
### `mark` - Peer Sync Tracking
|
|
310
385
|
|
|
311
|
-
|
|
386
|
+
Clients report their sync progress to the server:
|
|
312
387
|
|
|
313
|
-
|
|
388
|
+
```typescript
|
|
389
|
+
// Called automatically after applying changes
|
|
390
|
+
await convexClient.mutation(api.tasks.mark, {
|
|
391
|
+
peerId: "client-uuid",
|
|
392
|
+
syncedSeq: 42, // Last processed seq number
|
|
393
|
+
});
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
The server tracks:
|
|
397
|
+
- Which peers are actively syncing
|
|
398
|
+
- Each peer's last synced `seq` number
|
|
399
|
+
- Peer timeout for cleanup (configurable via `peerTimeout`)
|
|
400
|
+
|
|
401
|
+
### `compact` - Peer-Aware Compaction
|
|
314
402
|
|
|
315
|
-
|
|
316
|
-
2. Server returns all deltas since that checkpoint
|
|
317
|
-
3. Client applies deltas and updates checkpoint
|
|
318
|
-
4. Subscription stays open for live updates
|
|
403
|
+
Compaction is safe because it respects peer sync state:
|
|
319
404
|
|
|
320
|
-
|
|
405
|
+
1. Server checks minimum `syncedSeq` across all active peers
|
|
406
|
+
2. Only deletes deltas where `seq < minSyncedSeq`
|
|
407
|
+
3. Ensures no active peer loses data they haven't synced
|
|
408
|
+
|
|
409
|
+
**Compaction triggers:**
|
|
410
|
+
- **Automatic**: When document deltas exceed `sizeThreshold`
|
|
411
|
+
- **Manual**: Via `compact` mutation
|
|
321
412
|
|
|
322
413
|
### `recovery` - State Vector Sync
|
|
323
414
|
|
|
@@ -332,12 +423,7 @@ Used on startup to reconcile client and server state using Yjs state vectors:
|
|
|
332
423
|
**When recovery is used:**
|
|
333
424
|
- App startup (before stream subscription begins)
|
|
334
425
|
- After extended offline periods
|
|
335
|
-
- When
|
|
336
|
-
|
|
337
|
-
**Why both?**
|
|
338
|
-
- `stream` is optimized for real-time (small checkpoint, fast subscription)
|
|
339
|
-
- `recovery` handles cold starts and large gaps efficiently (state vectors)
|
|
340
|
-
- Together they ensure clients always sync correctly regardless of history
|
|
426
|
+
- When cursor-based sync can't satisfy the request (deltas compacted)
|
|
341
427
|
|
|
342
428
|
## Delete Pattern: Hard Delete with Event History
|
|
343
429
|
|
|
@@ -398,6 +484,8 @@ export const {
|
|
|
398
484
|
insert,
|
|
399
485
|
update,
|
|
400
486
|
remove,
|
|
487
|
+
mark,
|
|
488
|
+
compact,
|
|
401
489
|
} = r<Task>({
|
|
402
490
|
collection: 'tasks',
|
|
403
491
|
|
|
@@ -416,6 +504,16 @@ export const {
|
|
|
416
504
|
const userId = await ctx.auth.getUserIdentity();
|
|
417
505
|
if (!userId) throw new Error('Unauthorized');
|
|
418
506
|
},
|
|
507
|
+
evalMark: async (ctx, peerId) => {
|
|
508
|
+
// Validate peer identity
|
|
509
|
+
const userId = await ctx.auth.getUserIdentity();
|
|
510
|
+
if (!userId) throw new Error('Unauthorized');
|
|
511
|
+
},
|
|
512
|
+
evalCompact: async (ctx, documentId) => {
|
|
513
|
+
// Restrict compaction to admin users
|
|
514
|
+
const userId = await ctx.auth.getUserIdentity();
|
|
515
|
+
if (!userId) throw new Error('Unauthorized');
|
|
516
|
+
},
|
|
419
517
|
|
|
420
518
|
// Lifecycle callbacks (on* hooks run AFTER execution)
|
|
421
519
|
onStream: async (ctx, result) => { /* after stream query */ },
|
|
@@ -456,48 +554,82 @@ const binding = await collection.utils.prose(notebookId, 'content');
|
|
|
456
554
|
|
|
457
555
|
### Persistence Providers
|
|
458
556
|
|
|
459
|
-
Choose the right storage backend for your platform
|
|
557
|
+
Choose the right storage backend for your platform. Persistence is configured in the `persistence` factory of `collection.create()`:
|
|
460
558
|
|
|
461
559
|
```typescript
|
|
462
|
-
import {
|
|
463
|
-
|
|
464
|
-
// Browser: IndexedDB (default, no config needed)
|
|
465
|
-
convexCollectionOptions<Task>({
|
|
466
|
-
// ... other options
|
|
467
|
-
persistence: persistence.indexeddb(),
|
|
468
|
-
});
|
|
560
|
+
import { collection, persistence } from '@trestleinc/replicate/client';
|
|
469
561
|
|
|
470
562
|
// Browser SQLite: Uses sql.js WASM with OPFS persistence
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
const
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
563
|
+
export const tasks = collection.create({
|
|
564
|
+
persistence: async () => {
|
|
565
|
+
const initSqlJs = (await import('sql.js')).default;
|
|
566
|
+
const SQL = await initSqlJs({ locateFile: (f) => `/${f}` });
|
|
567
|
+
return persistence.sqlite.browser(SQL, 'my-app-db');
|
|
568
|
+
},
|
|
569
|
+
config: () => ({ /* ... */ }),
|
|
477
570
|
});
|
|
478
571
|
|
|
479
572
|
// React Native SQLite: Uses op-sqlite (native SQLite)
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
573
|
+
export const tasks = collection.create({
|
|
574
|
+
persistence: async () => {
|
|
575
|
+
const { open } = await import('@op-engineering/op-sqlite');
|
|
576
|
+
const db = open({ name: 'my-app-db' });
|
|
577
|
+
return persistence.sqlite.native(db, 'my-app-db');
|
|
578
|
+
},
|
|
579
|
+
config: () => ({ /* ... */ }),
|
|
485
580
|
});
|
|
486
581
|
|
|
487
582
|
// Testing: In-memory (no persistence)
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
583
|
+
export const tasks = collection.create({
|
|
584
|
+
persistence: async () => persistence.memory(),
|
|
585
|
+
config: () => ({ /* ... */ }),
|
|
491
586
|
});
|
|
492
|
-
```
|
|
493
587
|
|
|
494
|
-
|
|
588
|
+
// Custom backend: Implement StorageAdapter interface
|
|
589
|
+
export const tasks = collection.create({
|
|
590
|
+
persistence: async () => persistence.custom(new MyCustomAdapter()),
|
|
591
|
+
config: () => ({ /* ... */ }),
|
|
592
|
+
});
|
|
593
|
+
```
|
|
495
594
|
|
|
496
595
|
**SQLite Browser** - Uses sql.js (SQLite compiled to WASM) with OPFS persistence. You initialize sql.js yourself and pass the SQL object.
|
|
497
596
|
|
|
498
597
|
**SQLite Native** - Uses op-sqlite for React Native. You create the database and pass it.
|
|
499
598
|
|
|
500
|
-
**Memory** - No persistence, useful for testing
|
|
599
|
+
**Memory** - No persistence, useful for testing.
|
|
600
|
+
|
|
601
|
+
**Custom** - Implement `StorageAdapter` for any storage backend.
|
|
602
|
+
|
|
603
|
+
### Custom Storage Backends
|
|
604
|
+
|
|
605
|
+
Implement `StorageAdapter` for custom storage (Chrome extensions, localStorage, cloud storage):
|
|
606
|
+
|
|
607
|
+
```typescript
|
|
608
|
+
import { persistence, type StorageAdapter } from '@trestleinc/replicate/client';
|
|
609
|
+
|
|
610
|
+
class ChromeStorageAdapter implements StorageAdapter {
|
|
611
|
+
async get(key: string): Promise<Uint8Array | undefined> {
|
|
612
|
+
const result = await chrome.storage.local.get(key);
|
|
613
|
+
return result[key] ? new Uint8Array(result[key]) : undefined;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
async set(key: string, value: Uint8Array): Promise<void> {
|
|
617
|
+
await chrome.storage.local.set({ [key]: Array.from(value) });
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
async delete(key: string): Promise<void> {
|
|
621
|
+
await chrome.storage.local.remove(key);
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
async keys(prefix: string): Promise<string[]> {
|
|
625
|
+
const all = await chrome.storage.local.get(null);
|
|
626
|
+
return Object.keys(all).filter(k => k.startsWith(prefix));
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
// Use custom adapter
|
|
631
|
+
const chromePersistence = persistence.custom(new ChromeStorageAdapter());
|
|
632
|
+
```
|
|
501
633
|
|
|
502
634
|
### Logging Configuration
|
|
503
635
|
|
|
@@ -523,42 +655,90 @@ await configure({
|
|
|
523
655
|
|
|
524
656
|
### Client-Side (`@trestleinc/replicate/client`)
|
|
525
657
|
|
|
526
|
-
#### `
|
|
658
|
+
#### `collection.create({ persistence, config })`
|
|
527
659
|
|
|
528
|
-
Creates collection
|
|
660
|
+
Creates a lazy-initialized collection with deferred persistence and config resolution. Both `persistence` and `config` are factory functions that are only called when `init()` is invoked (browser-only).
|
|
661
|
+
|
|
662
|
+
**Parameters:**
|
|
663
|
+
- `persistence` - Async factory function that returns a `Persistence` instance
|
|
664
|
+
- `config` - Sync factory function that returns the collection config (ConvexClient, schema, api, etc.)
|
|
665
|
+
|
|
666
|
+
**Returns:** `LazyCollection` with `init(material?)` and `get()` methods
|
|
667
|
+
|
|
668
|
+
**Example:**
|
|
669
|
+
```typescript
|
|
670
|
+
import { collection, persistence } from '@trestleinc/replicate/client';
|
|
671
|
+
import { ConvexClient } from 'convex/browser';
|
|
672
|
+
import initSqlJs from 'sql.js';
|
|
673
|
+
|
|
674
|
+
export const tasks = collection.create({
|
|
675
|
+
persistence: async () => {
|
|
676
|
+
const SQL = await initSqlJs({ locateFile: (f) => `/${f}` });
|
|
677
|
+
return persistence.sqlite.browser(SQL, 'tasks');
|
|
678
|
+
},
|
|
679
|
+
config: () => ({
|
|
680
|
+
schema: taskSchema,
|
|
681
|
+
convexClient: new ConvexClient(import.meta.env.VITE_CONVEX_URL),
|
|
682
|
+
api: api.tasks,
|
|
683
|
+
getKey: (task) => task.id,
|
|
684
|
+
}),
|
|
685
|
+
});
|
|
686
|
+
|
|
687
|
+
// In your app initialization (browser only):
|
|
688
|
+
// Pass SSR-prefetched material for instant hydration
|
|
689
|
+
await tasks.init(material);
|
|
690
|
+
const collection = tasks.get();
|
|
691
|
+
```
|
|
692
|
+
|
|
693
|
+
**SSR Prefetch (server-side):**
|
|
694
|
+
```typescript
|
|
695
|
+
// SvelteKit: +layout.server.ts
|
|
696
|
+
import { ConvexHttpClient } from 'convex/browser';
|
|
697
|
+
const httpClient = new ConvexHttpClient(PUBLIC_CONVEX_URL);
|
|
698
|
+
|
|
699
|
+
export async function load() {
|
|
700
|
+
const material = await httpClient.query(api.tasks.material);
|
|
701
|
+
return { material };
|
|
702
|
+
}
|
|
703
|
+
```
|
|
704
|
+
|
|
705
|
+
#### Collection Config Options
|
|
706
|
+
|
|
707
|
+
The `config` factory in `collection.create()` accepts these options:
|
|
529
708
|
|
|
530
|
-
**Config:**
|
|
531
709
|
```typescript
|
|
532
|
-
interface
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
710
|
+
interface CollectionConfig<T> {
|
|
711
|
+
schema: ZodObject; // Required: Zod schema for type inference
|
|
712
|
+
getKey: (item: T) => string | number; // Extract unique key from item
|
|
713
|
+
convexClient: ConvexClient; // Convex client instance
|
|
714
|
+
api: { // API from replicate()
|
|
715
|
+
stream: FunctionReference; // Real-time subscription
|
|
536
716
|
insert: FunctionReference; // Insert mutation
|
|
537
717
|
update: FunctionReference; // Update mutation
|
|
538
718
|
remove: FunctionReference; // Delete mutation
|
|
719
|
+
recovery: FunctionReference; // State vector sync
|
|
720
|
+
mark: FunctionReference; // Peer sync tracking
|
|
721
|
+
compact: FunctionReference; // Manual compaction
|
|
722
|
+
material?: FunctionReference; // SSR hydration query
|
|
539
723
|
};
|
|
540
|
-
collection: string;
|
|
541
|
-
getKey: (item: T) => string | number;
|
|
542
|
-
persistence?: Persistence; // Optional: defaults to indexeddbPersistence()
|
|
543
|
-
material?: Materialized<T>; // SSR hydration data
|
|
544
|
-
prose?: Array<keyof T>; // Optional: prose fields for rich text
|
|
545
724
|
undoCaptureTimeout?: number; // Undo stack merge window (default: 500ms)
|
|
546
725
|
}
|
|
547
726
|
```
|
|
548
727
|
|
|
549
|
-
**Returns:** Collection options for `createCollection()`
|
|
550
|
-
|
|
551
728
|
**Example:**
|
|
552
729
|
```typescript
|
|
553
|
-
const
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
730
|
+
export const tasks = collection.create({
|
|
731
|
+
persistence: async () => {
|
|
732
|
+
const SQL = await initSqlJs({ locateFile: (f) => `/${f}` });
|
|
733
|
+
return persistence.sqlite.browser(SQL, 'tasks');
|
|
734
|
+
},
|
|
735
|
+
config: () => ({
|
|
736
|
+
schema: taskSchema,
|
|
558
737
|
getKey: (task) => task.id,
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
)
|
|
738
|
+
convexClient: new ConvexClient(import.meta.env.VITE_CONVEX_URL),
|
|
739
|
+
api: api.tasks,
|
|
740
|
+
}),
|
|
741
|
+
});
|
|
562
742
|
```
|
|
563
743
|
|
|
564
744
|
#### `prose.extract(proseJson)`
|
|
@@ -580,35 +760,54 @@ const plainText = prose.extract(task.content);
|
|
|
580
760
|
#### Persistence Providers
|
|
581
761
|
|
|
582
762
|
```typescript
|
|
583
|
-
import { persistence,
|
|
763
|
+
import { persistence, type StorageAdapter } from '@trestleinc/replicate/client';
|
|
584
764
|
|
|
585
|
-
// Persistence providers
|
|
586
|
-
persistence.indexeddb() // Browser: IndexedDB (default)
|
|
765
|
+
// Persistence providers (use in collection.create persistence factory)
|
|
587
766
|
persistence.sqlite.browser(SQL, name) // Browser: sql.js WASM + OPFS
|
|
588
767
|
persistence.sqlite.native(db, name) // React Native: op-sqlite
|
|
589
|
-
persistence.memory()
|
|
590
|
-
|
|
591
|
-
// SQLite adapters (for advanced use)
|
|
592
|
-
adapters.sqljs // SqlJsAdapter class for browser
|
|
593
|
-
adapters.opsqlite // OPSqliteAdapter class for React Native
|
|
768
|
+
persistence.memory() // Testing: in-memory (no persistence)
|
|
769
|
+
persistence.custom(adapter) // Custom: your StorageAdapter implementation
|
|
594
770
|
```
|
|
595
771
|
|
|
596
|
-
**`persistence.indexeddb()`** - Browser-only, uses y-indexeddb + browser-level.
|
|
597
|
-
|
|
598
772
|
**`persistence.sqlite.browser(SQL, name)`** - Browser SQLite using sql.js WASM. You initialize sql.js and pass the SQL object.
|
|
599
773
|
|
|
600
774
|
**`persistence.sqlite.native(db, name)`** - React Native SQLite using op-sqlite. You create the database and pass it.
|
|
601
775
|
|
|
602
776
|
**`persistence.memory()`** - In-memory, no persistence. Useful for testing.
|
|
603
777
|
|
|
778
|
+
**`persistence.custom(adapter)`** - Custom storage backend. Pass your `StorageAdapter` implementation.
|
|
779
|
+
|
|
780
|
+
#### `StorageAdapter` Interface
|
|
781
|
+
|
|
782
|
+
Implement for custom storage backends:
|
|
783
|
+
|
|
784
|
+
```typescript
|
|
785
|
+
interface StorageAdapter {
|
|
786
|
+
/** Get value by key, returns undefined if not found */
|
|
787
|
+
get(key: string): Promise<Uint8Array | undefined>;
|
|
788
|
+
|
|
789
|
+
/** Set value by key */
|
|
790
|
+
set(key: string, value: Uint8Array): Promise<void>;
|
|
791
|
+
|
|
792
|
+
/** Delete value by key */
|
|
793
|
+
delete(key: string): Promise<void>;
|
|
794
|
+
|
|
795
|
+
/** List all keys matching prefix */
|
|
796
|
+
keys(prefix: string): Promise<string[]>;
|
|
797
|
+
|
|
798
|
+
/** Optional: cleanup when persistence is destroyed */
|
|
799
|
+
close?(): void;
|
|
800
|
+
}
|
|
801
|
+
```
|
|
802
|
+
|
|
604
803
|
#### Error Classes
|
|
605
804
|
|
|
606
805
|
```typescript
|
|
607
806
|
import { errors } from '@trestleinc/replicate/client';
|
|
608
807
|
|
|
609
808
|
errors.Network // Network-related failures
|
|
610
|
-
errors.IDB //
|
|
611
|
-
errors.IDBWrite //
|
|
809
|
+
errors.IDB // Storage read errors
|
|
810
|
+
errors.IDBWrite // Storage write errors
|
|
612
811
|
errors.Reconciliation // Phantom document cleanup errors
|
|
613
812
|
errors.Prose // Rich text field errors
|
|
614
813
|
errors.CollectionNotReady// Collection not initialized
|
|
@@ -644,9 +843,10 @@ Configuration for the bound replicate function.
|
|
|
644
843
|
interface ReplicateConfig<T> {
|
|
645
844
|
collection: string; // Collection name (e.g., 'tasks')
|
|
646
845
|
|
|
647
|
-
// Optional:
|
|
846
|
+
// Optional: Compaction settings with type-safe values
|
|
648
847
|
compaction?: {
|
|
649
|
-
|
|
848
|
+
sizeThreshold?: Size; // Size threshold: "100kb", "5mb", "1gb" (default: "5mb")
|
|
849
|
+
peerTimeout?: Duration; // Peer timeout: "30m", "24h", "7d" (default: "24h")
|
|
650
850
|
};
|
|
651
851
|
|
|
652
852
|
// Optional: Hooks for permissions and lifecycle
|
|
@@ -655,6 +855,8 @@ interface ReplicateConfig<T> {
|
|
|
655
855
|
evalRead?: (ctx, collection) => Promise<void>;
|
|
656
856
|
evalWrite?: (ctx, doc) => Promise<void>;
|
|
657
857
|
evalRemove?: (ctx, documentId) => Promise<void>;
|
|
858
|
+
evalMark?: (ctx, peerId) => Promise<void>;
|
|
859
|
+
evalCompact?: (ctx, documentId) => Promise<void>;
|
|
658
860
|
|
|
659
861
|
// Lifecycle callbacks (run after operation)
|
|
660
862
|
onStream?: (ctx, result) => Promise<void>;
|
|
@@ -668,13 +870,19 @@ interface ReplicateConfig<T> {
|
|
|
668
870
|
}
|
|
669
871
|
```
|
|
670
872
|
|
|
873
|
+
**Type-safe values:**
|
|
874
|
+
- `Size`: `"100kb"`, `"5mb"`, `"1gb"`, etc.
|
|
875
|
+
- `Duration`: `"30m"`, `"24h"`, `"7d"`, etc.
|
|
876
|
+
|
|
671
877
|
**Returns:** Object with generated functions:
|
|
672
|
-
- `stream` - Real-time CRDT stream query (
|
|
878
|
+
- `stream` - Real-time CRDT stream query (cursor-based with `seq` numbers)
|
|
673
879
|
- `material` - SSR-friendly query for hydration
|
|
674
880
|
- `recovery` - State vector sync query (for startup reconciliation)
|
|
675
881
|
- `insert` - Dual-storage insert mutation (auto-compacts when threshold exceeded)
|
|
676
882
|
- `update` - Dual-storage update mutation (auto-compacts when threshold exceeded)
|
|
677
883
|
- `remove` - Dual-storage delete mutation (auto-compacts when threshold exceeded)
|
|
884
|
+
- `mark` - Peer sync tracking mutation (reports `syncedSeq` to server)
|
|
885
|
+
- `compact` - Manual compaction mutation (peer-aware, safe for active clients)
|
|
678
886
|
|
|
679
887
|
#### `schema.table(userFields, applyIndexes?)`
|
|
680
888
|
|
|
@@ -696,7 +904,7 @@ tasks: schema.table(
|
|
|
696
904
|
text: v.string(),
|
|
697
905
|
},
|
|
698
906
|
(t) => t
|
|
699
|
-
.index('
|
|
907
|
+
.index('by_doc_id', ['id'])
|
|
700
908
|
.index('by_timestamp', ['timestamp'])
|
|
701
909
|
)
|
|
702
910
|
```
|
|
@@ -712,27 +920,68 @@ Validator for ProseMirror-compatible JSON fields.
|
|
|
712
920
|
content: schema.prose() // Validates ProseMirror JSON structure
|
|
713
921
|
```
|
|
714
922
|
|
|
923
|
+
### Shared Types (`@trestleinc/replicate/shared`)
|
|
924
|
+
|
|
925
|
+
```typescript
|
|
926
|
+
import type { ProseValue } from '@trestleinc/replicate/shared';
|
|
927
|
+
|
|
928
|
+
// ProseValue - branded type for prose fields in Zod schemas
|
|
929
|
+
// Use the prose() helper from client to create fields of this type
|
|
930
|
+
```
|
|
931
|
+
|
|
932
|
+
## React Native
|
|
933
|
+
|
|
934
|
+
React Native doesn't include the Web Crypto API by default. Install these polyfills:
|
|
935
|
+
|
|
936
|
+
```bash
|
|
937
|
+
npm install react-native-get-random-values react-native-random-uuid
|
|
938
|
+
```
|
|
939
|
+
|
|
940
|
+
Import them at the **very top** of your app's entry point (before any other imports):
|
|
941
|
+
|
|
942
|
+
```javascript
|
|
943
|
+
// index.js or app/_layout.tsx - MUST be first!
|
|
944
|
+
import "react-native-get-random-values";
|
|
945
|
+
import "react-native-random-uuid";
|
|
946
|
+
|
|
947
|
+
// Then your other imports...
|
|
948
|
+
```
|
|
949
|
+
|
|
950
|
+
This provides:
|
|
951
|
+
- `crypto.getRandomValues()` - Required by Yjs for CRDT operations
|
|
952
|
+
- `crypto.randomUUID()` - Used for generating document and peer IDs
|
|
953
|
+
|
|
954
|
+
See [`examples/expo/`](./examples/expo/) for a complete React Native example using Expo.
|
|
955
|
+
|
|
715
956
|
## Examples
|
|
716
957
|
|
|
717
958
|
### Interval - Linear-style Issue Tracker
|
|
718
959
|
|
|
719
960
|
A full-featured offline-first issue tracker built with Replicate, demonstrating real-world usage patterns.
|
|
720
961
|
|
|
721
|
-
|
|
962
|
+
**Live Demo:** [interval.robelest.com](https://interval.robelest.com)
|
|
722
963
|
|
|
723
|
-
|
|
964
|
+
**Source Code:** Available in three framework variants:
|
|
965
|
+
- [`examples/tanstack-start/`](./examples/tanstack-start/) - TanStack Start (React, web)
|
|
966
|
+
- [`examples/sveltekit/`](./examples/sveltekit/) - SvelteKit (Svelte, web)
|
|
967
|
+
- [`examples/expo/`](./examples/expo/) - Expo (React Native, mobile)
|
|
724
968
|
|
|
725
|
-
**
|
|
969
|
+
**Web features demonstrated:**
|
|
726
970
|
- Offline-first with SQLite persistence (sql.js + OPFS)
|
|
727
971
|
- Rich text editing with TipTap + Yjs collaboration
|
|
728
972
|
- PWA with custom service worker
|
|
729
973
|
- Real-time sync across devices
|
|
730
974
|
- Search with client-side text extraction (`prose.extract()`)
|
|
731
975
|
|
|
976
|
+
**Mobile features demonstrated (Expo):**
|
|
977
|
+
- Native SQLite persistence (op-sqlite)
|
|
978
|
+
- Plain TextInput prose binding via `useProseField` hook
|
|
979
|
+
- Crypto polyfills for React Native
|
|
980
|
+
|
|
732
981
|
## Development
|
|
733
982
|
|
|
734
983
|
```bash
|
|
735
|
-
bun run build # Build with
|
|
984
|
+
bun run build # Build with tsdown (includes ESLint + TypeScript checking)
|
|
736
985
|
bun run dev # Watch mode
|
|
737
986
|
bun run clean # Remove build artifacts
|
|
738
987
|
```
|