@fatagnus/dink-sync 1.0.0

package/README.md

@@ -0,0 +1,312 @@
+# @fatagnus/dink-sync
+
+Offline-first sync SDK for Dink edge platform with Effect.ts.
+
+This SDK powers the **Offline-First Edge** type - edges that work offline and automatically sync when connectivity is restored.
+
+| Edge Type | SDK | Use Case |
+|-----------|-----|----------|
+| **Lite Edge** | `@fatagnus/dink-sdk` | Always-connected IoT, RPC services |
+| **Offline-First Edge** | ✅ This SDK | Offline data with auto-sync |
+| **Full Edge** | Not yet available | Local NATS services (planned) |
+
+## Installation
+
+```bash
+npm install @fatagnus/dink-sync
+# or
+pnpm add @fatagnus/dink-sync
+```
+
+## Features
+
+- **Offline-first**: Queue changes locally and sync when online
+- **CRDT-based**: Conflict-free replicated data types for automatic conflict resolution
+- **Real-time sync**: Bidirectional sync with dinkd server via NATS
+- **React hooks**: Ready-to-use hooks for React applications
+- **Persistence**: Pluggable persistence providers (memory, PGlite)
+- **Type-safe**: Full TypeScript support with strict typing
+- **Typed Client Factory**: Generate type-safe collection access from Convex schema
+
+## Quick Start — ⭐ Typed Client (STRONGLY PREFERRED)
+
+**Always use typed clients** generated from your Convex schema. This is the only recommended approach for production applications.
+
+> ⚠️ Do not use the low-level sync engine or generic collection APIs in production. They should only be used for quick prototyping.
+
+### 1. Define Your Schema
+
+```typescript
+// convex/schema.ts
+import { defineSchema, defineTable } from "convex/server";
+import { v } from "convex/values";
+
+export default defineSchema({
+  tasks: defineTable({
+    text: v.string(),
+    isCompleted: v.boolean(),
+    priority: v.optional(v.number()),
+  }),
+  users: defineTable({
+    name: v.string(),
+    email: v.string(),
+  }),
+});
+```
+
+### 2. Generate Typed Client
+
+```bash
+# Generate TypeScript types, validators, and typed client
+dink codegen --convex-schema ./convex --convex-output ./src/generated --zod
+```
+
+### 3. Use the Typed Client
+
+```typescript
+import { offlineEdge } from '@fatagnus/dink-sync';
+import { PGlitePersistence } from '@fatagnus/dink-sync/persistence';
+import { createTypedClient } from './generated';
+
+// Initialize the offline edge client
+const edge = offlineEdge.create({
+  persistence: new PGlitePersistence(),
+  config: {
+    serverUrl: 'nats://localhost:4222',
+    apiKey: 'your-edge-api-key',
+  },
+});
+await edge.init();
+
+// Create the typed client - this is the recommended way!
+const db = createTypedClient(edge.get());
+
+// Type-safe collection access with full IDE autocompletion
+const task = await db.tasks.insert({
+  text: 'Buy milk',
+  isCompleted: false,
+});
+console.log(`Created task: ${task.id}`);
+
+// List all tasks
+const allTasks = await db.tasks.list();
+
+// Update a task
+const updated = await db.tasks.update(task.id, { isCompleted: true });
+
+// Delete a task
+await db.tasks.delete(task.id);
+
+// Validation is automatic!
+try {
+  await db.tasks.insert({ text: '', isCompleted: false });
+} catch (err) {
+  console.error('Validation failed:', err.message); // "text is required"
+}
+```
+
+### Why Typed Client?
+
+| Feature | Typed Client | Generic Collection |
+|---------|--------------|--------------------|
+| Type Safety | ✅ Compile-time checks | ⚠️ Runtime only |
+| IDE Support | ✅ Full autocompletion | ⚠️ Limited |
+| Validation | ✅ Automatic | ❌ Manual setup |
+| Refactoring | ✅ Safe renames | ⚠️ String-based |
+
+## Alternative: Low-Level Sync Engine ⚠️ Advanced/Prototyping Only
+
+For advanced use cases or quick prototyping, you can use the sync engine directly. **Migrate to typed clients before production:**
+
+```typescript
+import { createSyncEngine, NatsTransport } from '@fatagnus/dink-sync/client';
+
+// Create transport and engine
+const transport = new NatsTransport({
+  serverUrl: 'nats://localhost:4222',
+  apiKey: 'your-edge-api-key',
+  appId: 'your-app-id',
+  edgeId: 'your-edge-id',
+});
+
+const engine = createSyncEngine({
+  serverUrl: 'nats://localhost:4222',
+  apiKey: 'your-edge-api-key',
+  transport,
+});
+
+// Connect and register documents
+await engine.connect();
+const actor = await engine.registerDocument('tasks', 'task-1');
+
+// Queue changes
+actor.queueChange(new Uint8Array([1, 2, 3]));
+
+// Listen for sync events
+engine.onSyncComplete((event) => {
+  console.log(`Synced ${event.documentId}`);
+});
+```
+
+## React Integration
+
+```tsx
+import { OfflineEdgeProvider, useCollection } from '@fatagnus/dink-sync/react';
+
+function App() {
+  return (
+    <OfflineEdgeProvider config={{ serverUrl, apiKey, edgeId }}>
+      <TaskList />
+    </OfflineEdgeProvider>
+  );
+}
+
+function TaskList() {
+  const { items, insert, update, delete: remove } = useCollection('tasks');
+  // ...
+}
+```
+
+## Testing
+
+### Unit Tests
+
+Run unit tests (uses mock transport):
+
+```bash
+pnpm test
+```
+
+### E2E Tests
+
+E2E tests use testcontainers to spin up a real dinkd server.
+
+### Browser Tests
+
+Browser tests verify PGlite persistence with IndexedDB in a real browser environment. 
+
+#### Prerequisites
+
+1. **Docker**: Install and ensure Docker daemon is running
+
+2. **dinkd image**: Build the dinkd Docker image from the project root:
+   ```bash
+   # From the project root directory
+   docker build -t dinkd:latest .
+   ```
+
+#### Running E2E Tests
+
+```bash
+pnpm test:e2e
+```
+
+#### Running Browser Tests
+
+**Prerequisites:**
+
+1. **Playwright browsers**: Install Chromium for browser testing:
+   ```bash
+   npx playwright install chromium
+   ```
+
+**Run browser tests:**
+
+```bash
+pnpm test:browser
+```
+
+**Browser test coverage:**
+
+- Data persistence with IndexedDB backend
+- Data persistence across page reloads/sessions
+- Large dataset handling (1000+ documents)
+- Concurrent read/write operations
+- Binary data integrity
+- Special character handling in document IDs
+
+#### E2E Test Coverage
+
+The E2E tests verify:
+
+- Edge connection to real dinkd server
+- Document registration and sync operations
+- External update subscription
+- Concurrent sync from multiple edges
+- Reconnection handling
+- Error handling and event emission
+
+#### Test Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|--------|
+| `DINK_TEST_IMAGE` | Docker image for dinkd | `dinkd:latest` |
+
+#### Troubleshooting Browser Tests
+
+**Playwright not installed:**
+- Run `npx playwright install chromium` to install the browser
+
+**Tests timing out:**
+- Large dataset tests may take longer; timeout is set to 60 seconds
+- Consider running browser tests separately from unit tests
+
+**IndexedDB errors:**
+- Ensure you're running in a supported browser (Chromium)
+- Check for quota limits in browser settings
+
+#### Troubleshooting E2E Tests
+
+**Container startup timeout**:
+- Ensure Docker is running
+- Check that the `dinkd:latest` image exists: `docker images | grep dinkd`
+- Increase startup timeout in test configuration if needed
+
+**Connection refused**:
+- Container may not be fully started
+- Check container logs for errors
+
+**Port conflicts**:
+- Testcontainers automatically maps to random ports, but conflicts can occur
+- Stop any running dinkd containers manually if needed
+
+## API Reference
+
+### SyncEngine
+
+- `connect()`: Connect to dinkd server
+- `disconnect()`: Disconnect from server
+- `destroy()`: Clean up all resources
+- `registerDocument(collection, docId)`: Register a document for sync
+- `unregisterDocument(collection, docId)`: Unregister a document
+- `getConnectionState()`: Get current connection state
+- `onStateChange(callback)`: Subscribe to connection state changes
+- `onSyncStarted(callback)`: Subscribe to sync start events
+- `onSyncComplete(callback)`: Subscribe to sync completion events
+- `onSyncError(callback)`: Subscribe to sync error events
+- `onSyncRejected(callback)`: Subscribe to sync rejection events
+- `goOffline()`: Manually force offline mode
+- `goOnline()`: Exit manual offline mode
+- `discardLocalChanges(collection, docId)`: Discard pending changes for a document
+- `forcePush(collection, docId)`: Force push local state to server
+
+### DocumentActor
+
+- `queueChange(delta)`: Queue a change for sync
+- `hasPendingChanges()`: Check if there are pending changes
+- `onPendingChange(callback)`: Subscribe to pending state changes
+- `getStateVector()`: Get current state vector
+- `setStateVector(vector)`: Set state vector
+- `applyExternalUpdate(update)`: Apply an external update
+- `onExternalUpdate(callback)`: Subscribe to external updates
+
+### ConnectionState
+
+- `Offline`: Not connected to server
+- `Connecting`: Connection in progress
+- `Online`: Connected and ready
+- `Reconnecting`: Connection lost, attempting to reconnect
+
+## License
+
+Apache-2.0

package/dist/client/attachment.d.ts

@@ -0,0 +1,225 @@
+/**
+ * Attachment support for offline-first sync.
+ *
+ * Provides:
+ * - Separate storage for binary attachments
+ * - Chunked upload for large files
+ * - Upload progress tracking
+ * - On-demand vs eager download policies
+ * - LRU caching with eviction
+ */
+import type { PersistenceProvider } from '../persistence/types.js';
+/**
+ * Download policy for attachments.
+ */
+export declare const DownloadPolicy: {
+    /** Download attachment data immediately when fetched */
+    readonly Eager: "eager";
+    /** Only download metadata; data loaded on explicit request */
+    readonly OnDemand: "on-demand";
+};
+export type DownloadPolicy = typeof DownloadPolicy[keyof typeof DownloadPolicy];
+/**
+ * Metadata for an attachment.
+ */
+export interface AttachmentMetadata {
+    /** Unique attachment ID */
+    id: string;
+    /** Original filename */
+    filename: string;
+    /** MIME type of the file */
+    mimeType: string;
+    /** Size in bytes */
+    size: number;
+    /** Checksum for integrity verification */
+    checksum: string;
+    /** Timestamp when the attachment was created */
+    createdAt: number;
+    /** ID of the document this attachment belongs to */
+    documentId: string;
+    /** Collection the document belongs to */
+    collection: string;
+}
+/**
+ * Upload progress information.
+ */
+export interface UploadProgress {
+    /** Percentage complete (0-100) */
+    percent: number;
+    /** Bytes uploaded so far */
+    uploadedBytes: number;
+    /** Total bytes to upload */
+    totalBytes: number;
+    /** Number of chunks completed */
+    chunksCompleted: number;
+    /** Total number of chunks */
+    totalChunks: number;
+}
+/**
+ * Options for uploading an attachment.
+ */
+export interface UploadOptions {
+    /** Original filename */
+    filename: string;
+    /** MIME type */
+    mimeType: string;
+    /** Binary data to upload */
+    data: Uint8Array;
+    /** Document ID to associate with */
+    documentId: string;
+    /** Collection name */
+    collection: string;
+    /** Progress callback */
+    onProgress?: (progress: UploadProgress) => void;
+}
+/**
+ * Result of an upload operation.
+ */
+export interface UploadResult {
+    /** Attachment metadata */
+    metadata: AttachmentMetadata;
+}
+/**
+ * Configuration for AttachmentManager.
+ */
+export interface AttachmentConfig {
+    /** Chunk size for large file uploads (default: 256KB) */
+    chunkSize?: number;
+    /** Maximum cache size in bytes (default: 50MB) */
+    maxCacheSize?: number;
+    /** Download policy (default: Eager) */
+    downloadPolicy?: DownloadPolicy;
+}
+/**
+ * Cache statistics.
+ */
+export interface CacheStats {
+    /** Number of cache hits */
+    hits: number;
+    /** Number of cache misses */
+    misses: number;
+    /** Number of evictions */
+    evictions: number;
+    /** Current cache size in bytes */
+    currentSize: number;
+    /** IDs of cached attachments */
+    cachedIds: string[];
+}
+/**
+ * An attachment with lazy-loadable data.
+ */
+export interface Attachment {
+    /** Attachment metadata */
+    metadata: AttachmentMetadata;
+    /** Binary data (may be undefined if not loaded) */
+    data: Uint8Array | undefined;
+    /** Check if data has been loaded */
+    isDataLoaded(): boolean;
+    /** Load the data (no-op if already loaded) */
+    loadData(): Promise<void>;
+}
+/**
+ * Stored attachment with metadata and data.
+ */
+interface StoredAttachment {
+    metadata: AttachmentMetadata;
+    data: Uint8Array;
+}
+/**
+ * Low-level storage for attachments.
+ * Stores metadata and data separately for efficient access.
+ */
+export declare class AttachmentStorage {
+    private readonly persistence;
+    constructor(persistence: PersistenceProvider);
+    /**
+     * Save an attachment.
+     */
+    save(metadata: AttachmentMetadata, data: Uint8Array): Promise<void>;
+    /**
+     * Load an attachment with its data.
+     */
+    load(attachmentId: string): Promise<StoredAttachment | undefined>;
+    /**
+     * Load just the data for an attachment.
+     */
+    loadData(attachmentId: string): Promise<Uint8Array | undefined>;
+    /**
+     * Get metadata for an attachment without loading data.
+     */
+    getMetadata(attachmentId: string): Promise<AttachmentMetadata | undefined>;
+    /**
+     * List all attachments for a document.
+     */
+    listByDocument(collection: string, documentId: string): Promise<AttachmentMetadata[]>;
+    /**
+     * Delete an attachment.
+     */
+    delete(attachmentId: string): Promise<void>;
+    /**
+     * Update the document-to-attachment index.
+     */
+    private updateIndex;
+    /**
+     * Remove an attachment from the document index.
+     */
+    private removeFromIndex;
+}
+/**
+ * Manages attachments with caching, chunked upload, and progress tracking.
+ */
+export declare class AttachmentManager {
+    private readonly storage;
+    private readonly chunkSize;
+    private readonly maxCacheSize;
+    private readonly downloadPolicy;
+    private readonly cache;
+    private currentCacheSize;
+    private cacheHits;
+    private cacheMisses;
+    private cacheEvictions;
+    constructor(storage: AttachmentStorage, config?: AttachmentConfig);
+    /**
+     * Upload an attachment.
+     */
+    upload(options: UploadOptions): Promise<UploadResult>;
+    /**
+     * Get an attachment by ID.
+     */
+    get(attachmentId: string): Promise<Attachment | undefined>;
+    /**
+     * Split data into chunks.
+     */
+    splitIntoChunks(data: Uint8Array): Uint8Array[];
+    /**
+     * Reassemble chunks into original data.
+     */
+    reassembleChunks(chunks: Uint8Array[]): Uint8Array;
+    /**
+     * Calculate checksum for data.
+     * Uses a simple hash in browser environments.
+     */
+    calculateChecksum(data: Uint8Array): Promise<string>;
+    /**
+     * Get cache statistics.
+     */
+    getCacheStats(): CacheStats;
+    /**
+     * Clear the cache.
+     */
+    clearCache(): void;
+    /**
+     * Add an attachment to the cache, evicting old entries if needed.
+     */
+    private addToCache;
+    /**
+     * Evict the least recently used cache entry.
+     */
+    private evictLRU;
+    /**
+     * Create an Attachment object with lazy loading support.
+     */
+    private createAttachment;
+}
+export {};
+//# sourceMappingURL=attachment.d.ts.map

package/dist/client/attachment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"attachment.d.ts","sourceRoot":"","sources":["../../src/client/attachment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAYnE;;GAEG;AACH,eAAO,MAAM,cAAc;IACzB,wDAAwD;;IAExD,8DAA8D;;CAEtD,CAAC;AAEX,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC,MAAM,OAAO,cAAc,CAAC,CAAC;AAEhF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,2BAA2B;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,UAAU,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,kCAAkC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,eAAe,EAAE,MAAM,CAAC;IACxB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,IAAI,EAAE,UAAU,CAAC;IACjB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,IAAI,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,0BAA0B;IAC1B,QAAQ,EAAE,kBAAkB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uCAAuC;IACvC,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,2BAA2B;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,mDAAmD;IACnD,IAAI,EAAE,UAAU,GAAG,SAAS,CAAC;IAC7B,oCAAoC;IACpC,YAAY,IAAI,OAAO,CAAC;IACxB,8CAA8C;IAC9C,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED;;GAEG;AACH,UAAU,gBAAgB;IACxB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,IAAI,EAAE,UAAU,CAAC;CAClB;AAiBD;;;GAGG;AACH,qBAAa,iBAAiB;IAChB,OAAO,CAAC,QAAQ,CAAC,WAAW;gBAAX,WAAW,EAAE,mBAAmB;IAE7D;;OAEG;IACG,IAAI,CAAC,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAYzE;;OAEG;IACG,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAcvE;;OAEG;IACG,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAIrE;;OAEG;IACG,WAAW,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC;IAahF;;OAEG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAyB3F;;OAEG;IACG,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcjD;;OAEG;YACW,WAAW;IAoBzB;;OAEG;YACW,eAAe;CAsB9B;AAUD;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoB;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAGhD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IACvD,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,cAAc,CAAK;gBAEf,OAAO,EAAE,iBAAiB,EAAE,MAAM,GAAE,gBAAqB;IAOrE;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAgD3D;;OAEG;IACG,GAAG,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAiChE;;OAEG;IACH,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,EAAE;IAa/C;;OAEG;IACH,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,UAAU;IAalD;;;OAGG;IACG,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IA0B1D;;OAEG;IACH,aAAa,IAAI,UAAU;IAU3B;;OAEG;IACH,UAAU,IAAI,IAAI;IAKlB;;OAEG;IACH,OAAO,CAAC,UAAU;IAgBlB;;OAEG;IACH,OAAO,CAAC,QAAQ;IAqBhB;;OAEG;IACH,OAAO,CAAC,gBAAgB;CA4BzB"}