@syncular/relay 0.0.1-60

package/src/relay.ts

@@ -0,0 +1,421 @@
+/**
+ * @syncular/relay - Relay Server
+ *
+ * An edge relay server that acts as a local server to nearby clients
+ * while simultaneously acting as a client to the main server.
+ *
+ * Enables offline-first architectures where local network devices
+ * continue syncing when internet is lost.
+ */
+
+import type { ScopeValues, SyncTransport } from '@syncular/core';
+import type { ServerSyncDialect, TableRegistry } from '@syncular/server';
+import type { Hono } from 'hono';
+import type { Kysely } from 'kysely';
+import { sql } from 'kysely';
+import { ForwardEngine } from './client-role/forward-engine';
+import { PullEngine } from './client-role/pull-engine';
+import { SequenceMapper } from './client-role/sequence-mapper';
+import { ensureRelaySchema } from './migrate';
+import { ModeManager, type RelayMode } from './mode-manager';
+import { RelayRealtime } from './realtime';
+import type { ForwardConflictEntry, RelayDatabase } from './schema';
+
+/**
+ * Events emitted by the relay server.
+ */
+export interface RelayEvents {
+  modeChange: (mode: RelayMode) => void;
+  forwardConflict: (conflict: ForwardConflictEntry) => void;
+  error: (error: Error) => void;
+}
+
+/**
+ * Configuration options for creating a relay server.
+ */
+export interface RelayServerOptions<DB extends RelayDatabase = RelayDatabase> {
+  /** Kysely database instance */
+  db: Kysely<DB>;
+  /** Server sync dialect (e.g., SQLite or Postgres) */
+  dialect: ServerSyncDialect;
+  /** Transport for communicating with the main server */
+  mainServerTransport: SyncTransport;
+  /** Client ID used when communicating with the main server */
+  mainServerClientId: string;
+  /** Actor ID used when communicating with the main server */
+  mainServerActorId: string;
+  /** Tables this relay subscribes to from the main server */
+  tables: string[];
+  /** Scope values for subscriptions to the main server */
+  scopes: ScopeValues;
+  /** Shape registry for handling operations */
+  shapes: TableRegistry<DB>;
+  /** Optional: WebSocket heartbeat interval in milliseconds (default: 30000) */
+  heartbeatIntervalMs?: number;
+  /** Optional: Forward engine retry interval in milliseconds (default: 5000) */
+  forwardRetryIntervalMs?: number;
+  /** Optional: Pull engine interval in milliseconds (default: 10000) */
+  pullIntervalMs?: number;
+  /** Optional: Health check interval in milliseconds (default: 30000) */
+  healthCheckIntervalMs?: number;
+  /** Optional: Prune interval in milliseconds (default: 3600000 = 1 hour). Set to 0 to disable. */
+  pruneIntervalMs?: number;
+  /** Optional: Maximum age of completed relay data before pruning (default: 604800000 = 7 days) */
+  pruneMaxAgeMs?: number;
+}
+
+/**
+ * Result of relay data pruning.
+ */
+export interface PruneRelayResult {
+  deletedMappings: number;
+  deletedOutbox: number;
+  deletedConflicts: number;
+}
+
+type EventHandler<K extends keyof RelayEvents> = RelayEvents[K];
+
+/**
+ * Relay server that acts as an edge relay between local clients and a main server.
+ *
+ * @example
+ * ```typescript
+ * import { createRelayServer } from '@syncular/relay';
+ * import { createSqliteServerDialect } from '@syncular/server-dialect-sqlite';
+ *
+ * const relay = createRelayServer({
+ *   db: sqliteDb,
+ *   dialect: createSqliteServerDialect(),
+ *   mainServerTransport: createHttpTransport({ baseUrl: 'https://main.example.com/sync' }),
+ *   mainServerClientId: 'relay-branch-001',
+ *   mainServerActorId: 'relay-service',
+ *   tables: ['tasks', 'projects'],
+ *   scopes: { project_id: 'acme' },
+ *   shapes: shapeRegistry,
+ * });
+ *
+ * // Mount routes for local clients
+ * app.route('/sync', relay.getRoutes());
+ *
+ * // Start background sync with main
+ * await relay.start();
+ *
+ * // Events
+ * relay.on('modeChange', (mode) => console.log(mode));
+ * relay.on('forwardConflict', (conflict) => handleConflict(conflict));
+ * ```
+ */
+export class RelayServer<DB extends RelayDatabase = RelayDatabase> {
+  private readonly db: Kysely<DB>;
+  private readonly dialect: ServerSyncDialect;
+  private readonly mainServerTransport: SyncTransport;
+  private readonly mainServerClientId: string;
+  private readonly mainServerActorId: string;
+  private readonly tables: string[];
+  private readonly scopes: ScopeValues;
+  private readonly shapes: TableRegistry<DB>;
+
+  private readonly modeManager: ModeManager;
+  private readonly sequenceMapper: SequenceMapper<DB>;
+  private readonly forwardEngine: ForwardEngine<DB>;
+  private readonly pullEngine: PullEngine<DB>;
+  private readonly realtime: RelayRealtime;
+
+  private readonly eventHandlers = new Map<
+    string,
+    Set<(...args: never[]) => unknown>
+  >();
+
+  private readonly pruneIntervalMs: number;
+  private readonly pruneMaxAgeMs: number;
+  private lastPruneAtMs = 0;
+  private pruneInFlight: Promise<PruneRelayResult> | null = null;
+
+  private started = false;
+  private schemaInitialized = false;
+  private routes: Hono | null = null;
+
+  constructor(options: RelayServerOptions<DB>) {
+    this.db = options.db;
+    this.dialect = options.dialect;
+    this.mainServerTransport = options.mainServerTransport;
+    this.mainServerClientId = options.mainServerClientId;
+    this.mainServerActorId = options.mainServerActorId;
+    this.tables = options.tables;
+    this.scopes = options.scopes;
+    this.shapes = options.shapes;
+
+    this.pruneIntervalMs = options.pruneIntervalMs ?? 3600000;
+    this.pruneMaxAgeMs = options.pruneMaxAgeMs ?? 7 * 24 * 60 * 60 * 1000;
+
+    // Initialize mode manager
+    this.modeManager = new ModeManager({
+      healthCheckIntervalMs: options.healthCheckIntervalMs ?? 30000,
+      onModeChange: (mode) => this.emit('modeChange', mode),
+    });
+
+    // Initialize sequence mapper
+    this.sequenceMapper = new SequenceMapper({
+      db: this.db,
+    });
+
+    // Initialize realtime manager for local clients
+    this.realtime = new RelayRealtime({
+      heartbeatIntervalMs: options.heartbeatIntervalMs ?? 30000,
+    });
+
+    // Initialize forward engine (forwards local commits to main)
+    this.forwardEngine = new ForwardEngine({
+      db: this.db,
+      transport: this.mainServerTransport,
+      clientId: this.mainServerClientId,
+      sequenceMapper: this.sequenceMapper,
+      retryIntervalMs: options.forwardRetryIntervalMs ?? 5000,
+      onConflict: (conflict) => this.emit('forwardConflict', conflict),
+      onError: (error) => this.emit('error', error),
+    });
+
+    // Initialize pull engine (pulls changes from main)
+    this.pullEngine = new PullEngine({
+      db: this.db,
+      dialect: this.dialect,
+      transport: this.mainServerTransport,
+      clientId: this.mainServerClientId,
+      tables: this.tables,
+      scopes: this.scopes,
+      shapes: this.shapes,
+      sequenceMapper: this.sequenceMapper,
+      realtime: this.realtime,
+      intervalMs: options.pullIntervalMs ?? 10000,
+      onError: (error) => this.emit('error', error),
+      onPullComplete: async () => {
+        await this.maybePruneRelay();
+      },
+    });
+  }
+
+  /**
+   * Subscribe to relay events.
+   */
+  on<K extends keyof RelayEvents>(
+    event: K,
+    handler: EventHandler<K>
+  ): () => void {
+    let handlers = this.eventHandlers.get(event);
+    if (!handlers) {
+      handlers = new Set();
+      this.eventHandlers.set(event, handlers);
+    }
+    handlers.add(handler);
+
+    return () => {
+      handlers?.delete(handler);
+    };
+  }
+
+  /**
+   * Get the current mode (online/offline/reconnecting).
+   */
+  getMode(): RelayMode {
+    return this.modeManager.getMode();
+  }
+
+  /**
+   * Get the tables this relay subscribes to.
+   */
+  getTables(): readonly string[] {
+    return this.tables;
+  }
+
+  /**
+   * Get the scope values for subscriptions.
+   */
+  getScopes(): ScopeValues {
+    return this.scopes;
+  }
+
+  /**
+   * Get the realtime manager for WebSocket connections.
+   */
+  getRealtime(): RelayRealtime {
+    return this.realtime;
+  }
+
+  /**
+   * Start the relay server background processes.
+   *
+   * This initializes the database schema and starts:
+   * - Forward engine (sends local commits to main)
+   * - Pull engine (receives changes from main)
+   * - Mode manager (tracks online/offline state)
+   */
+  async start(): Promise<void> {
+    if (this.started) return;
+
+    // Initialize schema if needed
+    if (!this.schemaInitialized) {
+      await ensureRelaySchema(this.db, this.dialect);
+      this.schemaInitialized = true;
+    }
+
+    this.started = true;
+
+    // Start background processes
+    this.modeManager.start(async () => {
+      // Health check: try an empty pull
+      try {
+        await this.mainServerTransport.sync({
+          clientId: this.mainServerClientId,
+          pull: {
+            subscriptions: [],
+            limitCommits: 0,
+          },
+        });
+        return true;
+      } catch {
+        return false;
+      }
+    });
+
+    this.forwardEngine.start();
+    this.pullEngine.start();
+  }
+
+  /**
+   * Stop the relay server background processes.
+   */
+  async stop(): Promise<void> {
+    if (!this.started) return;
+
+    this.started = false;
+    this.modeManager.stop();
+    this.forwardEngine.stop();
+    this.pullEngine.stop();
+    this.realtime.closeAll();
+  }
+
+  /**
+   * Manually trigger a forward cycle (useful for testing).
+   */
+  async forwardOnce(): Promise<boolean> {
+    return this.forwardEngine.forwardOnce();
+  }
+
+  /**
+   * Manually trigger a pull cycle (useful for testing).
+   */
+  async pullOnce(): Promise<boolean> {
+    return this.pullEngine.pullOnce();
+  }
+
+  /**
+   * Prune old relay data (sequence mappings, forwarded outbox, resolved conflicts).
+   */
+  async pruneRelay(options?: { maxAgeMs?: number }): Promise<PruneRelayResult> {
+    const maxAgeMs = options?.maxAgeMs ?? this.pruneMaxAgeMs;
+    const threshold = Date.now() - maxAgeMs;
+
+    const deletedMappings =
+      await this.sequenceMapper.pruneOldMappings(maxAgeMs);
+
+    const outboxResult = await sql`
+      delete from ${sql.table('relay_forward_outbox')}
+      where status in ('forwarded', 'failed')
+      and updated_at < ${threshold}
+    `.execute(this.db);
+    const deletedOutbox = Number(outboxResult.numAffectedRows ?? 0);
+
+    const conflictsResult = await sql`
+      delete from ${sql.table('relay_forward_conflicts')}
+      where resolved_at is not null
+      and resolved_at < ${threshold}
+    `.execute(this.db);
+    const deletedConflicts = Number(conflictsResult.numAffectedRows ?? 0);
+
+    return { deletedMappings, deletedOutbox, deletedConflicts };
+  }
+
+  /**
+   * Rate-limited pruning. Skips if called within `pruneIntervalMs` of last prune.
+   * Returns zero counts if skipped or if pruning is disabled.
+   */
+  async maybePruneRelay(): Promise<PruneRelayResult> {
+    if (this.pruneIntervalMs <= 0) {
+      return { deletedMappings: 0, deletedOutbox: 0, deletedConflicts: 0 };
+    }
+
+    const now = Date.now();
+    if (now - this.lastPruneAtMs < this.pruneIntervalMs) {
+      return { deletedMappings: 0, deletedOutbox: 0, deletedConflicts: 0 };
+    }
+
+    if (this.pruneInFlight) return this.pruneInFlight;
+
+    this.pruneInFlight = (async () => {
+      try {
+        const result = await this.pruneRelay();
+        this.lastPruneAtMs = Date.now();
+        return result;
+      } finally {
+        this.pruneInFlight = null;
+      }
+    })();
+
+    return this.pruneInFlight;
+  }
+
+  /**
+   * Get Hono routes for local clients.
+   *
+   * Mount these routes to serve local sync clients:
+   * - POST /pull
+   * - POST /push
+   * - GET /realtime (WebSocket)
+   */
+  getRoutes(): Hono {
+    if (this.routes) return this.routes;
+
+    // Lazy import to avoid requiring hono as a hard dependency
+    const { createRelayRoutes } = require('./server-role');
+
+    const routes = createRelayRoutes({
+      db: this.db,
+      dialect: this.dialect,
+      shapes: this.shapes,
+      realtime: this.realtime,
+      onCommit: async (localCommitSeq: number, affectedTables: string[]) => {
+        // Notify local clients via WebSocket
+        this.realtime.notifyScopeKeys(affectedTables, localCommitSeq);
+
+        // Wake up forward engine to send to main
+        this.forwardEngine.wakeUp();
+      },
+    });
+
+    this.routes = routes;
+    return routes;
+  }
+
+  private emit<K extends keyof RelayEvents>(
+    event: K,
+    ...args: Parameters<RelayEvents[K]>
+  ): void {
+    const handlers = this.eventHandlers.get(event);
+    if (!handlers) return;
+
+    for (const handler of handlers) {
+      try {
+        (handler as (...args: unknown[]) => unknown)(...args);
+      } catch (err) {
+        console.error(`Error in ${event} handler:`, err);
+      }
+    }
+  }
+}
+
+/**
+ * Create a new relay server instance.
+ */
+export function createRelayServer(options: RelayServerOptions): RelayServer {
+  return new RelayServer(options);
+}

package/src/schema.ts

@@ -0,0 +1,171 @@
+/**
+ * @syncular/relay - Database schema types
+ *
+ * Relay-specific tables for edge relay servers.
+ */
+
+import type { SyncCoreDb } from '@syncular/server';
+import type { Generated } from 'kysely';
+
+/**
+ * Forward outbox status for commits awaiting forwarding to main server.
+ */
+export type RelayForwardOutboxStatus =
+  | 'pending'
+  | 'forwarding'
+  | 'forwarded'
+  | 'failed';
+
+/**
+ * Sequence map status for tracking local to main commit mapping.
+ */
+export type RelaySequenceMapStatus = 'pending' | 'forwarded' | 'confirmed';
+
+/**
+ * Forward outbox - Queue for commits to forward to main server.
+ *
+ * When local clients push to the relay, commits are stored here
+ * for subsequent forwarding to the main server.
+ */
+interface RelayForwardOutboxTable {
+  /** Unique identifier for this outbox entry */
+  id: string;
+  /** Local commit sequence assigned by the relay */
+  local_commit_seq: number;
+  /** Original client_id from the local client */
+  client_id: string;
+  /** Original client_commit_id from the local client */
+  client_commit_id: string;
+  /** Operations JSON for forwarding */
+  operations_json: string;
+  /** Client schema version when commit was created */
+  schema_version: number;
+  /** Current status of this entry */
+  status: RelayForwardOutboxStatus;
+  /** Main server's commit_seq after forwarding (null until confirmed) */
+  main_commit_seq: number | null;
+  /** Error message if status is 'failed' */
+  error: string | null;
+  /** Last response JSON from main server */
+  last_response_json: string | null;
+  /** Creation timestamp */
+  created_at: Generated<number>;
+  /** Last update timestamp */
+  updated_at: Generated<number>;
+  /** Number of forward attempts */
+  attempt_count: Generated<number>;
+}
+
+/**
+ * Sequence map - Maps local to main commit sequences.
+ *
+ * Tracks the relationship between relay's local commit_seq
+ * and the main server's global commit_seq.
+ */
+interface RelaySequenceMapTable {
+  /** Relay's local commit sequence */
+  local_commit_seq: number;
+  /** Main server's commit sequence (null if not yet forwarded) */
+  main_commit_seq: number | null;
+  /** Mapping status */
+  status: RelaySequenceMapStatus;
+  /** When this mapping was created */
+  created_at: Generated<number>;
+  /** When this mapping was last updated */
+  updated_at: Generated<number>;
+}
+
+/**
+ * Forward conflicts - Conflicts encountered when forwarding to main.
+ *
+ * When a local commit is rejected by the main server due to conflicts,
+ * the details are stored here for application-level handling.
+ */
+interface RelayForwardConflictTable {
+  /** Unique identifier */
+  id: string;
+  /** Local commit sequence that caused the conflict */
+  local_commit_seq: number;
+  /** Original client_id from the local client */
+  client_id: string;
+  /** Original client_commit_id from the local client */
+  client_commit_id: string;
+  /** Full rejection response from main server */
+  response_json: string;
+  /** When the conflict was recorded */
+  created_at: number;
+  /** When the conflict was resolved (null if unresolved) */
+  resolved_at: number | null;
+}
+
+/**
+ * Relay config - Key-value store for relay state.
+ *
+ * Stores configuration and runtime state like:
+ * - scope_keys: subscribed scopes
+ * - main_cursor: last pulled commit_seq from main server
+ * - mode: online/offline/reconnecting
+ */
+interface RelayConfigTable {
+  /** Configuration key */
+  key: string;
+  /** JSON-encoded value */
+  value_json: string;
+}
+
+/**
+ * Database interface for relay-specific tables.
+ * Merge this with SyncCoreDb for the full database interface.
+ */
+export interface RelayDb {
+  relay_forward_outbox: RelayForwardOutboxTable;
+  relay_sequence_map: RelaySequenceMapTable;
+  relay_forward_conflicts: RelayForwardConflictTable;
+  relay_config: RelayConfigTable;
+}
+
+/**
+ * Full database interface required by the relay runtime.
+ *
+ * Includes:
+ * - Sync core tables (commit log, cursors, etc.)
+ * - Relay-specific tables (outbox, sequence map, config, conflicts)
+ */
+export type RelayDatabase = SyncCoreDb & RelayDb;
+
+/**
+ * Forward outbox entry (with parsed operations).
+ */
+export interface ForwardOutboxEntry {
+  id: string;
+  local_commit_seq: number;
+  client_id: string;
+  client_commit_id: string;
+  operations: Array<{
+    table: string;
+    row_id: string;
+    op: 'upsert' | 'delete';
+    payload: Record<string, unknown> | null;
+    base_version?: number | null;
+  }>;
+  schema_version: number;
+  status: RelayForwardOutboxStatus;
+  main_commit_seq: number | null;
+  error: string | null;
+  created_at: number;
+  updated_at: number;
+  attempt_count: number;
+}
+
+/**
+ * Forward conflict entry (with parsed response).
+ */
+export interface ForwardConflictEntry {
+  id: string;
+  local_commit_seq: number;
+  client_id: string;
+  client_commit_id: string;
+  response: unknown;
+  created_at: number;
+  resolved_at: number | null;
+}