@syncular/relay 0.0.1-60

package/src/client-role/forward-engine.ts

@@ -0,0 +1,352 @@
+/**
+ * @syncular/relay - Forward Engine
+ *
+ * Forwards commits from the relay's local outbox to the main server.
+ * Preserves original client_id + client_commit_id for idempotency.
+ */
+
+import type {
+  SyncOperation,
+  SyncPushResponse,
+  SyncTransport,
+} from '@syncular/core';
+import type { Kysely } from 'kysely';
+import { sql } from 'kysely';
+import type {
+  ForwardConflictEntry,
+  ForwardOutboxEntry,
+  RelayDatabase,
+  RelayForwardOutboxStatus,
+} from '../schema';
+import type { SequenceMapper } from './sequence-mapper';
+
+function randomId(): string {
+  if (
+    typeof crypto !== 'undefined' &&
+    typeof crypto.randomUUID === 'function'
+  ) {
+    return crypto.randomUUID();
+  }
+  return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
+}
+
+/**
+ * Forward engine options.
+ */
+export interface ForwardEngineOptions<
+  DB extends RelayDatabase = RelayDatabase,
+> {
+  db: Kysely<DB>;
+  transport: SyncTransport;
+  clientId: string;
+  sequenceMapper: SequenceMapper<DB>;
+  retryIntervalMs?: number;
+  onConflict?: (conflict: ForwardConflictEntry) => void;
+  onError?: (error: Error) => void;
+}
+
+/**
+ * Forward engine for sending local commits to the main server.
+ */
+export class ForwardEngine<DB extends RelayDatabase = RelayDatabase> {
+  private readonly db: Kysely<DB>;
+  private readonly transport: SyncTransport;
+  private readonly clientId: string;
+  private readonly sequenceMapper: SequenceMapper<DB>;
+  private readonly retryIntervalMs: number;
+  private readonly onConflict?: (conflict: ForwardConflictEntry) => void;
+  private readonly onError?: (error: Error) => void;
+
+  private running = false;
+  private timer: ReturnType<typeof setTimeout> | null = null;
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: assigned in scheduleWakeUp and read in tick
+  private wakeUpRequested = false;
+
+  constructor(options: ForwardEngineOptions<DB>) {
+    this.db = options.db;
+    this.transport = options.transport;
+    this.clientId = options.clientId;
+    this.sequenceMapper = options.sequenceMapper;
+    this.retryIntervalMs = options.retryIntervalMs ?? 5000;
+    this.onConflict = options.onConflict;
+    this.onError = options.onError;
+  }
+
+  /**
+   * Start the forward engine loop.
+   */
+  start(): void {
+    if (this.running) return;
+    this.running = true;
+    this.scheduleNext(0);
+  }
+
+  /**
+   * Stop the forward engine.
+   */
+  stop(): void {
+    this.running = false;
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+  }
+
+  /**
+   * Wake up the engine to process immediately.
+   */
+  wakeUp(): void {
+    if (!this.running) return;
+    this.wakeUpRequested = true;
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+    this.scheduleNext(0);
+  }
+
+  /**
+   * Forward a single commit (for manual/testing use).
+   */
+  async forwardOnce(): Promise<boolean> {
+    return this.processOne();
+  }
+
+  private scheduleNext(delayMs: number): void {
+    if (!this.running) return;
+    if (this.timer) return;
+
+    this.timer = setTimeout(async () => {
+      this.timer = null;
+      this.wakeUpRequested = false;
+
+      try {
+        const forwarded = await this.processOne();
+        // If we forwarded something, immediately try again
+        const nextDelay = forwarded ? 0 : this.retryIntervalMs;
+        this.scheduleNext(nextDelay);
+      } catch (err) {
+        this.onError?.(err instanceof Error ? err : new Error(String(err)));
+        this.scheduleNext(this.retryIntervalMs);
+      }
+    }, delayMs);
+  }
+
+  private async processOne(): Promise<boolean> {
+    const next = await this.getNextSendable();
+    if (!next) return false;
+
+    await this.markSending(next.id);
+
+    let response: SyncPushResponse;
+    try {
+      const combined = await this.transport.sync({
+        clientId: next.client_id,
+        push: {
+          clientCommitId: next.client_commit_id,
+          operations: next.operations,
+          schemaVersion: next.schema_version,
+        },
+      });
+      if (!combined.push) {
+        throw new Error('Server returned no push response');
+      }
+      response = combined.push;
+    } catch (err) {
+      // Network error - mark as pending for retry
+      await this.markPending(next.id, String(err));
+      throw err;
+    }
+
+    const responseJson = JSON.stringify(response);
+
+    if (response.status === 'applied' || response.status === 'cached') {
+      const mainCommitSeq = response.commitSeq ?? null;
+
+      // Update outbox entry
+      await this.markForwarded(next.id, mainCommitSeq, responseJson);
+
+      // Update sequence mapper
+      if (mainCommitSeq != null) {
+        await this.sequenceMapper.markForwarded(
+          next.local_commit_seq,
+          mainCommitSeq
+        );
+      }
+
+      return true;
+    }
+
+    // Rejected - store conflict and mark as failed
+    const conflict = await this.recordConflict(next, responseJson);
+    await this.markFailed(next.id, 'REJECTED', responseJson);
+
+    this.onConflict?.(conflict);
+
+    return true;
+  }
+
+  private async getNextSendable(): Promise<ForwardOutboxEntry | null> {
+    const staleThreshold = Date.now() - 30000;
+
+    const rowResult = await sql<{
+      id: string;
+      local_commit_seq: number;
+      client_id: string;
+      client_commit_id: string;
+      operations_json: string;
+      schema_version: number;
+      status: RelayForwardOutboxStatus;
+      main_commit_seq: number | null;
+      error: string | null;
+      last_response_json: string | null;
+      created_at: number;
+      updated_at: number;
+      attempt_count: number;
+    }>`
+      select
+        id,
+        local_commit_seq,
+        client_id,
+        client_commit_id,
+        operations_json,
+        schema_version,
+        status,
+        main_commit_seq,
+        error,
+        last_response_json,
+        created_at,
+        updated_at,
+        attempt_count
+      from ${sql.table('relay_forward_outbox')}
+      where
+        status = 'pending'
+        or (status = 'forwarding' and updated_at < ${staleThreshold})
+      order by created_at asc
+      limit 1
+    `.execute(this.db);
+    const row = rowResult.rows[0];
+
+    if (!row) return null;
+
+    const operations =
+      typeof row.operations_json === 'string'
+        ? (JSON.parse(row.operations_json) as SyncOperation[])
+        : (row.operations_json as SyncOperation[]);
+
+    return {
+      id: row.id,
+      local_commit_seq: row.local_commit_seq,
+      client_id: row.client_id,
+      client_commit_id: row.client_commit_id,
+      operations,
+      schema_version: row.schema_version,
+      status: row.status,
+      main_commit_seq: row.main_commit_seq,
+      error: row.error,
+      created_at: row.created_at,
+      updated_at: row.updated_at,
+      attempt_count: row.attempt_count,
+    };
+  }
+
+  private async markSending(id: string): Promise<void> {
+    const now = Date.now();
+
+    await sql`
+      update ${sql.table('relay_forward_outbox')}
+      set
+        status = 'forwarding',
+        updated_at = ${now},
+        attempt_count = attempt_count + 1,
+        error = ${null}
+      where id = ${id}
+    `.execute(this.db);
+  }
+
+  private async markPending(id: string, error: string): Promise<void> {
+    const now = Date.now();
+
+    await sql`
+      update ${sql.table('relay_forward_outbox')}
+      set status = 'pending', updated_at = ${now}, error = ${error}
+      where id = ${id}
+    `.execute(this.db);
+  }
+
+  private async markForwarded(
+    id: string,
+    mainCommitSeq: number | null,
+    responseJson: string
+  ): Promise<void> {
+    const now = Date.now();
+
+    await sql`
+      update ${sql.table('relay_forward_outbox')}
+      set
+        status = 'forwarded',
+        main_commit_seq = ${mainCommitSeq},
+        updated_at = ${now},
+        error = ${null},
+        last_response_json = ${responseJson}
+      where id = ${id}
+    `.execute(this.db);
+  }
+
+  private async markFailed(
+    id: string,
+    error: string,
+    responseJson: string
+  ): Promise<void> {
+    const now = Date.now();
+
+    await sql`
+      update ${sql.table('relay_forward_outbox')}
+      set
+        status = 'failed',
+        updated_at = ${now},
+        error = ${error},
+        last_response_json = ${responseJson}
+      where id = ${id}
+    `.execute(this.db);
+  }
+
+  private async recordConflict(
+    entry: ForwardOutboxEntry,
+    responseJson: string
+  ): Promise<ForwardConflictEntry> {
+    const now = Date.now();
+    const id = randomId();
+
+    await sql`
+      insert into ${sql.table('relay_forward_conflicts')} (
+        id,
+        local_commit_seq,
+        client_id,
+        client_commit_id,
+        response_json,
+        created_at,
+        resolved_at
+      )
+      values (
+        ${id},
+        ${entry.local_commit_seq},
+        ${entry.client_id},
+        ${entry.client_commit_id},
+        ${responseJson},
+        ${now},
+        ${null}
+      )
+    `.execute(this.db);
+
+    return {
+      id,
+      local_commit_seq: entry.local_commit_seq,
+      client_id: entry.client_id,
+      client_commit_id: entry.client_commit_id,
+      response: JSON.parse(responseJson),
+      created_at: now,
+      resolved_at: null,
+    };
+  }
+}

package/src/client-role/index.ts

@@ -0,0 +1,9 @@
+/**
+ * @syncular/relay - Client Role
+ *
+ * Components for the relay's role as a client to the main server.
+ */
+
+export * from './forward-engine';
+export * from './pull-engine';
+export * from './sequence-mapper';

package/src/client-role/pull-engine.ts

@@ -0,0 +1,301 @@
+/**
+ * @syncular/relay - Pull Engine
+ *
+ * Pulls changes from the main server and stores them locally
+ * on the relay for local clients to access.
+ */
+
+import type {
+  ScopeValues,
+  SyncCommit,
+  SyncPullResponse,
+  SyncSubscriptionRequest,
+  SyncTransport,
+} from '@syncular/core';
+import type { ServerSyncDialect, TableRegistry } from '@syncular/server';
+import { pushCommit } from '@syncular/server';
+import { type Kysely, sql } from 'kysely';
+import type { RelayRealtime } from '../realtime';
+import type { RelayDatabase } from '../schema';
+import type { SequenceMapper } from './sequence-mapper';
+
+/**
+ * Pull engine options.
+ */
+export interface PullEngineOptions<DB extends RelayDatabase = RelayDatabase> {
+  db: Kysely<DB>;
+  dialect: ServerSyncDialect;
+  transport: SyncTransport;
+  clientId: string;
+  /** Tables to subscribe to */
+  tables: string[];
+  /** Scope values for subscriptions */
+  scopes: ScopeValues;
+  shapes: TableRegistry<DB>;
+  sequenceMapper: SequenceMapper<DB>;
+  realtime: RelayRealtime;
+  intervalMs?: number;
+  onError?: (error: Error) => void;
+  onPullComplete?: () => Promise<void>;
+}
+
+/**
+ * Pull engine for receiving changes from the main server.
+ */
+export class PullEngine<DB extends RelayDatabase = RelayDatabase> {
+  private readonly db: Kysely<DB>;
+  private readonly dialect: ServerSyncDialect;
+  private readonly transport: SyncTransport;
+  private readonly clientId: string;
+  private readonly tables: string[];
+  private readonly scopes: ScopeValues;
+  private readonly shapes: TableRegistry<DB>;
+  private readonly sequenceMapper: SequenceMapper<DB>;
+  private readonly realtime: RelayRealtime;
+  private readonly intervalMs: number;
+  private readonly onError?: (error: Error) => void;
+  private readonly onPullComplete?: () => Promise<void>;
+
+  private running = false;
+  private timer: ReturnType<typeof setTimeout> | null = null;
+  private cursors = new Map<string, number>();
+
+  constructor(options: PullEngineOptions<DB>) {
+    this.db = options.db;
+    this.dialect = options.dialect;
+    this.transport = options.transport;
+    this.clientId = options.clientId;
+    this.tables = options.tables;
+    this.scopes = options.scopes;
+    this.shapes = options.shapes;
+    this.sequenceMapper = options.sequenceMapper;
+    this.realtime = options.realtime;
+    this.intervalMs = options.intervalMs ?? 10000;
+    this.onError = options.onError;
+    this.onPullComplete = options.onPullComplete;
+  }
+
+  /**
+   * Start the pull engine loop.
+   */
+  start(): void {
+    if (this.running) return;
+    this.running = true;
+    this.loadCursors().then(() => {
+      this.scheduleNext(0);
+    });
+  }
+
+  /**
+   * Stop the pull engine.
+   */
+  stop(): void {
+    this.running = false;
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+  }
+
+  /**
+   * Pull once (for manual/testing use).
+   */
+  async pullOnce(): Promise<boolean> {
+    return this.processOne();
+  }
+
+  private async loadCursors(): Promise<void> {
+    try {
+      // Load cursors from config
+      const rowResult = await sql<{ value_json: string }>`
+        select value_json
+        from ${sql.table('relay_config')}
+        where key = 'main_cursors'
+        limit 1
+      `.execute(this.db);
+      const row = rowResult.rows[0];
+
+      if (row?.value_json) {
+        const parsed = JSON.parse(row.value_json);
+        if (typeof parsed === 'object' && parsed !== null) {
+          for (const [key, value] of Object.entries(parsed)) {
+            if (typeof value === 'number') {
+              this.cursors.set(key, value);
+            }
+          }
+        }
+      }
+    } catch {
+      // Ignore - start from scratch
+    }
+  }
+
+  private async saveCursors(): Promise<void> {
+    const cursorObj: Record<string, number> = {};
+    for (const [key, value] of this.cursors) {
+      cursorObj[key] = value;
+    }
+
+    const valueJson = JSON.stringify(cursorObj);
+    await sql`
+      insert into ${sql.table('relay_config')} (key, value_json)
+      values ('main_cursors', ${valueJson})
+      on conflict (key)
+      do update set value_json = ${valueJson}
+    `.execute(this.db);
+  }
+
+  private scheduleNext(delayMs: number): void {
+    if (!this.running) return;
+    if (this.timer) return;
+
+    this.timer = setTimeout(async () => {
+      this.timer = null;
+
+      try {
+        const pulled = await this.processOne();
+        // If we pulled something, immediately try again
+        const nextDelay = pulled ? 0 : this.intervalMs;
+        this.scheduleNext(nextDelay);
+      } catch (err) {
+        this.onError?.(err instanceof Error ? err : new Error(String(err)));
+        this.scheduleNext(this.intervalMs);
+      }
+    }, delayMs);
+  }
+
+  private async processOne(): Promise<boolean> {
+    // Build subscriptions for each table
+    const subscriptionRequests: SyncSubscriptionRequest[] = this.tables.map(
+      (table) => ({
+        id: table,
+        shape: table,
+        scopes: this.scopes,
+        cursor: this.cursors.get(table) ?? -1,
+      })
+    );
+
+    let response: SyncPullResponse;
+    try {
+      const combined = await this.transport.sync({
+        clientId: this.clientId,
+        pull: {
+          subscriptions: subscriptionRequests,
+          limitCommits: 100,
+        },
+      });
+      if (!combined.pull) {
+        return false;
+      }
+      response = combined.pull;
+    } catch {
+      // Network error - will retry
+      return false;
+    }
+
+    if (!response.ok) {
+      return false;
+    }
+
+    let hasChanges = false;
+    const affectedTables = new Set<string>();
+
+    for (const sub of response.subscriptions) {
+      if (sub.status !== 'active') continue;
+
+      const table = sub.id;
+
+      // Process commits
+      for (const commit of sub.commits) {
+        const applied = await this.applyCommitLocally(commit, table);
+        if (applied) {
+          hasChanges = true;
+          affectedTables.add(table);
+        }
+      }
+
+      // Update cursor
+      if (sub.nextCursor > (this.cursors.get(table) ?? -1)) {
+        this.cursors.set(table, sub.nextCursor);
+      }
+    }
+
+    // Save updated cursors
+    await this.saveCursors();
+
+    // Notify local clients if we have changes
+    if (hasChanges && affectedTables.size > 0) {
+      const maxCursor = await this.dialect.readMaxCommitSeq(this.db);
+      this.realtime.notifyScopeKeys(Array.from(affectedTables), maxCursor);
+    }
+
+    // Trigger rate-limited prune after successful pull
+    await this.onPullComplete?.();
+
+    return hasChanges;
+  }
+
+  /**
+   * Apply a commit from main server locally.
+   *
+   * This re-applies the commit through the local shape handlers
+   * to ensure proper indexing and scope assignment.
+   */
+  private async applyCommitLocally(
+    commit: SyncCommit,
+    table: string
+  ): Promise<boolean> {
+    if (commit.changes.length === 0) return false;
+
+    // Convert changes to operations
+    const operations = commit.changes.map((change) => ({
+      table: change.table,
+      row_id: change.row_id,
+      op: change.op,
+      payload: change.row_json as Record<string, unknown> | null,
+    }));
+
+    // Generate a unique commit ID for this relay instance
+    const relayCommitId = `main:${commit.commitSeq}:${table}`;
+
+    // Push through local handler
+    const result = await pushCommit({
+      db: this.db,
+      dialect: this.dialect,
+      shapes: this.shapes,
+      actorId: commit.actorId,
+      request: {
+        clientId: `relay:${this.clientId}`,
+        clientCommitId: relayCommitId,
+        operations,
+        schemaVersion: 1,
+      },
+    });
+
+    if (
+      result.response.ok === true &&
+      result.response.status === 'applied' &&
+      typeof result.response.commitSeq === 'number'
+    ) {
+      // Record sequence mapping
+      await this.sequenceMapper.createConfirmedMapping(
+        result.response.commitSeq,
+        commit.commitSeq
+      );
+      return true;
+    }
+
+    // Already applied (cached) - that's fine
+    if (result.response.status === 'cached') {
+      return false;
+    }
+
+    // Rejected - this shouldn't happen for pulls from main
+    // Log but don't fail
+    console.warn(
+      `Relay: Failed to apply commit ${commit.commitSeq} locally:`,
+      result.response
+    );
+    return false;
+  }
+}