@syncular/server 0.0.1-100

package/src/push.ts

@@ -0,0 +1,583 @@
+import {
+  captureSyncException,
+  countSyncMetric,
+  distributionSyncMetric,
+  type SyncChange,
+  type SyncPushRequest,
+  type SyncPushResponse,
+  startSyncSpan,
+} from '@syncular/core';
+import type {
+  Insertable,
+  Kysely,
+  SelectQueryBuilder,
+  SqlBool,
+  Updateable,
+} from 'kysely';
+import { sql } from 'kysely';
+import type { ServerSyncDialect } from './dialect/types';
+import type { SyncCoreDb } from './schema';
+import type { TableRegistry } from './shapes/registry';
+
+// biome-ignore lint/complexity/noBannedTypes: Kysely uses `{}` as the initial "no selected columns yet" marker.
+type EmptySelection = {};
+
+export interface PushCommitResult {
+  response: SyncPushResponse;
+  /**
+   * Distinct tables affected by this commit.
+   * Empty for rejected commits and for commits that emit no changes.
+   */
+  affectedTables: string[];
+  /**
+   * Scope keys derived from emitted changes (e.g. "org:abc", "team:xyz").
+   * Computed in-transaction so callers don't need an extra DB query.
+   * Empty for rejected/cached commits.
+   */
+  scopeKeys: string[];
+  /**
+   * Changes emitted by this commit. Available for WS data delivery.
+   * Empty for rejected/cached commits.
+   */
+  emittedChanges: SyncChange[];
+}
+
+class RejectCommitError extends Error {
+  constructor(public readonly response: SyncPushResponse) {
+    super('REJECT_COMMIT');
+    this.name = 'RejectCommitError';
+  }
+}
+
+async function readCommitAffectedTables<DB extends SyncCoreDb>(
+  db: Kysely<DB>,
+  dialect: ServerSyncDialect,
+  commitSeq: number,
+  partitionId: string
+): Promise<string[]> {
+  try {
+    const commitsQ = db.selectFrom('sync_commits') as SelectQueryBuilder<
+      DB,
+      'sync_commits',
+      EmptySelection
+    >;
+
+    const row = await commitsQ
+      .selectAll()
+      .where(sql<SqlBool>`commit_seq = ${commitSeq}`)
+      .where(sql<SqlBool>`partition_id = ${partitionId}`)
+      .executeTakeFirst();
+
+    const raw = row?.affected_tables;
+    return dialect.dbToArray(raw);
+  } catch {
+    // ignore and fall back to scanning changes (best-effort)
+  }
+
+  // Fallback: read from changes using dialect-specific implementation
+  return dialect.readAffectedTablesFromChanges(db, commitSeq, { partitionId });
+}
+
+function scopeKeysFromEmitted(
+  emitted: Array<{ scopes: Record<string, string> }>
+): string[] {
+  const keys = new Set<string>();
+  for (const c of emitted) {
+    for (const [key, value] of Object.entries(c.scopes)) {
+      if (!value) continue;
+      const prefix = key.replace(/_id$/, '');
+      keys.add(`${prefix}:${value}`);
+    }
+  }
+  return Array.from(keys);
+}
+
+function recordPushMetrics(args: {
+  status: string;
+  durationMs: number;
+  operationCount: number;
+  emittedChangeCount: number;
+  affectedTableCount: number;
+}): void {
+  const {
+    status,
+    durationMs,
+    operationCount,
+    emittedChangeCount,
+    affectedTableCount,
+  } = args;
+
+  countSyncMetric('sync.server.push.requests', 1, {
+    attributes: { status },
+  });
+  countSyncMetric('sync.server.push.operations', operationCount, {
+    attributes: { status },
+  });
+  distributionSyncMetric('sync.server.push.duration_ms', durationMs, {
+    unit: 'millisecond',
+    attributes: { status },
+  });
+  distributionSyncMetric(
+    'sync.server.push.emitted_changes',
+    emittedChangeCount,
+    {
+      attributes: { status },
+    }
+  );
+  distributionSyncMetric(
+    'sync.server.push.affected_tables',
+    affectedTableCount,
+    {
+      attributes: { status },
+    }
+  );
+}
+
+export async function pushCommit<DB extends SyncCoreDb>(args: {
+  db: Kysely<DB>;
+  dialect: ServerSyncDialect;
+  shapes: TableRegistry<DB>;
+  actorId: string;
+  partitionId?: string;
+  request: SyncPushRequest;
+}): Promise<PushCommitResult> {
+  const { request, dialect } = args;
+  const db = args.db;
+  const partitionId = args.partitionId ?? 'default';
+  const requestedOps = Array.isArray(request.operations)
+    ? request.operations
+    : [];
+  const operationCount = requestedOps.length;
+  const startedAtMs = Date.now();
+
+  return startSyncSpan(
+    {
+      name: 'sync.server.push',
+      op: 'sync.push',
+      attributes: {
+        operation_count: operationCount,
+      },
+    },
+    async (span) => {
+      const finalizeResult = (result: PushCommitResult): PushCommitResult => {
+        const durationMs = Math.max(0, Date.now() - startedAtMs);
+        const status = result.response.status;
+
+        span.setAttribute('status', status);
+        span.setAttribute('duration_ms', durationMs);
+        span.setAttribute('emitted_change_count', result.emittedChanges.length);
+        span.setAttribute('affected_table_count', result.affectedTables.length);
+        span.setStatus('ok');
+
+        recordPushMetrics({
+          status,
+          durationMs,
+          operationCount,
+          emittedChangeCount: result.emittedChanges.length,
+          affectedTableCount: result.affectedTables.length,
+        });
+
+        return result;
+      };
+
+      try {
+        if (!request.clientId || !request.clientCommitId) {
+          return finalizeResult({
+            response: {
+              ok: true,
+              status: 'rejected',
+              results: [
+                {
+                  opIndex: 0,
+                  status: 'error',
+                  error: 'INVALID_REQUEST',
+                  code: 'INVALID_REQUEST',
+                  retriable: false,
+                },
+              ],
+            },
+            affectedTables: [],
+            scopeKeys: [],
+            emittedChanges: [],
+          });
+        }
+
+        const ops = request.operations ?? [];
+        if (!Array.isArray(ops) || ops.length === 0) {
+          return finalizeResult({
+            response: {
+              ok: true,
+              status: 'rejected',
+              results: [
+                {
+                  opIndex: 0,
+                  status: 'error',
+                  error: 'EMPTY_COMMIT',
+                  code: 'EMPTY_COMMIT',
+                  retriable: false,
+                },
+              ],
+            },
+            affectedTables: [],
+            scopeKeys: [],
+            emittedChanges: [],
+          });
+        }
+
+        return finalizeResult(
+          await dialect.executeInTransaction(db, async (trx) => {
+            type SyncTrx = Pick<
+              Kysely<SyncCoreDb>,
+              'selectFrom' | 'insertInto' | 'updateTable' | 'deleteFrom'
+            >;
+
+            const syncTrx = trx as SyncTrx;
+
+            // Clean up any stale commit row with null result_json.
+            // This can happen when a previous push inserted the commit row but crashed
+            // before writing the result (e.g., on D1 without transaction support).
+            await syncTrx
+              .deleteFrom('sync_commits')
+              .where('partition_id', '=', partitionId)
+              .where('client_id', '=', request.clientId)
+              .where('client_commit_id', '=', request.clientCommitId)
+              .where('result_json', 'is', null)
+              .execute();
+
+            // Insert commit row (idempotency key)
+            const commitRow: Insertable<SyncCoreDb['sync_commits']> = {
+              partition_id: partitionId,
+              actor_id: args.actorId,
+              client_id: request.clientId,
+              client_commit_id: request.clientCommitId,
+              meta: null,
+              result_json: null,
+            };
+
+            const insertResult = await syncTrx
+              .insertInto('sync_commits')
+              .values(commitRow)
+              .onConflict((oc) =>
+                oc
+                  .columns(['partition_id', 'client_id', 'client_commit_id'])
+                  .doNothing()
+              )
+              .executeTakeFirstOrThrow();
+
+            const insertedRows = Number(
+              insertResult.numInsertedOrUpdatedRows ?? 0
+            );
+            if (insertedRows === 0) {
+              // Existing commit: return cached response (applied or rejected)
+              // Use forUpdate() for row locking on databases that support it
+              let query = (
+                syncTrx.selectFrom('sync_commits') as SelectQueryBuilder<
+                  SyncCoreDb,
+                  'sync_commits',
+                  EmptySelection
+                >
+              )
+                .selectAll()
+                .where('partition_id', '=', partitionId)
+                .where('client_id', '=', request.clientId)
+                .where('client_commit_id', '=', request.clientCommitId);
+
+              if (dialect.supportsForUpdate) {
+                query = query.forUpdate();
+              }
+
+              const existing = await query.executeTakeFirstOrThrow();
+
+              const cached = existing.result_json as SyncPushResponse | null;
+              if (!cached || cached.ok !== true) {
+                return {
+                  response: {
+                    ok: true,
+                    status: 'rejected',
+                    results: [
+                      {
+                        opIndex: 0,
+                        status: 'error',
+                        error: 'IDEMPOTENCY_CACHE_MISS',
+                        code: 'INTERNAL',
+                        retriable: true,
+                      },
+                    ],
+                  },
+                  affectedTables: [],
+                  scopeKeys: [],
+                  emittedChanges: [],
+                };
+              }
+
+              const base: SyncPushResponse = {
+                ...cached,
+                commitSeq: Number(existing.commit_seq),
+              };
+
+              if (cached.status === 'applied') {
+                const tablesFromDb = dialect.dbToArray(
+                  existing.affected_tables
+                );
+                return {
+                  response: { ...base, status: 'cached' },
+                  affectedTables:
+                    tablesFromDb.length > 0
+                      ? tablesFromDb
+                      : await readCommitAffectedTables(
+                          trx,
+                          dialect,
+                          Number(existing.commit_seq),
+                          partitionId
+                        ),
+                  scopeKeys: [],
+                  emittedChanges: [],
+                };
+              }
+
+              return {
+                response: base,
+                affectedTables: [],
+                scopeKeys: [],
+                emittedChanges: [],
+              };
+            }
+
+            const insertedCommit = await (
+              syncTrx.selectFrom('sync_commits') as SelectQueryBuilder<
+                SyncCoreDb,
+                'sync_commits',
+                EmptySelection
+              >
+            )
+              .selectAll()
+              .where('partition_id', '=', partitionId)
+              .where('client_id', '=', request.clientId)
+              .where('client_commit_id', '=', request.clientCommitId)
+              .executeTakeFirstOrThrow();
+
+            const commitSeq = Number(insertedCommit.commit_seq);
+            const commitId = `${request.clientId}:${request.clientCommitId}`;
+
+            const savepointName = 'sync_apply';
+            const useSavepoints = dialect.supportsSavepoints;
+            let savepointCreated = false;
+
+            try {
+              // Apply the commit under a savepoint so we can roll back app writes on conflict
+              // while still persisting the commit-level cached response.
+              if (useSavepoints) {
+                await sql.raw(`SAVEPOINT ${savepointName}`).execute(trx);
+                savepointCreated = true;
+              }
+
+              const allEmitted = [];
+              const results = [];
+              const affectedTablesSet = new Set<string>();
+
+              for (let i = 0; i < ops.length; i++) {
+                const op = ops[i]!;
+                const handler = args.shapes.getOrThrow(op.table);
+                const applied = await handler.applyOperation(
+                  {
+                    db: trx,
+                    trx,
+                    actorId: args.actorId,
+                    clientId: request.clientId,
+                    commitId,
+                    schemaVersion: request.schemaVersion,
+                  },
+                  op,
+                  i
+                );
+
+                if (applied.result.status !== 'applied') {
+                  results.push(applied.result);
+                  throw new RejectCommitError({
+                    ok: true,
+                    status: 'rejected',
+                    commitSeq,
+                    results,
+                  });
+                }
+
+                // Framework-level enforcement: emitted changes must have scopes
+                for (const c of applied.emittedChanges ?? []) {
+                  const scopes = c?.scopes;
+                  if (!scopes || typeof scopes !== 'object') {
+                    results.push({
+                      opIndex: i,
+                      status: 'error' as const,
+                      error: 'MISSING_SCOPES',
+                      code: 'INVALID_SCOPE',
+                      retriable: false,
+                    });
+                    throw new RejectCommitError({
+                      ok: true,
+                      status: 'rejected',
+                      commitSeq,
+                      results,
+                    });
+                  }
+                }
+
+                results.push(applied.result);
+                allEmitted.push(...applied.emittedChanges);
+                for (const c of applied.emittedChanges) {
+                  affectedTablesSet.add(c.table);
+                }
+              }
+
+              if (allEmitted.length > 0) {
+                const changeRows: Array<
+                  Insertable<SyncCoreDb['sync_changes']>
+                > = allEmitted.map((c) => ({
+                  partition_id: partitionId,
+                  commit_seq: commitSeq,
+                  table: c.table,
+                  row_id: c.row_id,
+                  op: c.op,
+                  row_json: c.row_json,
+                  row_version: c.row_version,
+                  scopes: dialect.scopesToDb(c.scopes),
+                }));
+
+                await syncTrx
+                  .insertInto('sync_changes')
+                  .values(changeRows)
+                  .execute();
+              }
+
+              const appliedResponse: SyncPushResponse = {
+                ok: true,
+                status: 'applied',
+                commitSeq,
+                results,
+              };
+
+              const affectedTables = Array.from(affectedTablesSet).sort();
+
+              const appliedCommitUpdate: Updateable<
+                SyncCoreDb['sync_commits']
+              > = {
+                result_json: appliedResponse,
+                change_count: allEmitted.length,
+                affected_tables: affectedTables,
+              };
+
+              await syncTrx
+                .updateTable('sync_commits')
+                .set(appliedCommitUpdate)
+                .where('commit_seq', '=', commitSeq)
+                .execute();
+
+              // Insert table commits for subscription filtering
+              if (affectedTables.length > 0) {
+                const tableCommits: Array<
+                  Insertable<SyncCoreDb['sync_table_commits']>
+                > = affectedTables.map((table) => ({
+                  partition_id: partitionId,
+                  table,
+                  commit_seq: commitSeq,
+                }));
+
+                await syncTrx
+                  .insertInto('sync_table_commits')
+                  .values(tableCommits)
+                  .onConflict((oc) =>
+                    oc
+                      .columns(['partition_id', 'table', 'commit_seq'])
+                      .doNothing()
+                  )
+                  .execute();
+              }
+
+              if (useSavepoints) {
+                await sql
+                  .raw(`RELEASE SAVEPOINT ${savepointName}`)
+                  .execute(trx);
+              }
+
+              return {
+                response: appliedResponse,
+                affectedTables,
+                scopeKeys: scopeKeysFromEmitted(allEmitted),
+                emittedChanges: allEmitted.map((c) => ({
+                  table: c.table,
+                  row_id: c.row_id,
+                  op: c.op,
+                  row_json: c.row_json,
+                  row_version: c.row_version,
+                  scopes: c.scopes,
+                })),
+              };
+            } catch (err) {
+              // Roll back app writes but keep the commit row.
+              if (savepointCreated) {
+                try {
+                  await sql
+                    .raw(`ROLLBACK TO SAVEPOINT ${savepointName}`)
+                    .execute(trx);
+                  await sql
+                    .raw(`RELEASE SAVEPOINT ${savepointName}`)
+                    .execute(trx);
+                } catch (savepointErr) {
+                  // If savepoint rollback fails, the transaction may be in an
+                  // inconsistent state. Log and rethrow to fail the entire commit
+                  // rather than risk data corruption.
+                  console.error(
+                    '[pushCommit] Savepoint rollback failed:',
+                    savepointErr
+                  );
+                  throw savepointErr;
+                }
+              }
+
+              if (!(err instanceof RejectCommitError)) throw err;
+
+              const rejectedCommitUpdate: Updateable<
+                SyncCoreDb['sync_commits']
+              > = {
+                result_json: err.response,
+                change_count: 0,
+                affected_tables: [],
+              };
+
+              // Persist the rejected response for commit-level idempotency.
+              await syncTrx
+                .updateTable('sync_commits')
+                .set(rejectedCommitUpdate)
+                .where('commit_seq', '=', commitSeq)
+                .execute();
+
+              return {
+                response: err.response,
+                affectedTables: [],
+                scopeKeys: [],
+                emittedChanges: [],
+              };
+            }
+          })
+        );
+      } catch (error) {
+        const durationMs = Math.max(0, Date.now() - startedAtMs);
+        span.setAttribute('status', 'error');
+        span.setAttribute('duration_ms', durationMs);
+        span.setStatus('error');
+
+        recordPushMetrics({
+          status: 'error',
+          durationMs,
+          operationCount,
+          emittedChangeCount: 0,
+          affectedTableCount: 0,
+        });
+        captureSyncException(error, {
+          event: 'sync.server.push',
+          operationCount,
+        });
+        throw error;
+      }
+    }
+  );
+}

package/src/realtime/in-memory.ts

@@ -0,0 +1,33 @@
+import type { SyncRealtimeBroadcaster, SyncRealtimeEvent } from './types';
+
+/**
+ * In-memory broadcaster for tests and single-process setups.
+ *
+ * This simulates multi-instance fanout when shared across "instances" in a test.
+ */
+export class InMemorySyncRealtimeBroadcaster
+  implements SyncRealtimeBroadcaster
+{
+  private handlers = new Set<(event: SyncRealtimeEvent) => void>();
+
+  subscribe(handler: (event: SyncRealtimeEvent) => void): () => void {
+    this.handlers.add(handler);
+    return () => {
+      this.handlers.delete(handler);
+    };
+  }
+
+  async publish(event: SyncRealtimeEvent): Promise<void> {
+    for (const handler of this.handlers) {
+      try {
+        handler(event);
+      } catch {
+        // ignore individual handler errors; realtime is best-effort
+      }
+    }
+  }
+
+  async close(): Promise<void> {
+    this.handlers.clear();
+  }
+}

package/src/realtime/index.ts

@@ -0,0 +1,5 @@
+export { InMemorySyncRealtimeBroadcaster } from './in-memory';
+export type {
+  SyncRealtimeBroadcaster,
+  SyncRealtimeEvent,
+} from './types';

package/src/realtime/types.ts

@@ -0,0 +1,55 @@
+/**
+ * @syncular/server - Realtime broadcaster types
+ *
+ * Realtime is a best-effort "wake up" mechanism. Correctness always comes from pull + cursors.
+ */
+
+export interface SyncRealtimeCommitEvent {
+  type: 'commit';
+  commitSeq: number;
+  /** Logical partition key (tenant / demo / workspace). */
+  partitionId?: string;
+  /**
+   * Optional scopes affected by the commit.
+   * Broadcasters may omit this to keep payloads small (listeners can look up in DB).
+   */
+  scopeKeys?: string[];
+  /** Optional instance id to suppress echo on the originating instance. */
+  sourceInstanceId?: string;
+}
+
+/**
+ * Presence event for tracking which clients are connected to which scopes.
+ * Used for collaborative features like "who's online" indicators.
+ */
+export interface SyncRealtimePresenceEvent {
+  type: 'presence';
+  /** The action that occurred */
+  action: 'join' | 'leave' | 'update';
+  /** The scope key this presence event relates to */
+  scopeKey: string;
+  /** Client/device identifier */
+  clientId: string;
+  /** Actor/user identifier */
+  actorId: string;
+  /** Optional metadata (e.g., entity being viewed/edited) */
+  metadata?: Record<string, unknown>;
+  /** Optional instance id to suppress echo on the originating instance. */
+  sourceInstanceId?: string;
+}
+
+export type SyncRealtimeEvent =
+  | SyncRealtimeCommitEvent
+  | SyncRealtimePresenceEvent;
+
+export interface SyncRealtimeBroadcaster {
+  /** Publish an event to other app instances. */
+  publish(event: SyncRealtimeEvent): Promise<void>;
+  /**
+   * Subscribe to events from other app instances.
+   * Returns an unsubscribe function.
+   */
+  subscribe(handler: (event: SyncRealtimeEvent) => void): () => void;
+  /** Close underlying resources (best-effort). */
+  close(): Promise<void>;
+}