@syncular/server 0.0.1-100

package/src/dialect/types.ts

@@ -0,0 +1,197 @@
+/**
+ * @syncular/server - Server Sync Dialect Interface
+ *
+ * Abstracts database-specific operations for commit-log sync.
+ * Supports the new JSONB scopes model.
+ */
+
+import type { ScopeValues, StoredScopes, SyncOp } from '@syncular/core';
+import type { Kysely, Transaction } from 'kysely';
+import type { SyncChangeRow, SyncCommitRow, SyncCoreDb } from '../schema';
+
+/**
+ * Common database executor type that works with both Kysely and Transaction.
+ * Generic version allows for extended database types that include sync tables.
+ */
+export type DbExecutor<DB extends SyncCoreDb = SyncCoreDb> =
+  | Kysely<DB>
+  | Transaction<DB>;
+
+/**
+ * Supported dialect names.
+ */
+export type ServerSyncDialectName = string;
+
+export interface IncrementalPullRowsArgs {
+  table: string;
+  scopes: ScopeValues;
+  cursor: number;
+  limitCommits: number;
+  partitionId?: string;
+  batchSize?: number;
+}
+
+export interface IncrementalPullRow {
+  commit_seq: number;
+  actor_id: string;
+  created_at: string;
+  change_id: number;
+  table: string;
+  row_id: string;
+  op: SyncOp;
+  row_json: unknown | null;
+  row_version: number | null;
+  scopes: StoredScopes;
+}
+
+export interface ServerSyncDialect {
+  readonly name: ServerSyncDialectName;
+
+  /** Create sync tables + indexes (idempotent) */
+  ensureSyncSchema<DB extends SyncCoreDb>(db: Kysely<DB>): Promise<void>;
+
+  /** Create console-specific tables (e.g., sync_request_events) - optional */
+  ensureConsoleSchema?<DB extends SyncCoreDb>(db: Kysely<DB>): Promise<void>;
+
+  /** Execute callback in a transaction (or directly if transactions not supported). */
+  executeInTransaction<DB extends SyncCoreDb, T>(
+    db: Kysely<DB>,
+    fn: (executor: DbExecutor<DB>) => Promise<T>
+  ): Promise<T>;
+
+  /** Set REPEATABLE READ (or closest equivalent) */
+  setRepeatableRead<DB extends SyncCoreDb>(trx: DbExecutor<DB>): Promise<void>;
+
+  /** Read the maximum committed commit_seq (0 if none) */
+  readMaxCommitSeq<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    options?: { partitionId?: string }
+  ): Promise<number>;
+
+  /** Read the minimum committed commit_seq (0 if none) */
+  readMinCommitSeq<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    options?: { partitionId?: string }
+  ): Promise<number>;
+
+  /**
+   * Read the next commit sequence numbers that have changes for the given tables.
+   * Must return commit_seq values in ascending order.
+   */
+  readCommitSeqsForPull<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: {
+      cursor: number;
+      limitCommits: number;
+      tables: string[];
+      partitionId?: string;
+    }
+  ): Promise<number[]>;
+
+  /** Read commit metadata for commit_seq values */
+  readCommits<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    commitSeqs: number[],
+    options?: { partitionId?: string }
+  ): Promise<SyncCommitRow[]>;
+
+  /**
+   * Read changes for commit_seq values, filtered by table and scopes.
+   * Uses JSONB filtering for scope matching.
+   */
+  readChangesForCommits<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: {
+      commitSeqs: number[];
+      table: string;
+      scopes: ScopeValues;
+      partitionId?: string;
+    }
+  ): Promise<SyncChangeRow[]>;
+
+  /**
+   * Incremental pull iterator for a subscription.
+   *
+   * Yields change rows joined with commit metadata and filtered by
+   * the subscription's table and scope values.
+   */
+  iterateIncrementalPullRows<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: IncrementalPullRowsArgs
+  ): AsyncGenerator<IncrementalPullRow>;
+
+  /**
+   * Optional compaction of the change log to reduce storage.
+   *
+   * Keeps full history for the most recent N hours.
+   * For older history, keeps only the newest change per (table, row_id, scopes).
+   */
+  compactChanges<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: { fullHistoryHours: number }
+  ): Promise<number>;
+
+  /**
+   * Record/update a client cursor for tracking and pruning.
+   */
+  recordClientCursor<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: {
+      partitionId?: string;
+      clientId: string;
+      actorId: string;
+      cursor: number;
+      effectiveScopes: ScopeValues;
+    }
+  ): Promise<void>;
+
+  /**
+   * Convert a StoredScopes object to database representation.
+   * For Postgres: returns as-is (native JSONB)
+   * For SQLite: returns JSON.stringify()
+   */
+  scopesToDb(scopes: StoredScopes): unknown;
+
+  /**
+   * Convert database scopes representation to StoredScopes.
+   */
+  dbToScopes(value: unknown): StoredScopes;
+
+  /**
+   * Whether the dialect supports SELECT ... FOR UPDATE row locking.
+   * Postgres: true
+   * SQLite: false (uses database-level locking)
+   */
+  readonly supportsForUpdate: boolean;
+
+  /**
+   * Whether the dialect supports SAVEPOINT / ROLLBACK TO SAVEPOINT.
+   * Postgres/SQLite: true
+   * D1 (Durable Object): false (blocks raw SAVEPOINT statements)
+   */
+  readonly supportsSavepoints: boolean;
+
+  /**
+   * Read distinct tables from sync_changes for a given commit.
+   * Used for realtime notifications.
+   */
+  readAffectedTablesFromChanges<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    commitSeq: number,
+    options?: { partitionId?: string }
+  ): Promise<string[]>;
+
+  /**
+   * Convert database array representation to string[].
+   * For Postgres: returns as-is (native array)
+   * For SQLite: returns JSON.parse() or empty array if null
+   */
+  dbToArray(value: unknown): string[];
+
+  /**
+   * Convert string[] to database array representation.
+   * For Postgres: returns as-is (native array)
+   * For SQLite: returns JSON.stringify()
+   */
+  arrayToDb(values: string[]): unknown;
+}

package/src/helpers/conflict.ts

@@ -0,0 +1,64 @@
+/**
+ * @syncular/server - Conflict result builder
+ *
+ * Helper for building conflict results in server table handlers.
+ */
+
+import type { ApplyOperationResult } from '../shapes/types';
+
+export interface BuildConflictResultArgs {
+  /** Index of the operation in the batch */
+  opIndex: number;
+  /** Current server row data */
+  serverRow: unknown;
+  /** Current server version */
+  serverVersion: number;
+  /** Client's base version (what they thought they were updating) */
+  baseVersion: number | null;
+}
+
+/**
+ * Build a conflict result for applyOperation.
+ *
+ * Use this when the client's base version doesn't match the server's current version,
+ * indicating a concurrent modification conflict.
+ *
+ * @example
+ * ```typescript
+ * const handler: ServerTableHandler = {
+ *   table: 'tasks',
+ *   async applyOperation(ctx, op, opIndex) {
+ *     const existing = await ctx.db
+ *       .selectFrom('tasks')
+ *       .selectAll()
+ *       .where('id', '=', op.row_id)
+ *       .executeTakeFirst();
+ *
+ *     // Check for version conflict
+ *     if (existing && op.base_version !== null && existing.version !== op.base_version) {
+ *       return {
+ *         result: buildConflictResult({
+ *           opIndex,
+ *           serverRow: existing,
+ *           serverVersion: existing.version,
+ *           baseVersion: op.base_version,
+ *         }),
+ *       };
+ *     }
+ *
+ *     // ... apply the operation
+ *   },
+ * };
+ * ```
+ */
+export function buildConflictResult(
+  args: BuildConflictResultArgs
+): ApplyOperationResult['result'] {
+  return {
+    opIndex: args.opIndex,
+    status: 'conflict',
+    message: `Version conflict: server=${args.serverVersion}, base=${args.baseVersion}`,
+    server_version: args.serverVersion,
+    server_row: args.serverRow,
+  };
+}

package/src/helpers/emitted-change.ts

@@ -0,0 +1,69 @@
+/**
+ * @syncular/server - Emitted change builder
+ *
+ * Helper for creating emitted changes in server table handlers.
+ */
+
+import type { StoredScopes } from '@syncular/core';
+import type { EmittedChange } from '../shapes/types';
+
+export interface CreateEmittedChangeArgs {
+  /** Table name */
+  table: string;
+  /** Row primary key */
+  rowId: string;
+  /** Operation type */
+  op: 'upsert' | 'delete';
+  /** Row data (null for delete) */
+  row: unknown | null;
+  /** Row version (null if not versioned) */
+  version: number | null;
+  /**
+   * Scope values for this change (stored as JSONB).
+   * Example: { user_id: 'U1', project_id: 'P1' }
+   */
+  scopes: StoredScopes;
+}
+
+/**
+ * Create an emitted change for broadcasting to subscribed clients.
+ *
+ * @example
+ * ```typescript
+ * const handler: ServerTableHandler = {
+ *   table: 'tasks',
+ *   async applyOperation(ctx, op, opIndex) {
+ *     // ... apply the operation ...
+ *
+ *     const newVersion = await getTaskVersion(ctx.db, op.row_id);
+ *     const updatedRow = await getTask(ctx.db, op.row_id);
+ *
+ *     return {
+ *       result: { opIndex, status: 'applied', newVersion },
+ *       emittedChanges: [
+ *         createEmittedChange({
+ *           table: 'tasks',
+ *           rowId: op.row_id,
+ *           op: 'upsert',
+ *           row: updatedRow,
+ *           version: newVersion,
+ *           scopes: { user_id: updatedRow.user_id },
+ *         }),
+ *       ],
+ *     };
+ *   },
+ * };
+ * ```
+ */
+export function createEmittedChange(
+  args: CreateEmittedChangeArgs
+): EmittedChange {
+  return {
+    table: args.table,
+    row_id: args.rowId,
+    op: args.op,
+    row_json: args.op === 'delete' ? null : args.row,
+    row_version: args.version,
+    scopes: args.scopes,
+  };
+}

package/src/helpers/index.ts

@@ -0,0 +1,10 @@
+/**
+ * @syncular/server - Helper utilities
+ *
+ * Convenience helpers for implementing server table handlers.
+ */
+
+export * from './conflict';
+export * from './emitted-change';
+export * from './paginate';
+export * from './scope-strings';

package/src/helpers/paginate.ts

@@ -0,0 +1,82 @@
+/**
+ * @syncular/server - Pagination helper
+ *
+ * Simplifies cursor-based pagination for snapshot queries.
+ */
+
+import type { SelectQueryBuilder } from 'kysely';
+
+export interface PaginateOptions {
+  /** Cursor value to start from (null for first page) */
+  cursor: string | null;
+  /** Number of rows per page */
+  limit: number;
+  /** Column to use for cursor-based pagination (default: 'id') */
+  cursorColumn?: string;
+}
+
+export interface PaginateResult<T> {
+  /** Rows for this page */
+  rows: T[];
+  /** Cursor for next page (null if no more pages) */
+  nextCursor: string | null;
+}
+
+/**
+ * Apply cursor-based pagination to a Kysely query.
+ *
+ * This helper simplifies implementing snapshot pagination by:
+ * - Applying cursor filter if provided
+ * - Ordering by the cursor column
+ * - Fetching limit + 1 to determine if there's a next page
+ * - Computing the next cursor
+ *
+ * @example
+ * ```typescript
+ * const handler: ServerTableHandler = {
+ *   table: 'tasks',
+ *   async snapshot(ctx) {
+ *     const query = ctx.db
+ *       .selectFrom('tasks')
+ *       .selectAll()
+ *       .where('user_id', '=', ctx.actorId);
+ *
+ *     return paginate(query, {
+ *       cursor: ctx.cursor,
+ *       limit: ctx.limit,
+ *     });
+ *   },
+ * };
+ * ```
+ */
+export async function paginate<T>(
+  query: SelectQueryBuilder<any, any, T>,
+  options: PaginateOptions
+): Promise<PaginateResult<T>> {
+  const { cursor, limit, cursorColumn = 'id' } = options;
+
+  // Apply cursor filter if resuming from a previous page
+  let q = query;
+  if (cursor) {
+    q = q.where(cursorColumn, '>', cursor) as typeof q;
+  }
+
+  // Order by cursor column and fetch limit + 1 to check for more pages
+  const rows = await q
+    .orderBy(cursorColumn, 'asc')
+    .limit(limit + 1)
+    .execute();
+
+  // Determine if there are more pages
+  const hasMore = rows.length > limit;
+  const pageRows = hasMore ? rows.slice(0, limit) : rows;
+
+  // Compute next cursor from last row
+  const nextCursor = hasMore
+    ? (((pageRows[pageRows.length - 1] as Record<string, unknown>)?.[
+        cursorColumn
+      ] as string) ?? null)
+    : null;
+
+  return { rows: pageRows, nextCursor };
+}

package/src/helpers/scope-strings.ts

@@ -0,0 +1,101 @@
+/**
+ * @syncular/server - Scope string utilities
+ *
+ * Helpers for creating and parsing scope strings.
+ * Scope strings identify partitions of data for sync subscriptions.
+ */
+
+/**
+ * Result from parsing a scope key
+ */
+interface ParsedScopeKey {
+  /** The prefix (first segment) */
+  prefix: string;
+  /** The remaining values (segments after prefix) */
+  values: string[];
+}
+
+/**
+ * Create a scope string from a prefix and values.
+ *
+ * Scope strings use colon separators: `prefix:value1:value2`
+ *
+ * @example
+ * ```typescript
+ * // Simple scope string
+ * createScopeKey('user', 'alice')
+ * // => 'user:alice'
+ *
+ * // Multi-value scope string
+ * createScopeKey('user', 'alice', 'project', 'proj-1')
+ * // => 'user:alice:project:proj-1'
+ * ```
+ */
+export function createScopeKey(prefix: string, ...values: string[]): string {
+  return [prefix, ...values].join(':');
+}
+
+/**
+ * Parse a scope string into its prefix and values.
+ *
+ * Returns null if the key is invalid or doesn't match the expected prefix.
+ *
+ * @example
+ * ```typescript
+ * // Parse any scope string
+ * parseScopeKey('user:alice')
+ * // => { prefix: 'user', values: ['alice'] }
+ *
+ * // Parse with expected prefix
+ * parseScopeKey('user:alice', 'user')
+ * // => { prefix: 'user', values: ['alice'] }
+ *
+ * // Returns null if prefix doesn't match
+ * parseScopeKey('user:alice', 'project')
+ * // => null
+ *
+ * // Multi-value scope string
+ * parseScopeKey('user:alice:project:proj-1')
+ * // => { prefix: 'user', values: ['alice', 'project', 'proj-1'] }
+ * ```
+ */
+export function parseScopeKey(
+  key: string,
+  expectedPrefix?: string
+): ParsedScopeKey | null {
+  const parts = key.split(':');
+  if (parts.length < 1) return null;
+
+  const [prefix, ...values] = parts;
+  if (!prefix) return null;
+
+  // Check expected prefix if provided
+  if (expectedPrefix && prefix !== expectedPrefix) return null;
+
+  return { prefix, values };
+}
+
+/**
+ * Extract a specific value from a scope string by index.
+ *
+ * @example
+ * ```typescript
+ * // Get first value after prefix
+ * getScopeKeyValue('user:alice:project:proj-1', 0)
+ * // => 'alice'
+ *
+ * // Get second value
+ * getScopeKeyValue('user:alice:project:proj-1', 2)
+ * // => 'proj-1'
+ * ```
+ */
+export function getScopeKeyValue(
+  key: string,
+  valueIndex: number
+): string | null {
+  const parsed = parseScopeKey(key);
+  if (!parsed) return null;
+  return parsed.values[valueIndex] ?? null;
+}
+
+export type { ParsedScopeKey };

package/src/index.ts

@@ -0,0 +1,28 @@
+/**
+ * @syncular/server - Server-side sync infrastructure
+ *
+ * Commit-log based sync with:
+ * - commit log + change log
+ * - scopes + subscriptions (partial sync + auth)
+ * - commit-level idempotency
+ * - blob/media storage
+ */
+export * from '@syncular/core';
+
+export * from './blobs';
+export * from './clients';
+export * from './compaction';
+export * from './dialect';
+export * from './helpers';
+export * from './migrate';
+export * from './proxy';
+export * from './prune';
+export * from './pull';
+export * from './push';
+export * from './realtime';
+export * from './schema';
+export * from './shapes';
+export * from './snapshot-chunks';
+export type { SnapshotChunkStorage } from './snapshot-chunks/types';
+export * from './stats';
+export * from './subscriptions';

package/src/migrate.ts

@@ -0,0 +1,20 @@
+/**
+ * @syncular/server - Schema setup
+ */
+
+import type { Kysely } from 'kysely';
+import type { ServerSyncDialect } from './dialect/types';
+import type { SyncCoreDb } from './schema';
+
+/**
+ * Ensures the sync schema exists in the database.
+ * Safe to call multiple times (idempotent).
+ *
+ * @typeParam DB - Your database type that extends SyncCoreDb
+ */
+export async function ensureSyncSchema<DB extends SyncCoreDb>(
+  db: Kysely<DB>,
+  dialect: ServerSyncDialect
+): Promise<void> {
+  await dialect.ensureSyncSchema(db);
+}

package/src/proxy/handler.test.ts

@@ -0,0 +1,120 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import type { Kysely } from 'kysely';
+import { createBunSqliteDb } from '../../../dialect-bun-sqlite/src';
+import { createSqliteServerDialect } from '../../../server-dialect-sqlite/src';
+import { ensureSyncSchema } from '../migrate';
+import type { SyncCoreDb } from '../schema';
+import { executeProxyQuery } from './handler';
+import { ProxyTableRegistry } from './registry';
+
+interface TasksTable {
+  id: string;
+  user_id: string;
+  title: string;
+  server_version: number;
+}
+
+interface ProxyTestDb extends SyncCoreDb {
+  tasks: TasksTable;
+}
+
+describe('executeProxyQuery', () => {
+  let db: Kysely<ProxyTestDb>;
+  const dialect = createSqliteServerDialect();
+  const shapes = new ProxyTableRegistry().register({
+    table: 'tasks',
+    computeScopes: (row) => ({
+      user_id: String(row.user_id),
+    }),
+  });
+
+  beforeEach(async () => {
+    db = createBunSqliteDb<ProxyTestDb>({ path: ':memory:' });
+    await ensureSyncSchema(db, dialect);
+
+    await db.schema
+      .createTable('tasks')
+      .addColumn('id', 'text', (col) => col.primaryKey())
+      .addColumn('user_id', 'text', (col) => col.notNull())
+      .addColumn('title', 'text', (col) => col.notNull())
+      .addColumn('server_version', 'integer', (col) => col.notNull())
+      .execute();
+
+    await db
+      .insertInto('tasks')
+      .values({
+        id: 't1',
+        user_id: 'u1',
+        title: 'old title',
+        server_version: 1,
+      })
+      .execute();
+  });
+
+  afterEach(async () => {
+    await db.destroy();
+  });
+
+  it('tracks comment-prefixed mutations in the sync oplog', async () => {
+    const result = await executeProxyQuery({
+      db,
+      dialect,
+      shapes,
+      ctx: { actorId: 'actor-1', clientId: 'proxy-client-1' },
+      sqlQuery:
+        '/* admin */ UPDATE tasks SET title = $1, server_version = server_version + 1 WHERE id = $2',
+      parameters: ['new title', 't1'],
+    });
+
+    expect(result.rowCount).toBe(1);
+    expect(result.commitSeq).toBeGreaterThan(0);
+
+    const commitCount = await db
+      .selectFrom('sync_commits')
+      .select(({ fn }) => fn.countAll().as('count'))
+      .executeTakeFirstOrThrow();
+    expect(Number(commitCount.count)).toBe(1);
+
+    const changeCount = await db
+      .selectFrom('sync_changes')
+      .select(({ fn }) => fn.countAll().as('count'))
+      .executeTakeFirstOrThrow();
+    expect(Number(changeCount.count)).toBe(1);
+
+    const updated = await db
+      .selectFrom('tasks')
+      .select(['title', 'server_version'])
+      .where('id', '=', 't1')
+      .executeTakeFirstOrThrow();
+    expect(updated.title).toBe('new title');
+    expect(updated.server_version).toBe(2);
+  });
+
+  it('rejects non-wildcard RETURNING on synced-table mutations', async () => {
+    await expect(
+      executeProxyQuery({
+        db,
+        dialect,
+        shapes,
+        ctx: { actorId: 'actor-1', clientId: 'proxy-client-1' },
+        sqlQuery: 'UPDATE tasks SET title = $1 WHERE id = $2 RETURNING id',
+        parameters: ['blocked title', 't1'],
+      })
+    ).rejects.toThrow(
+      'Proxy mutation on synced table "tasks" must use RETURNING * (or omit RETURNING)'
+    );
+
+    const commitCount = await db
+      .selectFrom('sync_commits')
+      .select(({ fn }) => fn.countAll().as('count'))
+      .executeTakeFirstOrThrow();
+    expect(Number(commitCount.count)).toBe(0);
+
+    const row = await db
+      .selectFrom('tasks')
+      .select(['title'])
+      .where('id', '=', 't1')
+      .executeTakeFirstOrThrow();
+    expect(row.title).toBe('old title');
+  });
+});