@syncular/server 0.0.1-60

package/src/helpers/paginate.ts

@@ -0,0 +1,82 @@
+/**
+ * @syncular/server - Pagination helper
+ *
+ * Simplifies cursor-based pagination for snapshot queries.
+ */
+
+import type { SelectQueryBuilder } from 'kysely';
+
+export interface PaginateOptions {
+  /** Cursor value to start from (null for first page) */
+  cursor: string | null;
+  /** Number of rows per page */
+  limit: number;
+  /** Column to use for cursor-based pagination (default: 'id') */
+  cursorColumn?: string;
+}
+
+export interface PaginateResult<T> {
+  /** Rows for this page */
+  rows: T[];
+  /** Cursor for next page (null if no more pages) */
+  nextCursor: string | null;
+}
+
+/**
+ * Apply cursor-based pagination to a Kysely query.
+ *
+ * This helper simplifies implementing snapshot pagination by:
+ * - Applying cursor filter if provided
+ * - Ordering by the cursor column
+ * - Fetching limit + 1 to determine if there's a next page
+ * - Computing the next cursor
+ *
+ * @example
+ * ```typescript
+ * const handler: ServerTableHandler = {
+ *   table: 'tasks',
+ *   async snapshot(ctx) {
+ *     const query = ctx.db
+ *       .selectFrom('tasks')
+ *       .selectAll()
+ *       .where('user_id', '=', ctx.actorId);
+ *
+ *     return paginate(query, {
+ *       cursor: ctx.cursor,
+ *       limit: ctx.limit,
+ *     });
+ *   },
+ * };
+ * ```
+ */
+export async function paginate<T>(
+  query: SelectQueryBuilder<any, any, T>,
+  options: PaginateOptions
+): Promise<PaginateResult<T>> {
+  const { cursor, limit, cursorColumn = 'id' } = options;
+
+  // Apply cursor filter if resuming from a previous page
+  let q = query;
+  if (cursor) {
+    q = q.where(cursorColumn, '>', cursor) as typeof q;
+  }
+
+  // Order by cursor column and fetch limit + 1 to check for more pages
+  const rows = await q
+    .orderBy(cursorColumn, 'asc')
+    .limit(limit + 1)
+    .execute();
+
+  // Determine if there are more pages
+  const hasMore = rows.length > limit;
+  const pageRows = hasMore ? rows.slice(0, limit) : rows;
+
+  // Compute next cursor from last row
+  const nextCursor = hasMore
+    ? (((pageRows[pageRows.length - 1] as Record<string, unknown>)?.[
+        cursorColumn
+      ] as string) ?? null)
+    : null;
+
+  return { rows: pageRows, nextCursor };
+}

package/src/helpers/scope-strings.ts

@@ -0,0 +1,101 @@
+/**
+ * @syncular/server - Scope string utilities
+ *
+ * Helpers for creating and parsing scope strings.
+ * Scope strings identify partitions of data for sync subscriptions.
+ */
+
+/**
+ * Result from parsing a scope key
+ */
+interface ParsedScopeKey {
+  /** The prefix (first segment) */
+  prefix: string;
+  /** The remaining values (segments after prefix) */
+  values: string[];
+}
+
+/**
+ * Create a scope string from a prefix and values.
+ *
+ * Scope strings use colon separators: `prefix:value1:value2`
+ *
+ * @example
+ * ```typescript
+ * // Simple scope string
+ * createScopeKey('user', 'alice')
+ * // => 'user:alice'
+ *
+ * // Multi-value scope string
+ * createScopeKey('user', 'alice', 'project', 'proj-1')
+ * // => 'user:alice:project:proj-1'
+ * ```
+ */
+export function createScopeKey(prefix: string, ...values: string[]): string {
+  return [prefix, ...values].join(':');
+}
+
+/**
+ * Parse a scope string into its prefix and values.
+ *
+ * Returns null if the key is invalid or doesn't match the expected prefix.
+ *
+ * @example
+ * ```typescript
+ * // Parse any scope string
+ * parseScopeKey('user:alice')
+ * // => { prefix: 'user', values: ['alice'] }
+ *
+ * // Parse with expected prefix
+ * parseScopeKey('user:alice', 'user')
+ * // => { prefix: 'user', values: ['alice'] }
+ *
+ * // Returns null if prefix doesn't match
+ * parseScopeKey('user:alice', 'project')
+ * // => null
+ *
+ * // Multi-value scope string
+ * parseScopeKey('user:alice:project:proj-1')
+ * // => { prefix: 'user', values: ['alice', 'project', 'proj-1'] }
+ * ```
+ */
+export function parseScopeKey(
+  key: string,
+  expectedPrefix?: string
+): ParsedScopeKey | null {
+  const parts = key.split(':');
+  if (parts.length < 1) return null;
+
+  const [prefix, ...values] = parts;
+  if (!prefix) return null;
+
+  // Check expected prefix if provided
+  if (expectedPrefix && prefix !== expectedPrefix) return null;
+
+  return { prefix, values };
+}
+
+/**
+ * Extract a specific value from a scope string by index.
+ *
+ * @example
+ * ```typescript
+ * // Get first value after prefix
+ * getScopeKeyValue('user:alice:project:proj-1', 0)
+ * // => 'alice'
+ *
+ * // Get second value
+ * getScopeKeyValue('user:alice:project:proj-1', 2)
+ * // => 'proj-1'
+ * ```
+ */
+export function getScopeKeyValue(
+  key: string,
+  valueIndex: number
+): string | null {
+  const parsed = parseScopeKey(key);
+  if (!parsed) return null;
+  return parsed.values[valueIndex] ?? null;
+}
+
+export type { ParsedScopeKey };

package/src/index.ts

@@ -0,0 +1,28 @@
+/**
+ * @syncular/server - Server-side sync infrastructure
+ *
+ * Commit-log based sync with:
+ * - commit log + change log
+ * - scopes + subscriptions (partial sync + auth)
+ * - commit-level idempotency
+ * - blob/media storage
+ */
+export * from '@syncular/core';
+
+export * from './blobs';
+export * from './clients';
+export * from './compaction';
+export * from './dialect';
+export * from './helpers';
+export * from './migrate';
+export * from './proxy';
+export * from './prune';
+export * from './pull';
+export * from './push';
+export * from './realtime';
+export * from './schema';
+export * from './shapes';
+export * from './snapshot-chunks';
+export type { SnapshotChunkStorage } from './snapshot-chunks/types';
+export * from './stats';
+export * from './subscriptions';

package/src/migrate.ts

@@ -0,0 +1,20 @@
+/**
+ * @syncular/server - Schema setup
+ */
+
+import type { Kysely } from 'kysely';
+import type { ServerSyncDialect } from './dialect/types';
+import type { SyncCoreDb } from './schema';
+
+/**
+ * Ensures the sync schema exists in the database.
+ * Safe to call multiple times (idempotent).
+ *
+ * @typeParam DB - Your database type that extends SyncCoreDb
+ */
+export async function ensureSyncSchema<DB extends SyncCoreDb>(
+  db: Kysely<DB>,
+  dialect: ServerSyncDialect
+): Promise<void> {
+  await dialect.ensureSyncSchema(db);
+}

package/src/proxy/handler.ts

@@ -0,0 +1,152 @@
+/**
+ * @syncular/server - Proxy Query Handler
+ *
+ * Executes proxied queries with automatic oplog generation for mutations.
+ */
+
+import type { Kysely, RawBuilder } from 'kysely';
+import { sql } from 'kysely';
+import type { ServerSyncDialect } from '../dialect/types';
+import type { SyncCoreDb } from '../schema';
+import {
+  appendReturning,
+  detectMutation,
+  hasReturningClause,
+} from './mutation-detector';
+import { createOplogEntries } from './oplog';
+import type { ProxyTableRegistry } from './registry';
+import type { ProxyQueryContext } from './types';
+
+export interface ExecuteProxyQueryArgs<DB extends SyncCoreDb = SyncCoreDb> {
+  /** Database connection or transaction */
+  db: Kysely<DB>;
+  /** Server sync dialect */
+  dialect: ServerSyncDialect;
+  /** Proxy table registry for oplog generation */
+  shapes: ProxyTableRegistry;
+  /** Query context (actor/client IDs) */
+  ctx: ProxyQueryContext;
+  /** SQL query string */
+  sqlQuery: string;
+  /** Query parameters */
+  parameters: readonly unknown[];
+}
+
+export interface ExecuteProxyQueryResult {
+  /** Query result rows (for SELECT or RETURNING) */
+  rows?: unknown[];
+  /** Number of affected rows (for mutations) */
+  rowCount?: number;
+  /** Commit sequence if oplog was created */
+  commitSeq?: number;
+  /** Affected tables if oplog was created */
+  affectedTables?: string[];
+}
+
+/**
+ * Build a raw SQL query with parameters using Kysely's sql helper.
+ *
+ * This converts parameterized SQL (using $1, $2, etc.) to Kysely's format.
+ */
+function buildRawQuery(
+  sqlQuery: string,
+  parameters: readonly unknown[]
+): RawBuilder<unknown> {
+  // If no parameters, just use sql.raw
+  if (parameters.length === 0) {
+    return sql.raw(sqlQuery);
+  }
+
+  // Parse the SQL and split by parameter placeholders ($1, $2, etc.)
+  // Then use sql.join to build the query with proper parameter binding
+  const parts: RawBuilder<unknown>[] = [];
+  let lastIndex = 0;
+  const paramRegex = /\$(\d+)/g;
+  let match: RegExpExecArray | null;
+
+  while ((match = paramRegex.exec(sqlQuery)) !== null) {
+    // Add the SQL before this parameter
+    if (match.index > lastIndex) {
+      parts.push(sql.raw(sqlQuery.slice(lastIndex, match.index)));
+    }
+    // Add the parameter value (1-indexed in SQL, 0-indexed in array)
+    const paramIndex = Number.parseInt(match[1]!, 10) - 1;
+    if (paramIndex >= 0 && paramIndex < parameters.length) {
+      // Use sql.value to create a proper parameter binding
+      parts.push(sql.val(parameters[paramIndex]));
+    } else {
+      // Keep the original placeholder if out of bounds (shouldn't happen)
+      parts.push(sql.raw(match[0]));
+    }
+    lastIndex = match.index + match[0].length;
+  }
+
+  // Add remaining SQL after last parameter
+  if (lastIndex < sqlQuery.length) {
+    parts.push(sql.raw(sqlQuery.slice(lastIndex)));
+  }
+
+  // Join all parts together
+  return sql.join(parts, sql.raw(''));
+}
+
+/**
+ * Execute a proxied query with automatic oplog generation for mutations.
+ *
+ * - Read queries: Execute directly and return rows
+ * - Mutations: Append RETURNING *, execute, create oplog entries
+ */
+export async function executeProxyQuery<DB extends SyncCoreDb>(
+  args: ExecuteProxyQueryArgs<DB>
+): Promise<ExecuteProxyQueryResult> {
+  const { db, dialect, shapes, ctx, sqlQuery, parameters } = args;
+
+  const mutation = detectMutation(sqlQuery);
+
+  if (!mutation) {
+    // Read query - execute directly
+    const result = await buildRawQuery(sqlQuery, parameters).execute(db);
+    return { rows: result.rows };
+  }
+
+  // Check if this table has a registered shape
+  const shape = shapes.get(mutation.tableName);
+  if (!shape) {
+    // No shape registered - execute without oplog
+    // This allows proxy operations on non-synced tables
+    const result = await buildRawQuery(sqlQuery, parameters).execute(db);
+    return {
+      rows: result.rows,
+      rowCount: Number(result.numAffectedRows ?? 0),
+    };
+  }
+
+  // Mutation with registered shape - append RETURNING * and create oplog
+  const needsReturning = !hasReturningClause(sqlQuery);
+  const finalSql = needsReturning ? appendReturning(sqlQuery) : sqlQuery;
+
+  const result = await buildRawQuery(finalSql, parameters).execute(db);
+  const affectedRows = result.rows as Record<string, unknown>[];
+
+  if (affectedRows.length === 0) {
+    return { rowCount: 0 };
+  }
+
+  // Create oplog entries
+  const { commitSeq, affectedTables } = await createOplogEntries({
+    trx: db,
+    dialect,
+    actorId: ctx.actorId,
+    clientId: ctx.clientId,
+    partitionId: ctx.partitionId,
+    shape,
+    operation: mutation.operation,
+    rows: affectedRows,
+  });
+
+  return {
+    rowCount: affectedRows.length,
+    commitSeq,
+    affectedTables,
+  };
+}

package/src/proxy/index.ts

@@ -0,0 +1,18 @@
+/**
+ * @syncular/server - Proxy Exports
+ *
+ * Server-side proxy functionality for database access.
+ */
+
+// Query execution
+export {
+  type ExecuteProxyQueryArgs,
+  type ExecuteProxyQueryResult,
+  executeProxyQuery,
+} from './handler';
+// Mutation detection
+export { type DetectedMutation, detectMutation } from './mutation-detector';
+// Oplog creation
+// Registry
+export { ProxyTableRegistry } from './registry';
+// Types

package/src/proxy/mutation-detector.ts

@@ -0,0 +1,83 @@
+/**
+ * @syncular/server - Mutation Detector
+ *
+ * Detects whether a SQL query is a mutation (INSERT/UPDATE/DELETE).
+ */
+
+import type { SyncOp } from '@syncular/core';
+
+export interface DetectedMutation {
+  /** Operation type */
+  operation: SyncOp;
+  /** Table name being modified */
+  tableName: string;
+}
+
+/**
+ * Detect if a SQL query is a mutation and extract table info.
+ *
+ * @param sql - The SQL query string
+ * @returns Mutation info if detected, null for read queries
+ */
+export function detectMutation(sql: string): DetectedMutation | null {
+  const trimmed = sql.trim();
+
+  // INSERT INTO [schema.]table
+  const insertMatch = trimmed.match(
+    /^\s*INSERT\s+INTO\s+(?:["']?(\w+)["']?\.)?["']?(\w+)["']?/i
+  );
+  if (insertMatch) {
+    return {
+      operation: 'upsert',
+      tableName: insertMatch[2]!,
+    };
+  }
+
+  // UPDATE [schema.]table
+  const updateMatch = trimmed.match(
+    /^\s*UPDATE\s+(?:["']?(\w+)["']?\.)?["']?(\w+)["']?/i
+  );
+  if (updateMatch) {
+    return {
+      operation: 'upsert',
+      tableName: updateMatch[2]!,
+    };
+  }
+
+  // DELETE FROM [schema.]table
+  const deleteMatch = trimmed.match(
+    /^\s*DELETE\s+FROM\s+(?:["']?(\w+)["']?\.)?["']?(\w+)["']?/i
+  );
+  if (deleteMatch) {
+    return {
+      operation: 'delete',
+      tableName: deleteMatch[2]!,
+    };
+  }
+
+  return null;
+}
+
+/**
+ * Check if SQL already has a RETURNING clause.
+ */
+export function hasReturningClause(sql: string): boolean {
+  // Simple check - look for RETURNING keyword not in a string
+  return /\bRETURNING\b/i.test(sql);
+}
+
+/**
+ * Append RETURNING * to a mutation query if not already present.
+ *
+ * @param sql - The SQL query string
+ * @returns Modified SQL with RETURNING *
+ */
+export function appendReturning(sql: string): string {
+  if (hasReturningClause(sql)) {
+    return sql;
+  }
+
+  // Remove trailing semicolon if present
+  const trimmed = sql.trim().replace(/;\s*$/, '');
+  return `${trimmed} RETURNING *`;
+}

package/src/proxy/oplog.ts

@@ -0,0 +1,144 @@
+/**
+ * @syncular/server - Oplog Creation
+ *
+ * Creates sync oplog entries for proxy mutations.
+ */
+
+import { randomUUID } from 'node:crypto';
+import type { SyncOp } from '@syncular/core';
+import { type Kysely, sql } from 'kysely';
+import type { ServerSyncDialect } from '../dialect/types';
+import type { SyncCoreDb } from '../schema';
+import type { ProxyTableHandler } from './types';
+
+/**
+ * Generate a random ID for commit tracking.
+ */
+function generateId(): string {
+  return randomUUID();
+}
+
+/**
+ * Create oplog entries for affected rows.
+ *
+ * This is called after a mutation to record the changes in the sync oplog,
+ * making them visible to sync clients.
+ */
+export async function createOplogEntries<DB extends SyncCoreDb>(args: {
+  trx: Kysely<DB>;
+  dialect: ServerSyncDialect;
+  actorId: string;
+  clientId: string;
+  partitionId?: string;
+  shape: ProxyTableHandler;
+  operation: SyncOp;
+  rows: Record<string, unknown>[];
+}): Promise<{ commitSeq: number; affectedTables: string[] }> {
+  const { trx, dialect, actorId, clientId, shape, operation, rows } = args;
+  const partitionId = args.partitionId ?? 'default';
+
+  if (rows.length === 0) {
+    return { commitSeq: 0, affectedTables: [] };
+  }
+
+  const pk = shape.primaryKey ?? 'id';
+  const versionCol = shape.versionColumn ?? 'server_version';
+
+  // Create commit record
+  const commitResult = await sql<{ commit_seq: number }>`
+    insert into ${sql.table('sync_commits')} (
+      partition_id,
+      actor_id,
+      client_id,
+      client_commit_id,
+      meta,
+      result_json
+    )
+    values (
+      ${partitionId},
+      ${actorId},
+      ${clientId},
+      ${`proxy:${generateId()}`},
+      ${null},
+      ${null}
+    )
+    returning commit_seq
+  `.execute(trx);
+
+  const commitRow = commitResult.rows[0];
+  if (!commitRow) {
+    throw new Error('Failed to insert sync_commits row');
+  }
+  const commitSeq = Number(commitRow.commit_seq);
+
+  // Compute scopes for all rows and collect changes
+  const affectedTablesSet = new Set<string>();
+  affectedTablesSet.add(shape.table);
+
+  const changes = rows.map((row) => {
+    const scopes = shape.computeScopes(row);
+
+    return {
+      commit_seq: commitSeq,
+      partition_id: partitionId,
+      table: shape.table,
+      row_id: String(row[pk]),
+      op: operation,
+      row_json: operation === 'delete' ? null : row,
+      row_version: row[versionCol] != null ? Number(row[versionCol]) : null,
+      scopes: dialect.scopesToDb(scopes),
+    };
+  });
+
+  // Insert changes
+  await sql`
+	    insert into ${sql.table('sync_changes')} (
+	      commit_seq,
+	      partition_id,
+	      "table",
+	      row_id,
+	      op,
+	      row_json,
+	      row_version,
+	      scopes
+    )
+    values ${sql.join(
+      changes.map(
+        (c) => sql`(
+          ${c.commit_seq},
+          ${c.partition_id},
+          ${c.table},
+          ${c.row_id},
+          ${c.op},
+          ${c.row_json},
+          ${c.row_version},
+          ${c.scopes}
+        )`
+      ),
+      sql`, `
+    )}
+  `.execute(trx);
+
+  // Update commit with affected tables
+  const affectedTables = Array.from(affectedTablesSet);
+  const sortedAffectedTables = affectedTables.sort();
+  await sql`
+    update ${sql.table('sync_commits')}
+    set change_count = ${rows.length}, affected_tables = ${sortedAffectedTables}
+    where commit_seq = ${commitSeq}
+  `.execute(trx);
+
+  // Insert table commits for subscription filtering
+  if (affectedTables.length > 0) {
+    await sql`
+	      insert into ${sql.table('sync_table_commits')} (partition_id, "table", commit_seq)
+	      values ${sql.join(
+          sortedAffectedTables.map((table) => sql`(${partitionId}, ${table}, ${commitSeq})`),
+          sql`, `
+        )}
+	      on conflict (partition_id, "table", commit_seq) do nothing
+	    `.execute(trx);
+  }
+
+  return { commitSeq, affectedTables };
+}

package/src/proxy/registry.ts

@@ -0,0 +1,56 @@
+/**
+ * @syncular/server - Proxy Table Registry
+ *
+ * Registry for proxy table handlers.
+ */
+
+import type { ProxyTableHandler } from './types';
+
+/**
+ * Registry for proxy table handlers.
+ *
+ * Maps table names to table handlers for oplog generation.
+ */
+export class ProxyTableRegistry {
+  private handlers = new Map<string, ProxyTableHandler>();
+
+  /**
+   * Register a proxy table handler.
+   */
+  register(handler: ProxyTableHandler): this {
+    this.handlers.set(handler.table, handler);
+    return this;
+  }
+
+  /**
+   * Get handler by table name.
+   */
+  get(tableName: string): ProxyTableHandler | undefined {
+    return this.handlers.get(tableName);
+  }
+
+  /**
+   * Get handler by table name or throw.
+   */
+  getOrThrow(tableName: string): ProxyTableHandler {
+    const handler = this.handlers.get(tableName);
+    if (!handler) {
+      throw new Error(`No proxy table registered for table: ${tableName}`);
+    }
+    return handler;
+  }
+
+  /**
+   * Check if a table has a registered handler.
+   */
+  has(tableName: string): boolean {
+    return this.handlers.has(tableName);
+  }
+
+  /**
+   * Get all registered handlers.
+   */
+  getAll(): ProxyTableHandler[] {
+    return Array.from(this.handlers.values());
+  }
+}