@syncular/server 0.0.1-60

package/src/blobs/migrate.ts

@@ -0,0 +1,150 @@
+/**
+ * @syncular/server - Blob storage migrations
+ *
+ * These migrations are separate from core sync migrations because
+ * blob storage is optional and may use external storage (S3/R2).
+ */
+
+import type { Kysely } from 'kysely';
+import { sql } from 'kysely';
+import type { SyncBlobDb, SyncBlobUploadsDb } from './types';
+
+/**
+ * Ensures the blob uploads tracking schema exists.
+ * This table is required for the blob manager regardless of storage backend.
+ *
+ * For PostgreSQL.
+ */
+async function ensureBlobUploadsSchemaPostgres<DB extends SyncBlobUploadsDb>(
+  db: Kysely<DB>
+): Promise<void> {
+  await db.schema
+    .createTable('sync_blob_uploads')
+    .ifNotExists()
+    .addColumn('hash', 'text', (col) => col.primaryKey())
+    .addColumn('size', 'bigint', (col) => col.notNull())
+    .addColumn('mime_type', 'text', (col) => col.notNull())
+    .addColumn('status', 'text', (col) => col.notNull())
+    .addColumn('actor_id', 'text', (col) => col.notNull())
+    .addColumn('created_at', 'timestamptz', (col) =>
+      col.notNull().defaultTo(sql`now()`)
+    )
+    .addColumn('expires_at', 'timestamptz', (col) => col.notNull())
+    .addColumn('completed_at', 'timestamptz')
+    .execute();
+
+  await db.schema
+    .createIndex('idx_sync_blob_uploads_status')
+    .ifNotExists()
+    .on('sync_blob_uploads')
+    .columns(['status'])
+    .execute();
+
+  await db.schema
+    .createIndex('idx_sync_blob_uploads_expires_at')
+    .ifNotExists()
+    .on('sync_blob_uploads')
+    .columns(['expires_at'])
+    .execute();
+}
+
+/**
+ * Ensures the blob uploads tracking schema exists.
+ * This table is required for the blob manager regardless of storage backend.
+ *
+ * For SQLite.
+ */
+async function ensureBlobUploadsSchemasSqlite<DB extends SyncBlobUploadsDb>(
+  db: Kysely<DB>
+): Promise<void> {
+  await db.schema
+    .createTable('sync_blob_uploads')
+    .ifNotExists()
+    .addColumn('hash', 'text', (col) => col.primaryKey())
+    .addColumn('size', 'integer', (col) => col.notNull())
+    .addColumn('mime_type', 'text', (col) => col.notNull())
+    .addColumn('status', 'text', (col) => col.notNull())
+    .addColumn('actor_id', 'text', (col) => col.notNull())
+    .addColumn('created_at', 'text', (col) =>
+      col.notNull().defaultTo(sql`(datetime('now'))`)
+    )
+    .addColumn('expires_at', 'text', (col) => col.notNull())
+    .addColumn('completed_at', 'text')
+    .execute();
+
+  await db.schema
+    .createIndex('idx_sync_blob_uploads_status')
+    .ifNotExists()
+    .on('sync_blob_uploads')
+    .columns(['status'])
+    .execute();
+
+  await db.schema
+    .createIndex('idx_sync_blob_uploads_expires_at')
+    .ifNotExists()
+    .on('sync_blob_uploads')
+    .columns(['expires_at'])
+    .execute();
+}
+
+/**
+ * Ensures the blob storage schema exists (for database adapter).
+ * Only needed if using the database blob storage adapter.
+ *
+ * For PostgreSQL.
+ */
+export async function ensureBlobStorageSchemaPostgres<DB extends SyncBlobDb>(
+  db: Kysely<DB>
+): Promise<void> {
+  // First ensure uploads table
+  await ensureBlobUploadsSchemaPostgres(db);
+
+  // Then create blobs table
+  await db.schema
+    .createTable('sync_blobs')
+    .ifNotExists()
+    .addColumn('hash', 'text', (col) => col.primaryKey())
+    .addColumn('size', 'bigint', (col) => col.notNull())
+    .addColumn('mime_type', 'text', (col) => col.notNull())
+    .addColumn('body', 'bytea', (col) => col.notNull())
+    .addColumn('created_at', 'timestamptz', (col) =>
+      col.notNull().defaultTo(sql`now()`)
+    )
+    .execute();
+}
+
+/**
+ * Ensures the blob storage schema exists (for database adapter).
+ * Only needed if using the database blob storage adapter.
+ *
+ * For SQLite.
+ */
+export async function ensureBlobStorageSchemaSqlite<DB extends SyncBlobDb>(
+  db: Kysely<DB>
+): Promise<void> {
+  // First ensure uploads table
+  await ensureBlobUploadsSchemasSqlite(db);
+
+  // Then create blobs table
+  await db.schema
+    .createTable('sync_blobs')
+    .ifNotExists()
+    .addColumn('hash', 'text', (col) => col.primaryKey())
+    .addColumn('size', 'integer', (col) => col.notNull())
+    .addColumn('mime_type', 'text', (col) => col.notNull())
+    .addColumn('body', 'blob', (col) => col.notNull())
+    .addColumn('created_at', 'text', (col) =>
+      col.notNull().defaultTo(sql`(datetime('now'))`)
+    )
+    .execute();
+}
+
+/**
+ * Drops the blob schema from the database.
+ */
+export async function dropBlobSchema<DB extends SyncBlobDb>(
+  db: Kysely<DB>
+): Promise<void> {
+  await db.schema.dropTable('sync_blobs').ifExists().execute();
+  await db.schema.dropTable('sync_blob_uploads').ifExists().execute();
+}

package/src/blobs/types.ts

@@ -0,0 +1,70 @@
+/**
+ * @syncular/server - Blob storage types
+ */
+
+import type { Generated } from 'kysely';
+
+// ============================================================================
+// Blob Upload Tracking
+// ============================================================================
+
+/**
+ * Blob uploads tracking table.
+ * Tracks initiated uploads and their completion status.
+ */
+export interface SyncBlobUploadsTable {
+  /** SHA-256 hash with prefix: "sha256:<hex>" */
+  hash: string;
+  /** Expected size in bytes */
+  size: number;
+  /** MIME type */
+  mime_type: string;
+  /** Upload status */
+  status: 'pending' | 'complete';
+  /** Actor who initiated the upload */
+  actor_id: string;
+  /** When the upload was initiated */
+  created_at: Generated<string>;
+  /** When the upload expires (for cleanup of incomplete uploads) */
+  expires_at: string;
+  /** When the upload was completed */
+  completed_at: string | null;
+}
+
+export interface SyncBlobUploadsDb {
+  sync_blob_uploads: SyncBlobUploadsTable;
+}
+
+// ============================================================================
+// Blob Storage (Database Adapter)
+// ============================================================================
+
+/**
+ * Blob storage table (for database adapter).
+ * Stores blob content directly in the database.
+ */
+export interface SyncBlobsTable {
+  /** SHA-256 hash with prefix: "sha256:<hex>" */
+  hash: string;
+  /** Size in bytes */
+  size: number;
+  /** MIME type */
+  mime_type: string;
+  /** Blob content */
+  body: Uint8Array;
+  /** When the blob was created */
+  created_at: Generated<string>;
+}
+
+export interface SyncBlobsDb {
+  sync_blobs: SyncBlobsTable;
+}
+
+// ============================================================================
+// Combined DB Interface
+// ============================================================================
+
+/**
+ * Full database interface for blob storage.
+ */
+export interface SyncBlobDb extends SyncBlobUploadsDb, SyncBlobsDb {}

package/src/clients.ts

@@ -0,0 +1,21 @@
+/**
+ * @syncular/server - Client cursor tracking
+ */
+
+import type { ScopeValues } from '@syncular/core';
+import type { DbExecutor, ServerSyncDialect } from './dialect/types';
+import type { SyncCoreDb } from './schema';
+
+export async function recordClientCursor<DB extends SyncCoreDb>(
+  db: DbExecutor<DB>,
+  dialect: ServerSyncDialect,
+  args: {
+    partitionId?: string;
+    clientId: string;
+    actorId: string;
+    cursor: number;
+    effectiveScopes: ScopeValues;
+  }
+): Promise<void> {
+  await dialect.recordClientCursor(db, args);
+}

package/src/compaction.ts

@@ -0,0 +1,77 @@
+/**
+ * @syncular/server - Change-log compaction utilities
+ *
+ * Compaction reduces storage by deleting intermediate history while preserving
+ * the newest change per (shape, table_name, row_id, scope_key) for older data.
+ *
+ * Dialect-specific implementation lives in `ServerSyncDialect.compactChanges`.
+ */
+
+import type { DbExecutor, ServerSyncDialect } from './dialect/types';
+import type { SyncCoreDb } from './schema';
+
+export interface CompactOptions {
+  /**
+   * Keep full (non-compacted) history for the most recent N hours.
+   * Older history may be compacted depending on dialect strategy.
+   */
+  fullHistoryHours: number;
+}
+
+export async function compactChanges<DB extends SyncCoreDb>(
+  db: DbExecutor<DB>,
+  args: { dialect: ServerSyncDialect; options: CompactOptions }
+): Promise<number> {
+  const fullHistoryHours = Math.max(0, args.options.fullHistoryHours);
+  if (fullHistoryHours <= 0) return 0;
+  return args.dialect.compactChanges(db, { fullHistoryHours });
+}
+
+interface CompactState {
+  lastCompactAtMs: number;
+  compactInFlight: Promise<number> | null;
+}
+
+const compactStateByDb = new WeakMap<object, CompactState>();
+
+function getCompactState(db: object): CompactState {
+  const existing = compactStateByDb.get(db);
+  if (existing) return existing;
+
+  const created: CompactState = {
+    lastCompactAtMs: 0,
+    compactInFlight: null,
+  };
+  compactStateByDb.set(db, created);
+  return created;
+}
+
+export async function maybeCompactChanges<DB extends SyncCoreDb>(
+  db: DbExecutor<DB>,
+  args: {
+    dialect: ServerSyncDialect;
+    minIntervalMs: number;
+    options: CompactOptions;
+  }
+): Promise<number> {
+  const state = getCompactState(db);
+  const now = Date.now();
+  if (now - state.lastCompactAtMs < args.minIntervalMs) return 0;
+
+  if (state.compactInFlight) return state.compactInFlight;
+
+  state.compactInFlight = (async () => {
+    try {
+      const deleted = await compactChanges(db, {
+        dialect: args.dialect,
+        options: args.options,
+      });
+      state.lastCompactAtMs = Date.now();
+      return deleted;
+    } finally {
+      state.compactInFlight = null;
+    }
+  })();
+
+  return state.compactInFlight;
+}

package/src/dialect/index.ts

@@ -0,0 +1,5 @@
+/**
+ * @syncular/server - Dialect exports
+ */
+
+export * from './types';

package/src/dialect/types.ts

@@ -0,0 +1,222 @@
+/**
+ * @syncular/server - Server Sync Dialect Interface
+ *
+ * Abstracts database-specific operations for commit-log sync.
+ * Supports the new JSONB scopes model.
+ */
+
+import type { ScopeValues, StoredScopes, SyncOp } from '@syncular/core';
+import type { Kysely, Transaction } from 'kysely';
+import type { SyncChangeRow, SyncCommitRow, SyncCoreDb } from '../schema';
+
+/**
+ * Common database executor type that works with both Kysely and Transaction.
+ * Generic version allows for extended database types that include sync tables.
+ */
+export type DbExecutor<DB extends SyncCoreDb = SyncCoreDb> =
+  | Kysely<DB>
+  | Transaction<DB>;
+
+/**
+ * Supported dialect names.
+ */
+export type ServerSyncDialectName = string;
+
+export interface ServerSyncDialect {
+  readonly name: ServerSyncDialectName;
+
+  /** Create sync tables + indexes (idempotent) */
+  ensureSyncSchema<DB extends SyncCoreDb>(db: Kysely<DB>): Promise<void>;
+
+  /** Create console-specific tables (e.g., sync_request_events) - optional */
+  ensureConsoleSchema?<DB extends SyncCoreDb>(db: Kysely<DB>): Promise<void>;
+
+  /** Execute callback in a transaction (or directly if transactions not supported). */
+  executeInTransaction<DB extends SyncCoreDb, T>(
+    db: Kysely<DB>,
+    fn: (executor: DbExecutor<DB>) => Promise<T>
+  ): Promise<T>;
+
+  /** Set REPEATABLE READ (or closest equivalent) */
+  setRepeatableRead<DB extends SyncCoreDb>(trx: DbExecutor<DB>): Promise<void>;
+
+  /** Read the maximum committed commit_seq (0 if none) */
+  readMaxCommitSeq<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    options?: { partitionId?: string }
+  ): Promise<number>;
+
+  /** Read the minimum committed commit_seq (0 if none) */
+  readMinCommitSeq<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    options?: { partitionId?: string }
+  ): Promise<number>;
+
+  /**
+   * Read the next commit sequence numbers that have changes for the given tables.
+   * Must return commit_seq values in ascending order.
+   */
+  readCommitSeqsForPull<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: {
+      cursor: number;
+      limitCommits: number;
+      tables: string[];
+      partitionId?: string;
+    }
+  ): Promise<number[]>;
+
+  /** Read commit metadata for commit_seq values */
+  readCommits<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    commitSeqs: number[],
+    options?: { partitionId?: string }
+  ): Promise<SyncCommitRow[]>;
+
+  /**
+   * Read changes for commit_seq values, filtered by table and scopes.
+   * Uses JSONB filtering for scope matching.
+   */
+  readChangesForCommits<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: {
+      commitSeqs: number[];
+      table: string;
+      scopes: ScopeValues;
+      partitionId?: string;
+    }
+  ): Promise<SyncChangeRow[]>;
+
+  /**
+   * Optimized incremental pull for a subscription.
+   *
+   * Returns change rows joined with commit metadata and filtered by
+   * the subscription's table and scope values.
+   */
+  readIncrementalPullRows<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: {
+      table: string;
+      scopes: ScopeValues;
+      cursor: number;
+      limitCommits: number;
+      partitionId?: string;
+    }
+  ): Promise<
+    Array<{
+      commit_seq: number;
+      actor_id: string;
+      created_at: string;
+      change_id: number;
+      table: string;
+      row_id: string;
+      op: SyncOp;
+      row_json: unknown | null;
+      row_version: number | null;
+      scopes: StoredScopes;
+    }>
+  >;
+
+  /**
+   * Streaming incremental pull for large result sets.
+   *
+   * Yields changes one at a time instead of loading all into memory.
+   * Use this when expecting large numbers of changes.
+   */
+  streamIncrementalPullRows?<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: {
+      table: string;
+      scopes: ScopeValues;
+      cursor: number;
+      limitCommits: number;
+      partitionId?: string;
+    }
+  ): AsyncGenerator<{
+    commit_seq: number;
+    actor_id: string;
+    created_at: string;
+    change_id: number;
+    table: string;
+    row_id: string;
+    op: SyncOp;
+    row_json: unknown | null;
+    row_version: number | null;
+    scopes: StoredScopes;
+  }>;
+
+  /**
+   * Optional compaction of the change log to reduce storage.
+   *
+   * Keeps full history for the most recent N hours.
+   * For older history, keeps only the newest change per (table, row_id, scopes).
+   */
+  compactChanges<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: { fullHistoryHours: number }
+  ): Promise<number>;
+
+  /**
+   * Record/update a client cursor for tracking and pruning.
+   */
+  recordClientCursor<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: {
+      partitionId?: string;
+      clientId: string;
+      actorId: string;
+      cursor: number;
+      effectiveScopes: ScopeValues;
+    }
+  ): Promise<void>;
+
+  /**
+   * Convert a StoredScopes object to database representation.
+   * For Postgres: returns as-is (native JSONB)
+   * For SQLite: returns JSON.stringify()
+   */
+  scopesToDb(scopes: StoredScopes): unknown;
+
+  /**
+   * Convert database scopes representation to StoredScopes.
+   */
+  dbToScopes(value: unknown): StoredScopes;
+
+  /**
+   * Whether the dialect supports SELECT ... FOR UPDATE row locking.
+   * Postgres: true
+   * SQLite: false (uses database-level locking)
+   */
+  readonly supportsForUpdate: boolean;
+
+  /**
+   * Whether the dialect supports SAVEPOINT / ROLLBACK TO SAVEPOINT.
+   * Postgres/SQLite: true
+   * D1 (Durable Object): false (blocks raw SAVEPOINT statements)
+   */
+  readonly supportsSavepoints: boolean;
+
+  /**
+   * Read distinct tables from sync_changes for a given commit.
+   * Used for realtime notifications.
+   */
+  readAffectedTablesFromChanges<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    commitSeq: number,
+    options?: { partitionId?: string }
+  ): Promise<string[]>;
+
+  /**
+   * Convert database array representation to string[].
+   * For Postgres: returns as-is (native array)
+   * For SQLite: returns JSON.parse() or empty array if null
+   */
+  dbToArray(value: unknown): string[];
+
+  /**
+   * Convert string[] to database array representation.
+   * For Postgres: returns as-is (native array)
+   * For SQLite: returns JSON.stringify()
+   */
+  arrayToDb(values: string[]): unknown;
+}

package/src/helpers/conflict.ts

@@ -0,0 +1,64 @@
+/**
+ * @syncular/server - Conflict result builder
+ *
+ * Helper for building conflict results in server table handlers.
+ */
+
+import type { ApplyOperationResult } from '../shapes/types';
+
+export interface BuildConflictResultArgs {
+  /** Index of the operation in the batch */
+  opIndex: number;
+  /** Current server row data */
+  serverRow: unknown;
+  /** Current server version */
+  serverVersion: number;
+  /** Client's base version (what they thought they were updating) */
+  baseVersion: number | null;
+}
+
+/**
+ * Build a conflict result for applyOperation.
+ *
+ * Use this when the client's base version doesn't match the server's current version,
+ * indicating a concurrent modification conflict.
+ *
+ * @example
+ * ```typescript
+ * const handler: ServerTableHandler = {
+ *   table: 'tasks',
+ *   async applyOperation(ctx, op, opIndex) {
+ *     const existing = await ctx.db
+ *       .selectFrom('tasks')
+ *       .selectAll()
+ *       .where('id', '=', op.row_id)
+ *       .executeTakeFirst();
+ *
+ *     // Check for version conflict
+ *     if (existing && op.base_version !== null && existing.version !== op.base_version) {
+ *       return {
+ *         result: buildConflictResult({
+ *           opIndex,
+ *           serverRow: existing,
+ *           serverVersion: existing.version,
+ *           baseVersion: op.base_version,
+ *         }),
+ *       };
+ *     }
+ *
+ *     // ... apply the operation
+ *   },
+ * };
+ * ```
+ */
+export function buildConflictResult(
+  args: BuildConflictResultArgs
+): ApplyOperationResult['result'] {
+  return {
+    opIndex: args.opIndex,
+    status: 'conflict',
+    message: `Version conflict: server=${args.serverVersion}, base=${args.baseVersion}`,
+    server_version: args.serverVersion,
+    server_row: args.serverRow,
+  };
+}

package/src/helpers/emitted-change.ts

@@ -0,0 +1,69 @@
+/**
+ * @syncular/server - Emitted change builder
+ *
+ * Helper for creating emitted changes in server table handlers.
+ */
+
+import type { StoredScopes } from '@syncular/core';
+import type { EmittedChange } from '../shapes/types';
+
+export interface CreateEmittedChangeArgs {
+  /** Table name */
+  table: string;
+  /** Row primary key */
+  rowId: string;
+  /** Operation type */
+  op: 'upsert' | 'delete';
+  /** Row data (null for delete) */
+  row: unknown | null;
+  /** Row version (null if not versioned) */
+  version: number | null;
+  /**
+   * Scope values for this change (stored as JSONB).
+   * Example: { user_id: 'U1', project_id: 'P1' }
+   */
+  scopes: StoredScopes;
+}
+
+/**
+ * Create an emitted change for broadcasting to subscribed clients.
+ *
+ * @example
+ * ```typescript
+ * const handler: ServerTableHandler = {
+ *   table: 'tasks',
+ *   async applyOperation(ctx, op, opIndex) {
+ *     // ... apply the operation ...
+ *
+ *     const newVersion = await getTaskVersion(ctx.db, op.row_id);
+ *     const updatedRow = await getTask(ctx.db, op.row_id);
+ *
+ *     return {
+ *       result: { opIndex, status: 'applied', newVersion },
+ *       emittedChanges: [
+ *         createEmittedChange({
+ *           table: 'tasks',
+ *           rowId: op.row_id,
+ *           op: 'upsert',
+ *           row: updatedRow,
+ *           version: newVersion,
+ *           scopes: { user_id: updatedRow.user_id },
+ *         }),
+ *       ],
+ *     };
+ *   },
+ * };
+ * ```
+ */
+export function createEmittedChange(
+  args: CreateEmittedChangeArgs
+): EmittedChange {
+  return {
+    table: args.table,
+    row_id: args.rowId,
+    op: args.op,
+    row_json: args.op === 'delete' ? null : args.row,
+    row_version: args.version,
+    scopes: args.scopes,
+  };
+}

package/src/helpers/index.ts

@@ -0,0 +1,10 @@
+/**
+ * @syncular/server - Helper utilities
+ *
+ * Convenience helpers for implementing server table handlers.
+ */
+
+export * from './conflict';
+export * from './emitted-change';
+export * from './paginate';
+export * from './scope-strings';