@syncular/server 0.0.1-100

package/src/blobs/migrate.ts

@@ -0,0 +1,150 @@
+/**
+ * @syncular/server - Blob storage migrations
+ *
+ * These migrations are separate from core sync migrations because
+ * blob storage is optional and may use external storage (S3/R2).
+ */
+
+import type { Kysely } from 'kysely';
+import { sql } from 'kysely';
+import type { SyncBlobDb, SyncBlobUploadsDb } from './types';
+
+/**
+ * Ensures the blob uploads tracking schema exists.
+ * This table is required for the blob manager regardless of storage backend.
+ *
+ * For PostgreSQL.
+ */
+async function ensureBlobUploadsSchemaPostgres<DB extends SyncBlobUploadsDb>(
+  db: Kysely<DB>
+): Promise<void> {
+  await db.schema
+    .createTable('sync_blob_uploads')
+    .ifNotExists()
+    .addColumn('hash', 'text', (col) => col.primaryKey())
+    .addColumn('size', 'bigint', (col) => col.notNull())
+    .addColumn('mime_type', 'text', (col) => col.notNull())
+    .addColumn('status', 'text', (col) => col.notNull())
+    .addColumn('actor_id', 'text', (col) => col.notNull())
+    .addColumn('created_at', 'timestamptz', (col) =>
+      col.notNull().defaultTo(sql`now()`)
+    )
+    .addColumn('expires_at', 'timestamptz', (col) => col.notNull())
+    .addColumn('completed_at', 'timestamptz')
+    .execute();
+
+  await db.schema
+    .createIndex('idx_sync_blob_uploads_status')
+    .ifNotExists()
+    .on('sync_blob_uploads')
+    .columns(['status'])
+    .execute();
+
+  await db.schema
+    .createIndex('idx_sync_blob_uploads_expires_at')
+    .ifNotExists()
+    .on('sync_blob_uploads')
+    .columns(['expires_at'])
+    .execute();
+}
+
+/**
+ * Ensures the blob uploads tracking schema exists.
+ * This table is required for the blob manager regardless of storage backend.
+ *
+ * For SQLite.
+ */
+async function ensureBlobUploadsSchemasSqlite<DB extends SyncBlobUploadsDb>(
+  db: Kysely<DB>
+): Promise<void> {
+  await db.schema
+    .createTable('sync_blob_uploads')
+    .ifNotExists()
+    .addColumn('hash', 'text', (col) => col.primaryKey())
+    .addColumn('size', 'integer', (col) => col.notNull())
+    .addColumn('mime_type', 'text', (col) => col.notNull())
+    .addColumn('status', 'text', (col) => col.notNull())
+    .addColumn('actor_id', 'text', (col) => col.notNull())
+    .addColumn('created_at', 'text', (col) =>
+      col.notNull().defaultTo(sql`(datetime('now'))`)
+    )
+    .addColumn('expires_at', 'text', (col) => col.notNull())
+    .addColumn('completed_at', 'text')
+    .execute();
+
+  await db.schema
+    .createIndex('idx_sync_blob_uploads_status')
+    .ifNotExists()
+    .on('sync_blob_uploads')
+    .columns(['status'])
+    .execute();
+
+  await db.schema
+    .createIndex('idx_sync_blob_uploads_expires_at')
+    .ifNotExists()
+    .on('sync_blob_uploads')
+    .columns(['expires_at'])
+    .execute();
+}
+
+/**
+ * Ensures the blob storage schema exists (for database adapter).
+ * Only needed if using the database blob storage adapter.
+ *
+ * For PostgreSQL.
+ */
+export async function ensureBlobStorageSchemaPostgres<DB extends SyncBlobDb>(
+  db: Kysely<DB>
+): Promise<void> {
+  // First ensure uploads table
+  await ensureBlobUploadsSchemaPostgres(db);
+
+  // Then create blobs table
+  await db.schema
+    .createTable('sync_blobs')
+    .ifNotExists()
+    .addColumn('hash', 'text', (col) => col.primaryKey())
+    .addColumn('size', 'bigint', (col) => col.notNull())
+    .addColumn('mime_type', 'text', (col) => col.notNull())
+    .addColumn('body', 'bytea', (col) => col.notNull())
+    .addColumn('created_at', 'timestamptz', (col) =>
+      col.notNull().defaultTo(sql`now()`)
+    )
+    .execute();
+}
+
+/**
+ * Ensures the blob storage schema exists (for database adapter).
+ * Only needed if using the database blob storage adapter.
+ *
+ * For SQLite.
+ */
+export async function ensureBlobStorageSchemaSqlite<DB extends SyncBlobDb>(
+  db: Kysely<DB>
+): Promise<void> {
+  // First ensure uploads table
+  await ensureBlobUploadsSchemasSqlite(db);
+
+  // Then create blobs table
+  await db.schema
+    .createTable('sync_blobs')
+    .ifNotExists()
+    .addColumn('hash', 'text', (col) => col.primaryKey())
+    .addColumn('size', 'integer', (col) => col.notNull())
+    .addColumn('mime_type', 'text', (col) => col.notNull())
+    .addColumn('body', 'blob', (col) => col.notNull())
+    .addColumn('created_at', 'text', (col) =>
+      col.notNull().defaultTo(sql`(datetime('now'))`)
+    )
+    .execute();
+}
+
+/**
+ * Drops the blob schema from the database.
+ */
+export async function dropBlobSchema<DB extends SyncBlobDb>(
+  db: Kysely<DB>
+): Promise<void> {
+  await db.schema.dropTable('sync_blobs').ifExists().execute();
+  await db.schema.dropTable('sync_blob_uploads').ifExists().execute();
+}

package/src/blobs/types.ts

@@ -0,0 +1,70 @@
+/**
+ * @syncular/server - Blob storage types
+ */
+
+import type { Generated } from 'kysely';
+
+// ============================================================================
+// Blob Upload Tracking
+// ============================================================================
+
+/**
+ * Blob uploads tracking table.
+ * Tracks initiated uploads and their completion status.
+ */
+export interface SyncBlobUploadsTable {
+  /** SHA-256 hash with prefix: "sha256:<hex>" */
+  hash: string;
+  /** Expected size in bytes */
+  size: number;
+  /** MIME type */
+  mime_type: string;
+  /** Upload status */
+  status: 'pending' | 'complete';
+  /** Actor who initiated the upload */
+  actor_id: string;
+  /** When the upload was initiated */
+  created_at: Generated<string>;
+  /** When the upload expires (for cleanup of incomplete uploads) */
+  expires_at: string;
+  /** When the upload was completed */
+  completed_at: string | null;
+}
+
+export interface SyncBlobUploadsDb {
+  sync_blob_uploads: SyncBlobUploadsTable;
+}
+
+// ============================================================================
+// Blob Storage (Database Adapter)
+// ============================================================================
+
+/**
+ * Blob storage table (for database adapter).
+ * Stores blob content directly in the database.
+ */
+export interface SyncBlobsTable {
+  /** SHA-256 hash with prefix: "sha256:<hex>" */
+  hash: string;
+  /** Size in bytes */
+  size: number;
+  /** MIME type */
+  mime_type: string;
+  /** Blob content */
+  body: Uint8Array;
+  /** When the blob was created */
+  created_at: Generated<string>;
+}
+
+export interface SyncBlobsDb {
+  sync_blobs: SyncBlobsTable;
+}
+
+// ============================================================================
+// Combined DB Interface
+// ============================================================================
+
+/**
+ * Full database interface for blob storage.
+ */
+export interface SyncBlobDb extends SyncBlobUploadsDb, SyncBlobsDb {}

package/src/clients.ts

@@ -0,0 +1,21 @@
+/**
+ * @syncular/server - Client cursor tracking
+ */
+
+import type { ScopeValues } from '@syncular/core';
+import type { DbExecutor, ServerSyncDialect } from './dialect/types';
+import type { SyncCoreDb } from './schema';
+
+export async function recordClientCursor<DB extends SyncCoreDb>(
+  db: DbExecutor<DB>,
+  dialect: ServerSyncDialect,
+  args: {
+    partitionId?: string;
+    clientId: string;
+    actorId: string;
+    cursor: number;
+    effectiveScopes: ScopeValues;
+  }
+): Promise<void> {
+  await dialect.recordClientCursor(db, args);
+}

package/src/compaction.ts

@@ -0,0 +1,77 @@
+/**
+ * @syncular/server - Change-log compaction utilities
+ *
+ * Compaction reduces storage by deleting intermediate history while preserving
+ * the newest change per (shape, table_name, row_id, scope_key) for older data.
+ *
+ * Dialect-specific implementation lives in `ServerSyncDialect.compactChanges`.
+ */
+
+import type { DbExecutor, ServerSyncDialect } from './dialect/types';
+import type { SyncCoreDb } from './schema';
+
+export interface CompactOptions {
+  /**
+   * Keep full (non-compacted) history for the most recent N hours.
+   * Older history may be compacted depending on dialect strategy.
+   */
+  fullHistoryHours: number;
+}
+
+export async function compactChanges<DB extends SyncCoreDb>(
+  db: DbExecutor<DB>,
+  args: { dialect: ServerSyncDialect; options: CompactOptions }
+): Promise<number> {
+  const fullHistoryHours = Math.max(0, args.options.fullHistoryHours);
+  if (fullHistoryHours <= 0) return 0;
+  return args.dialect.compactChanges(db, { fullHistoryHours });
+}
+
+interface CompactState {
+  lastCompactAtMs: number;
+  compactInFlight: Promise<number> | null;
+}
+
+const compactStateByDb = new WeakMap<object, CompactState>();
+
+function getCompactState(db: object): CompactState {
+  const existing = compactStateByDb.get(db);
+  if (existing) return existing;
+
+  const created: CompactState = {
+    lastCompactAtMs: 0,
+    compactInFlight: null,
+  };
+  compactStateByDb.set(db, created);
+  return created;
+}
+
+export async function maybeCompactChanges<DB extends SyncCoreDb>(
+  db: DbExecutor<DB>,
+  args: {
+    dialect: ServerSyncDialect;
+    minIntervalMs: number;
+    options: CompactOptions;
+  }
+): Promise<number> {
+  const state = getCompactState(db);
+  const now = Date.now();
+  if (now - state.lastCompactAtMs < args.minIntervalMs) return 0;
+
+  if (state.compactInFlight) return state.compactInFlight;
+
+  state.compactInFlight = (async () => {
+    try {
+      const deleted = await compactChanges(db, {
+        dialect: args.dialect,
+        options: args.options,
+      });
+      state.lastCompactAtMs = Date.now();
+      return deleted;
+    } finally {
+      state.compactInFlight = null;
+    }
+  })();
+
+  return state.compactInFlight;
+}

package/src/dialect/base.ts

@@ -0,0 +1,292 @@
+/**
+ * @syncular/server - Base Server Sync Dialect
+ *
+ * Abstract base class that implements shared query methods for all
+ * database-specific sync dialect implementations.
+ */
+
+import type { ScopeValues, StoredScopes } from '@syncular/core';
+import type { Kysely, RawBuilder, Transaction } from 'kysely';
+import { sql } from 'kysely';
+import type { SyncChangeRow, SyncCommitRow, SyncCoreDb } from '../schema';
+import { coerceIsoString, coerceNumber, parseScopes } from './helpers';
+import type {
+  DbExecutor,
+  IncrementalPullRow,
+  IncrementalPullRowsArgs,
+  ServerSyncDialect,
+} from './types';
+
+/**
+ * Abstract base class for server sync dialects.
+ *
+ * Implements methods that are identical across dialects (pure SQL with no
+ * dialect-specific syntax) and methods that differ only in trivial SQL
+ * fragments (IN vs ANY, jsonb casts). Dialect-specific fragments are
+ * provided via small abstract hook methods.
+ *
+ * Genuinely different methods (DDL, transaction control, scope filtering,
+ * compaction) remain abstract for each dialect to implement.
+ */
+export abstract class BaseServerSyncDialect implements ServerSyncDialect {
+  abstract readonly name: string;
+  abstract readonly supportsForUpdate: boolean;
+  abstract readonly supportsSavepoints: boolean;
+
+  // ===========================================================================
+  // Abstract SQL fragment hooks
+  // ===========================================================================
+
+  /**
+   * Build a SQL fragment for "column IN/= list of numbers".
+   * SQLite: `IN (1, 2, 3)` via sql.join
+   * Postgres: `= ANY(ARRAY[1,2,3]::bigint[])`
+   */
+  protected abstract buildNumberListFilter(
+    values: number[]
+  ): RawBuilder<unknown>;
+
+  /**
+   * Build a SQL fragment for "column IN/= list of strings".
+   * SQLite: `IN ('a', 'b')` via sql.join
+   * Postgres: `= ANY(ARRAY['a','b']::text[])`
+   */
+  protected abstract buildStringListFilter(
+    values: string[]
+  ): RawBuilder<unknown>;
+
+  // ===========================================================================
+  // Abstract methods (genuinely different implementations)
+  // ===========================================================================
+
+  abstract ensureSyncSchema<DB extends SyncCoreDb>(
+    db: Kysely<DB>
+  ): Promise<void>;
+
+  abstract ensureConsoleSchema?<DB extends SyncCoreDb>(
+    db: Kysely<DB>
+  ): Promise<void>;
+
+  abstract executeInTransaction<DB extends SyncCoreDb, T>(
+    db: Kysely<DB>,
+    fn: (executor: DbExecutor<DB>) => Promise<T>
+  ): Promise<T>;
+
+  abstract setRepeatableRead<DB extends SyncCoreDb>(
+    trx: DbExecutor<DB>
+  ): Promise<void>;
+
+  abstract readChangesForCommits<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: {
+      commitSeqs: number[];
+      table: string;
+      scopes: ScopeValues;
+      partitionId?: string;
+    }
+  ): Promise<SyncChangeRow[]>;
+
+  protected abstract readIncrementalPullRowsBatch<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: Omit<IncrementalPullRowsArgs, 'batchSize'>
+  ): Promise<IncrementalPullRow[]>;
+
+  abstract compactChanges<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: { fullHistoryHours: number }
+  ): Promise<number>;
+
+  abstract scopesToDb(scopes: StoredScopes): unknown;
+  abstract dbToArray(value: unknown): string[];
+  abstract arrayToDb(values: string[]): unknown;
+
+  // ===========================================================================
+  // Concrete methods (identical SQL across dialects)
+  // ===========================================================================
+
+  async readMaxCommitSeq<DB extends SyncCoreDb>(
+    db: Kysely<DB> | Transaction<DB>,
+    options?: { partitionId?: string }
+  ): Promise<number> {
+    const partitionId = options?.partitionId ?? 'default';
+    const res = await sql<{ max_seq: unknown }>`
+      SELECT max(commit_seq) as max_seq
+      FROM sync_commits
+      WHERE partition_id = ${partitionId}
+    `.execute(db);
+
+    return coerceNumber(res.rows[0]?.max_seq) ?? 0;
+  }
+
+  async readMinCommitSeq<DB extends SyncCoreDb>(
+    db: Kysely<DB> | Transaction<DB>,
+    options?: { partitionId?: string }
+  ): Promise<number> {
+    const partitionId = options?.partitionId ?? 'default';
+    const res = await sql<{ min_seq: unknown }>`
+      SELECT min(commit_seq) as min_seq
+      FROM sync_commits
+      WHERE partition_id = ${partitionId}
+    `.execute(db);
+
+    return coerceNumber(res.rows[0]?.min_seq) ?? 0;
+  }
+
+  async readAffectedTablesFromChanges<DB extends SyncCoreDb>(
+    db: Kysely<DB> | Transaction<DB>,
+    commitSeq: number,
+    options?: { partitionId?: string }
+  ): Promise<string[]> {
+    const partitionId = options?.partitionId ?? 'default';
+    const res = await sql<{ table: string }>`
+      SELECT DISTINCT "table"
+      FROM sync_changes
+      WHERE commit_seq = ${commitSeq}
+        AND partition_id = ${partitionId}
+    `.execute(db);
+
+    return res.rows
+      .map((r) => r.table)
+      .filter((t): t is string => typeof t === 'string' && t.length > 0);
+  }
+
+  dbToScopes(value: unknown): StoredScopes {
+    return parseScopes(value);
+  }
+
+  // ===========================================================================
+  // Concrete methods using hooks (trivial dialect diffs)
+  // ===========================================================================
+
+  async readCommitSeqsForPull<DB extends SyncCoreDb>(
+    db: Kysely<DB> | Transaction<DB>,
+    args: {
+      cursor: number;
+      limitCommits: number;
+      tables: string[];
+      partitionId?: string;
+    }
+  ): Promise<number[]> {
+    const partitionId = args.partitionId ?? 'default';
+    if (args.tables.length === 0) return [];
+
+    const tablesFilter = this.buildStringListFilter(args.tables);
+
+    const res = await sql<{ commit_seq: unknown }>`
+      SELECT DISTINCT commit_seq
+      FROM sync_table_commits
+      WHERE partition_id = ${partitionId}
+        AND "table" ${tablesFilter}
+        AND commit_seq > ${args.cursor}
+      ORDER BY commit_seq ASC
+      LIMIT ${args.limitCommits}
+    `.execute(db);
+
+    return res.rows
+      .map((r) => coerceNumber(r.commit_seq))
+      .filter(
+        (n): n is number =>
+          typeof n === 'number' && Number.isFinite(n) && n > args.cursor
+      );
+  }
+
+  async readCommits<DB extends SyncCoreDb>(
+    db: Kysely<DB> | Transaction<DB>,
+    commitSeqs: number[],
+    options?: { partitionId?: string }
+  ): Promise<SyncCommitRow[]> {
+    const partitionId = options?.partitionId ?? 'default';
+    if (commitSeqs.length === 0) return [];
+
+    const seqsFilter = this.buildNumberListFilter(commitSeqs);
+
+    const res = await sql<{
+      commit_seq: unknown;
+      actor_id: string;
+      created_at: unknown;
+      result_json: unknown | null;
+    }>`
+      SELECT commit_seq, actor_id, created_at, result_json
+      FROM sync_commits
+      WHERE commit_seq ${seqsFilter}
+        AND partition_id = ${partitionId}
+      ORDER BY commit_seq ASC
+    `.execute(db);
+
+    return res.rows.map((row) => ({
+      commit_seq: coerceNumber(row.commit_seq) ?? 0,
+      actor_id: row.actor_id,
+      created_at: coerceIsoString(row.created_at),
+      result_json: row.result_json ?? null,
+    }));
+  }
+
+  async recordClientCursor<DB extends SyncCoreDb>(
+    db: Kysely<DB> | Transaction<DB>,
+    args: {
+      partitionId?: string;
+      clientId: string;
+      actorId: string;
+      cursor: number;
+      effectiveScopes: ScopeValues;
+    }
+  ): Promise<void> {
+    const partitionId = args.partitionId ?? 'default';
+    const now = new Date().toISOString();
+    const scopesJson = JSON.stringify(args.effectiveScopes);
+
+    await sql`
+      INSERT INTO sync_client_cursors (partition_id, client_id, actor_id, cursor, effective_scopes, updated_at)
+      VALUES (${partitionId}, ${args.clientId}, ${args.actorId}, ${args.cursor}, ${scopesJson}, ${now})
+      ON CONFLICT(partition_id, client_id) DO UPDATE SET
+        actor_id = ${args.actorId},
+        cursor = ${args.cursor},
+        effective_scopes = ${scopesJson},
+        updated_at = ${now}
+    `.execute(db);
+  }
+
+  async *iterateIncrementalPullRows<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: IncrementalPullRowsArgs
+  ): AsyncGenerator<IncrementalPullRow> {
+    const limitCommits = Math.max(1, Math.min(500, args.limitCommits));
+    const batchSize = Math.max(
+      1,
+      Math.min(limitCommits, args.batchSize ?? 100, 500)
+    );
+
+    let processedCommits = 0;
+    let cursor = args.cursor;
+
+    while (processedCommits < limitCommits) {
+      const remainingCommits = limitCommits - processedCommits;
+      const commitLimit = Math.min(batchSize, remainingCommits);
+      const rows = await this.readIncrementalPullRowsBatch(db, {
+        table: args.table,
+        scopes: args.scopes,
+        cursor,
+        limitCommits: commitLimit,
+        partitionId: args.partitionId,
+      });
+
+      if (rows.length === 0) break;
+
+      let maxCommitSeq = cursor;
+      const commitSeqs = new Set<number>();
+
+      for (const row of rows) {
+        maxCommitSeq = Math.max(maxCommitSeq, row.commit_seq);
+        commitSeqs.add(row.commit_seq);
+        yield row;
+      }
+
+      if (maxCommitSeq <= cursor) break;
+
+      processedCommits += commitSeqs.size;
+      cursor = maxCommitSeq;
+
+      if (commitSeqs.size < commitLimit) break;
+    }
+  }
+}

package/src/dialect/helpers.ts

@@ -0,0 +1,61 @@
+/**
+ * @syncular/server - Shared dialect helpers
+ *
+ * Pure helper functions used by all server sync dialect implementations.
+ */
+
+import type { StoredScopes } from '@syncular/core';
+
+export function coerceNumber(value: unknown): number | null {
+  if (value === null || value === undefined) return null;
+  if (typeof value === 'number') return Number.isFinite(value) ? value : null;
+  if (typeof value === 'bigint')
+    return Number.isFinite(Number(value)) ? Number(value) : null;
+  if (typeof value === 'string') {
+    const n = Number(value);
+    return Number.isFinite(n) ? n : null;
+  }
+  return null;
+}
+
+export function coerceIsoString(value: unknown): string {
+  if (typeof value === 'string') return value;
+  if (value instanceof Date) return value.toISOString();
+  return String(value);
+}
+
+export function parseScopes(value: unknown): StoredScopes {
+  if (value === null || value === undefined) return {};
+  if (typeof value === 'object' && !Array.isArray(value)) {
+    const result: StoredScopes = {};
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      if (typeof v === 'string') {
+        result[k] = v;
+      }
+    }
+    return result;
+  }
+  if (typeof value === 'string') {
+    try {
+      const parsed = JSON.parse(value);
+      if (
+        typeof parsed === 'object' &&
+        parsed !== null &&
+        !Array.isArray(parsed)
+      ) {
+        const result: StoredScopes = {};
+        for (const [k, v] of Object.entries(
+          parsed as Record<string, unknown>
+        )) {
+          if (typeof v === 'string') {
+            result[k] = v;
+          }
+        }
+        return result;
+      }
+    } catch {
+      // ignore
+    }
+  }
+  return {};
+}

package/src/dialect/index.ts

@@ -0,0 +1,7 @@
+/**
+ * @syncular/server - Dialect exports
+ */
+
+export * from './base';
+export * from './helpers';
+export * from './types';