@syncular/server 0.0.1-60

package/src/shapes/types.ts

@@ -0,0 +1,267 @@
+import type {
+  ScopePattern,
+  ScopeValues,
+  StoredScopes,
+  SyncOp,
+  SyncOperation,
+  SyncOperationResult,
+} from '@syncular/core';
+import type { ZodSchema, z } from 'zod';
+import type { DbExecutor } from '../dialect/types';
+import type { SyncCoreDb } from '../schema';
+
+/**
+ * Emitted change to be stored in the oplog.
+ * Uses JSONB scopes instead of scope_keys array.
+ */
+export interface EmittedChange {
+  /** Table name */
+  table: string;
+  /** Row primary key */
+  row_id: string;
+  /** Operation type */
+  op: SyncOp;
+  /** Row data as JSON (null for deletes) */
+  row_json: unknown | null;
+  /** Row version for optimistic concurrency */
+  row_version: number | null;
+  /**
+   * Scope values for this change (stored as JSONB).
+   * Example: { user_id: 'U1', project_id: 'P1' }
+   */
+  scopes: StoredScopes;
+}
+
+export interface ApplyOperationResult {
+  result: SyncOperationResult;
+  emittedChanges: EmittedChange[];
+}
+
+/**
+ * Context for server operations.
+ */
+export interface ServerContext<DB extends SyncCoreDb = SyncCoreDb> {
+  /** Database connection (transaction in applyOperation) */
+  db: DbExecutor<DB>;
+  /** Actor ID (user ID from auth) */
+  actorId: string;
+}
+
+/**
+ * Context passed to snapshot method.
+ */
+export interface ServerSnapshotContext<DB extends SyncCoreDb = SyncCoreDb>
+  extends ServerContext<DB> {
+  /** Database executor for the snapshot */
+  db: DbExecutor<DB>;
+  /** Effective scope values for this subscription */
+  scopeValues: ScopeValues;
+  /** Pagination cursor (row_id for keyset pagination) */
+  cursor: string | null;
+  /** Max rows to return */
+  limit: number;
+}
+
+/**
+ * Context passed to applyOperation method.
+ */
+export interface ServerApplyOperationContext<DB extends SyncCoreDb = SyncCoreDb>
+  extends ServerContext<DB> {
+  /** Database executor for the operation */
+  trx: DbExecutor<DB>;
+  /** Client/device identifier */
+  clientId: string;
+  /** Unique commit identifier */
+  commitId: string;
+  /**
+   * Client's schema version when the commit was created.
+   * Use this to transform payloads from older client versions.
+   */
+  schemaVersion: number;
+}
+
+/**
+ * Server-side scope configuration for advanced use cases.
+ * Use this when you need custom extraction, access control, or filtering.
+ *
+ * For simple cases, use the simplified scope array format:
+ * `scopes: ['user:{user_id}']`
+ */
+interface ServerScopeConfig {
+  /**
+   * Column name containing the scope value.
+   * For simple patterns like 'user:{user_id}' → column: 'user_id'
+   */
+  column?: string;
+
+  /**
+   * Custom extractor for complex patterns.
+   * Example: extract year/month from a date column
+   */
+  extract?: (row: Record<string, unknown>) => Record<string, string>;
+
+  /**
+   * Optional access control per scope pattern.
+   * Return true if the actor can access this scope value.
+   */
+  access?: (
+    ctx: ServerContext,
+    vars: Record<string, string>
+  ) => Promise<boolean>;
+
+  /**
+   * Optional filter builder for wildcard subscriptions.
+   * Called when the subscription uses wildcards for this pattern.
+   */
+  toFilter?: (
+    vars: Record<string, string | undefined>,
+    query: unknown
+  ) => unknown;
+}
+
+/**
+ * Server shape options - configuration for a table's sync behavior.
+ */
+export interface ServerShapeOptions<
+  DB extends SyncCoreDb = SyncCoreDb,
+  Scopes extends Record<ScopePattern, Record<string, string>> = Record<
+    ScopePattern,
+    Record<string, string>
+  >,
+  TableName extends string = string,
+  Params extends ZodSchema = ZodSchema,
+> {
+  /**
+   * Scope patterns this shape uses.
+   * Array of pattern keys from SharedScopes.
+   */
+  scopes: (keyof Scopes)[];
+
+  /**
+   * Scope definitions - how each pattern maps to row data.
+   * Defaults to using column name = variable name.
+   */
+  scopeDefinitions?: Partial<Record<keyof Scopes, ServerScopeConfig>>;
+
+  /**
+   * Resolve allowed scope values for the current actor.
+   * Called once per request to determine what the actor can access.
+   *
+   * Returns scope values the actor is allowed to access.
+   * The server will intersect requested scopes with these.
+   *
+   * @example
+   * resolveScopes: async (ctx) => ({
+   *   user_id: [ctx.user.id],
+   *   project_id: ctx.user.projectIds,
+   * })
+   */
+  resolveScopes: (ctx: ServerContext<DB>) => Promise<ScopeValues>;
+
+  /**
+   * Optional Zod schema for subscription parameters.
+   */
+  params?: Params;
+
+  /**
+   * Primary key column (default: 'id')
+   */
+  primaryKey?: string;
+
+  /**
+   * Version column for optimistic concurrency (default: 'server_version')
+   */
+  versionColumn?: string;
+
+  /**
+   * Tables that must be bootstrapped before this one.
+   */
+  dependsOn?: string[];
+
+  /**
+   * TTL for cached snapshot chunks (ms). Default: 24 hours.
+   */
+  snapshotChunkTtlMs?: number;
+
+  /**
+   * Transform client payload → server row on writes.
+   */
+  transformInbound?: (
+    payload: Record<string, unknown>,
+    ctx: ServerApplyOperationContext<DB>
+  ) => Partial<DB[TableName & keyof DB]>;
+
+  /**
+   * Transform server row → client payload on reads.
+   */
+  transformOutbound?: (
+    row: DB[TableName & keyof DB]
+  ) => Record<string, unknown>;
+
+  /**
+   * Custom snapshot implementation.
+   * Default uses keyset pagination ordered by primary key.
+   */
+  snapshot?: (
+    ctx: ServerSnapshotContext<DB>,
+    params: Params extends ZodSchema ? z.infer<Params> : undefined
+  ) => Promise<{ rows: unknown[]; nextCursor: string | null }>;
+
+  /**
+   * Custom apply operation implementation.
+   */
+  applyOperation?: (
+    ctx: ServerApplyOperationContext<DB>,
+    op: SyncOperation,
+    opIndex: number
+  ) => Promise<ApplyOperationResult>;
+}
+
+/**
+ * Server-side table handler for snapshots and mutations.
+ * This is the internal handler interface used by the sync engine.
+ */
+export interface ServerTableHandler<DB extends SyncCoreDb = SyncCoreDb> {
+  /** Table name */
+  table: string;
+
+  /** Scope patterns used by this shape */
+  scopePatterns: ScopePattern[];
+
+  /**
+   * Tables that must be bootstrapped before this one.
+   */
+  dependsOn?: string[];
+
+  /**
+   * TTL for cached snapshot chunks (ms).
+   */
+  snapshotChunkTtlMs?: number;
+
+  /**
+   * Resolve allowed scope values for the current actor.
+   */
+  resolveScopes: (ctx: ServerContext<DB>) => Promise<ScopeValues>;
+
+  /**
+   * Extract stored scopes from a row.
+   */
+  extractScopes: (row: Record<string, unknown>) => StoredScopes;
+
+  /**
+   * Build a bootstrap snapshot page.
+   */
+  snapshot(
+    ctx: ServerSnapshotContext<DB>,
+    params: Record<string, unknown> | undefined
+  ): Promise<{ rows: unknown[]; nextCursor: string | null }>;
+
+  /**
+   * Apply a single operation.
+   */
+  applyOperation(
+    ctx: ServerApplyOperationContext<DB>,
+    op: SyncOperation,
+    opIndex: number
+  ): Promise<ApplyOperationResult>;
+}

package/src/snapshot-chunks/adapters/s3.ts

@@ -0,0 +1,68 @@
+/**
+ * @syncular/server - S3-compatible snapshot chunk storage adapter
+ *
+ * Stores snapshot chunk bodies in S3/R2/MinIO with metadata in database.
+ */
+
+import type { BlobStorageAdapter } from '@syncular/core';
+import type { Kysely } from 'kysely';
+import type { SyncCoreDb } from '../../schema';
+import { createDbMetadataChunkStorage } from '../db-metadata';
+
+export interface S3SnapshotChunkStorageOptions {
+  /** Database instance for metadata */
+  db: Kysely<SyncCoreDb>;
+  /** S3 blob storage adapter */
+  s3Adapter: BlobStorageAdapter;
+  /** Optional key prefix for all chunks */
+  keyPrefix?: string;
+}
+
+/**
+ * Create S3-compatible snapshot chunk storage.
+ *
+ * Stores chunk bodies in S3/R2/MinIO and metadata in the database.
+ * Supports presigned URLs for direct client downloads.
+ *
+ * @example
+ * ```typescript
+ * import { createS3BlobStorageAdapter } from '@syncular/server/blobs/adapters/s3';
+ * import { createS3SnapshotChunkStorage } from '@syncular/server/snapshot-chunks/adapters/s3';
+ *
+ * const s3Adapter = createS3BlobStorageAdapter({
+ *   client: new S3Client({ region: 'us-east-1' }),
+ *   bucket: 'my-snapshot-chunks',
+ *   commands: { PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand },
+ *   getSignedUrl,
+ * });
+ *
+ * const chunkStorage = createS3SnapshotChunkStorage({
+ *   db: kysely,
+ *   s3Adapter,
+ *   keyPrefix: 'snapshots/',
+ * });
+ * ```
+ */
+export function createS3SnapshotChunkStorage(
+  options: S3SnapshotChunkStorageOptions
+) {
+  const { db, s3Adapter, keyPrefix } = options;
+
+  // Wrap the S3 adapter to use prefixed keys
+  const prefixedAdapter: BlobStorageAdapter = keyPrefix
+    ? {
+        ...s3Adapter,
+        name: `${s3Adapter.name}+prefixed`,
+        // Keys are already handled by the S3 adapter, prefix is applied there
+      }
+    : s3Adapter;
+
+  // Use the database metadata storage with S3 for bodies
+  const storage = createDbMetadataChunkStorage({
+    db,
+    blobAdapter: prefixedAdapter,
+    chunkIdPrefix: keyPrefix ? `${keyPrefix.replace(/\/$/, '')}_` : 'chunk_',
+  });
+
+  return storage;
+}

package/src/snapshot-chunks/db-metadata.ts

@@ -0,0 +1,238 @@
+/**
+ * @syncular/server - Database-backed metadata store for snapshot chunks
+ *
+ * Stores chunk metadata in sync_snapshot_chunks_metadata table,
+ * body content in blob storage adapter.
+ */
+
+import { createHash } from 'node:crypto';
+import type { BlobStorageAdapter, SyncSnapshotChunkRef } from '@syncular/core';
+import type { Kysely } from 'kysely';
+import type { SyncCoreDb } from '../schema';
+import type { SnapshotChunkMetadata, SnapshotChunkPageKey } from './types';
+
+export interface DbMetadataSnapshotChunkStorageOptions {
+  /** Database instance */
+  db: Kysely<SyncCoreDb>;
+  /** Blob storage adapter for body content */
+  blobAdapter: BlobStorageAdapter;
+  /** Optional prefix for chunk IDs */
+  chunkIdPrefix?: string;
+}
+
+/**
+ * Create a snapshot chunk storage that uses:
+ * - Database for metadata (scope, commit seq, etc.)
+ * - Blob adapter for body content
+ */
+export function createDbMetadataChunkStorage(
+  options: DbMetadataSnapshotChunkStorageOptions
+): {
+  name: string;
+  storeChunk: (
+    metadata: Omit<
+      SnapshotChunkMetadata,
+      'chunkId' | 'byteLength' | 'blobHash'
+    > & {
+      body: Uint8Array;
+    }
+  ) => Promise<SyncSnapshotChunkRef>;
+  readChunk: (chunkId: string) => Promise<Uint8Array | null>;
+  findChunk: (
+    pageKey: SnapshotChunkPageKey
+  ) => Promise<SyncSnapshotChunkRef | null>;
+  cleanupExpired: (beforeIso: string) => Promise<number>;
+} {
+  const { db, blobAdapter, chunkIdPrefix = 'chunk_' } = options;
+
+  // Generate deterministic blob hash from content
+  function computeBlobHash(body: Uint8Array): string {
+    return `sha256:${createHash('sha256').update(body).digest('hex')}`;
+  }
+
+  // Generate unique chunk ID
+  function generateChunkId(): string {
+    return `${chunkIdPrefix}${Date.now()}_${Math.random().toString(36).slice(2, 11)}`;
+  }
+
+  return {
+    name: `db-metadata+${blobAdapter.name}`,
+
+    async storeChunk(
+      metadata: Omit<
+        SnapshotChunkMetadata,
+        'chunkId' | 'byteLength' | 'blobHash'
+      > & {
+        body: Uint8Array;
+      }
+    ): Promise<SyncSnapshotChunkRef> {
+      const { body, ...metaWithoutBody } = metadata;
+      const blobHash = computeBlobHash(body);
+      const chunkId = generateChunkId();
+      const now = new Date().toISOString();
+
+      // Check if blob already exists (content-addressed dedup)
+      const blobExists = await blobAdapter.exists(blobHash);
+
+      if (!blobExists) {
+        // Store body in blob adapter
+        if (blobAdapter.put) {
+          await blobAdapter.put(blobHash, body);
+        } else {
+          throw new Error(
+            `Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`
+          );
+        }
+      }
+
+      // Upsert metadata in database
+      await db
+        .insertInto('sync_snapshot_chunks')
+        .values({
+          chunk_id: chunkId,
+          partition_id: metaWithoutBody.partitionId,
+          scope_key: metaWithoutBody.scopeKey,
+          scope: metaWithoutBody.scope,
+          as_of_commit_seq: metaWithoutBody.asOfCommitSeq,
+          row_cursor: metaWithoutBody.rowCursor ?? '',
+          row_limit: metaWithoutBody.rowLimit,
+          encoding: metaWithoutBody.encoding,
+          compression: metaWithoutBody.compression,
+          sha256: metaWithoutBody.sha256,
+          byte_length: body.length,
+          blob_hash: blobHash,
+          expires_at: metaWithoutBody.expiresAt,
+          created_at: now,
+        })
+        .onConflict((oc) =>
+          oc
+            .columns([
+              'partition_id',
+              'scope_key',
+              'scope',
+              'as_of_commit_seq',
+              'row_cursor',
+              'row_limit',
+              'encoding',
+              'compression',
+            ])
+            .doUpdateSet({
+              expires_at: metaWithoutBody.expiresAt,
+              blob_hash: blobHash,
+              sha256: metaWithoutBody.sha256,
+              byte_length: body.length,
+              row_cursor: metaWithoutBody.rowCursor ?? '',
+            })
+        )
+        .execute();
+
+      return {
+        id: chunkId,
+        sha256: metaWithoutBody.sha256,
+        byteLength: body.length,
+        encoding: metaWithoutBody.encoding,
+        compression: metaWithoutBody.compression,
+      };
+    },
+
+    async readChunk(chunkId: string): Promise<Uint8Array | null> {
+      // Get metadata to find blob hash
+      const row = await db
+        .selectFrom('sync_snapshot_chunks')
+        .select(['blob_hash'])
+        .where('chunk_id', '=', chunkId)
+        .executeTakeFirst();
+
+      if (!row) return null;
+
+      // Read from blob adapter
+      if (blobAdapter.get) {
+        return blobAdapter.get(row.blob_hash);
+      }
+
+      throw new Error(
+        `Blob adapter ${blobAdapter.name} does not support direct get() for snapshot chunks`
+      );
+    },
+
+    async findChunk(
+      pageKey: SnapshotChunkPageKey
+    ): Promise<SyncSnapshotChunkRef | null> {
+      const nowIso = new Date().toISOString();
+      const rowCursorKey = pageKey.rowCursor ?? '';
+
+      const row = await db
+        .selectFrom('sync_snapshot_chunks')
+        .select([
+          'chunk_id',
+          'sha256',
+          'byte_length',
+          'encoding',
+          'compression',
+        ])
+        .where('partition_id', '=', pageKey.partitionId)
+        .where('scope_key', '=', pageKey.scopeKey)
+        .where('scope', '=', pageKey.scope)
+        .where('as_of_commit_seq', '=', pageKey.asOfCommitSeq)
+        .where('row_cursor', '=', rowCursorKey)
+        .where('row_limit', '=', pageKey.rowLimit)
+        .where('encoding', '=', pageKey.encoding)
+        .where('compression', '=', pageKey.compression)
+        .where('expires_at', '>', nowIso)
+        .executeTakeFirst();
+
+      if (!row) return null;
+
+      if (row.encoding !== 'ndjson') {
+        throw new Error(
+          `Unexpected snapshot chunk encoding: ${String(row.encoding)}`
+        );
+      }
+      if (row.compression !== 'gzip') {
+        throw new Error(
+          `Unexpected snapshot chunk compression: ${String(row.compression)}`
+        );
+      }
+
+      return {
+        id: row.chunk_id,
+        sha256: row.sha256,
+        byteLength: Number(row.byte_length ?? 0),
+        encoding: row.encoding,
+        compression: row.compression,
+      };
+    },
+
+    async cleanupExpired(beforeIso: string): Promise<number> {
+      // Find expired chunks
+      const expiredRows = await db
+        .selectFrom('sync_snapshot_chunks')
+        .select(['chunk_id', 'blob_hash'])
+        .where('expires_at', '<=', beforeIso)
+        .execute();
+
+      if (expiredRows.length === 0) return 0;
+
+      // Delete from blob storage (best effort)
+      for (const row of expiredRows) {
+        try {
+          await blobAdapter.delete(row.blob_hash);
+        } catch {
+          // Ignore deletion errors - blob may be shared or already deleted
+          // Log for observability but don't fail the cleanup
+          console.warn(
+            `Failed to delete blob ${row.blob_hash} for chunk ${row.chunk_id}, may be already deleted or shared`
+          );
+        }
+      }
+
+      // Delete metadata from database
+      const result = await db
+        .deleteFrom('sync_snapshot_chunks')
+        .where('expires_at', '<=', beforeIso)
+        .executeTakeFirst();
+
+      return Number(result.numDeletedRows ?? 0);
+    },
+  };
+}

package/src/snapshot-chunks/index.ts

@@ -0,0 +1,9 @@
+/**
+ * @syncular/server - Snapshot chunk storage
+ *
+ * Separates chunk metadata (database) from body content (blob storage).
+ */
+
+export * from './adapters/s3';
+export * from './db-metadata';
+export * from './types';

package/src/snapshot-chunks/types.ts

@@ -0,0 +1,79 @@
+/**
+ * @syncular/server - Snapshot chunk storage types
+ *
+ * Separates chunk metadata (in database) from chunk body (in blob storage).
+ * Enables flexible storage backends (database, S3, R2, etc.)
+ */
+
+import type { SyncSnapshotChunkRef } from '@syncular/core';
+
+/**
+ * Page key for identifying a specific chunk
+ */
+export interface SnapshotChunkPageKey {
+  partitionId: string;
+  scopeKey: string;
+  scope: string;
+  asOfCommitSeq: number;
+  rowCursor: string | null;
+  rowLimit: number;
+  encoding: 'ndjson';
+  compression: 'gzip';
+}
+
+/**
+ * Metadata stored in the database for each chunk
+ */
+export interface SnapshotChunkMetadata {
+  chunkId: string;
+  partitionId: string;
+  scopeKey: string;
+  scope: string;
+  asOfCommitSeq: number;
+  rowCursor: string | null;
+  rowLimit: number;
+  encoding: 'ndjson';
+  compression: 'gzip';
+  sha256: string;
+  byteLength: number;
+  blobHash: string; // Reference to blob storage
+  expiresAt: string;
+}
+
+/**
+ * Storage interface for snapshot chunks
+ */
+export interface SnapshotChunkStorage {
+  /** Storage adapter name */
+  readonly name: string;
+
+  /**
+   * Store a chunk. Returns chunk reference.
+   * If chunk with same content already exists (by hash), returns existing reference.
+   */
+  storeChunk(
+    metadata: Omit<
+      SnapshotChunkMetadata,
+      'chunkId' | 'byteLength' | 'blobHash'
+    > & {
+      body: Uint8Array;
+    }
+  ): Promise<SyncSnapshotChunkRef>;
+
+  /**
+   * Read chunk body by chunk ID
+   */
+  readChunk(chunkId: string): Promise<Uint8Array | null>;
+
+  /**
+   * Find existing chunk by page key
+   */
+  findChunk(
+    pageKey: SnapshotChunkPageKey
+  ): Promise<SyncSnapshotChunkRef | null>;
+
+  /**
+   * Delete expired chunks. Returns number deleted.
+   */
+  cleanupExpired(beforeIso: string): Promise<number>;
+}