@syncular/server 0.0.6-95 → 0.1.0

package/src/s3/index.ts

@@ -0,0 +1,346 @@
+/**
+ * S3-compatible blob storage adapter.
+ *
+ * Works with AWS S3, Cloudflare R2, MinIO, and other S3-compatible services.
+ * Requires @aws-sdk/client-s3 and @aws-sdk/s3-request-presigner as peer dependencies.
+ */
+
+import type {
+  BlobSignDownloadOptions,
+  BlobSignedUpload,
+  BlobSignUploadOptions,
+  BlobStorageAdapter,
+} from '@syncular/core';
+
+/**
+ * S3 client interface (minimal subset of @aws-sdk/client-s3).
+ * This allows users to pass in their own configured S3 client.
+ */
+export interface S3ClientLike {
+  send(command: unknown): Promise<unknown>;
+}
+
+/**
+ * Function to create presigned URLs.
+ * This should be getSignedUrl from @aws-sdk/s3-request-presigner.
+ */
+export type GetSignedUrlFn = (
+  client: S3ClientLike,
+  command: unknown,
+  options: { expiresIn: number }
+) => Promise<string>;
+
+/**
+ * S3 command constructors.
+ * These should be imported from @aws-sdk/client-s3.
+ */
+export interface S3Commands {
+  PutObjectCommand: new (input: {
+    Bucket: string;
+    Key: string;
+    ContentLength?: number;
+    ContentType?: string;
+    ChecksumSHA256?: string;
+    Body?: Uint8Array | ReadableStream<Uint8Array>;
+  }) => unknown;
+  GetObjectCommand: new (input: { Bucket: string; Key: string }) => unknown;
+  HeadObjectCommand: new (input: { Bucket: string; Key: string }) => unknown;
+  DeleteObjectCommand: new (input: { Bucket: string; Key: string }) => unknown;
+}
+
+export interface S3BlobStorageAdapterOptions {
+  /** S3 client instance */
+  client: S3ClientLike;
+  /** S3 bucket name */
+  bucket: string;
+  /** Optional key prefix for all blobs */
+  keyPrefix?: string;
+  /** S3 command constructors */
+  commands: S3Commands;
+  /** getSignedUrl function from @aws-sdk/s3-request-presigner */
+  getSignedUrl: GetSignedUrlFn;
+  /**
+   * Whether to require SHA-256 checksum validation on upload.
+   * Supported by S3 and R2. Default: true.
+   */
+  requireChecksum?: boolean;
+}
+
+/**
+ * Create an S3-compatible blob storage adapter.
+ *
+ * @example
+ * ```typescript
+ * import { S3Client, PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
+ * import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+ *
+ * const adapter = createS3BlobStorageAdapter({
+ *   client: new S3Client({ region: 'us-east-1' }),
+ *   bucket: 'my-bucket',
+ *   keyPrefix: 'blobs/',
+ *   commands: { PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand },
+ *   getSignedUrl,
+ * });
+ * ```
+ */
+export function createS3BlobStorageAdapter(
+  options: S3BlobStorageAdapterOptions
+): BlobStorageAdapter {
+  const {
+    client,
+    bucket,
+    keyPrefix = '',
+    commands,
+    getSignedUrl,
+    requireChecksum = true,
+  } = options;
+
+  function getKey(hash: string, partitionId?: string): string {
+    // Remove "sha256:" prefix and use hex as key
+    const hex = hash.startsWith('sha256:') ? hash.slice(7) : hash;
+    const normalizedPartition = partitionId?.trim();
+    if (!normalizedPartition || normalizedPartition === 'default') {
+      return `${keyPrefix}${hex}`;
+    }
+    return `${keyPrefix}${encodeURIComponent(normalizedPartition)}/${hex}`;
+  }
+
+  return {
+    name: 's3',
+
+    async signUpload(opts: BlobSignUploadOptions): Promise<BlobSignedUpload> {
+      const key = getKey(opts.hash, opts.partitionId);
+
+      // Extract hex hash for checksum (S3 expects base64-encoded SHA-256)
+      const hexHash = opts.hash.startsWith('sha256:')
+        ? opts.hash.slice(7)
+        : opts.hash;
+
+      // Convert hex to base64 for S3 checksum header
+      const checksumBase64 = hexToBase64(hexHash);
+
+      const commandInput: {
+        Bucket: string;
+        Key: string;
+        ContentLength: number;
+        ContentType: string;
+        ChecksumSHA256?: string;
+      } = {
+        Bucket: bucket,
+        Key: key,
+        ContentLength: opts.size,
+        ContentType: opts.mimeType,
+      };
+
+      if (requireChecksum) {
+        commandInput.ChecksumSHA256 = checksumBase64;
+      }
+
+      const command = new commands.PutObjectCommand(commandInput);
+      const url = await getSignedUrl(client, command, {
+        expiresIn: opts.expiresIn,
+      });
+
+      const headers: Record<string, string> = {
+        'Content-Type': opts.mimeType,
+        'Content-Length': String(opts.size),
+      };
+
+      if (requireChecksum) {
+        headers['x-amz-checksum-sha256'] = checksumBase64;
+      }
+
+      return {
+        url,
+        method: 'PUT',
+        headers,
+      };
+    },
+
+    async signDownload(opts: BlobSignDownloadOptions): Promise<string> {
+      const key = getKey(opts.hash, opts.partitionId);
+      const command = new commands.GetObjectCommand({
+        Bucket: bucket,
+        Key: key,
+      });
+      return getSignedUrl(client, command, { expiresIn: opts.expiresIn });
+    },
+
+    async exists(
+      hash: string,
+      options?: { partitionId?: string }
+    ): Promise<boolean> {
+      const key = getKey(hash, options?.partitionId);
+      try {
+        const command = new commands.HeadObjectCommand({
+          Bucket: bucket,
+          Key: key,
+        });
+        await client.send(command);
+        return true;
+      } catch (err) {
+        // Check for NotFound error
+        if (isNotFoundError(err)) {
+          return false;
+        }
+        throw err;
+      }
+    },
+
+    async delete(
+      hash: string,
+      options?: { partitionId?: string }
+    ): Promise<void> {
+      const key = getKey(hash, options?.partitionId);
+      const command = new commands.DeleteObjectCommand({
+        Bucket: bucket,
+        Key: key,
+      });
+      await client.send(command);
+    },
+
+    async getMetadata(
+      hash: string,
+      options?: { partitionId?: string }
+    ): Promise<{ size: number; mimeType?: string } | null> {
+      const key = getKey(hash, options?.partitionId);
+      try {
+        const command = new commands.HeadObjectCommand({
+          Bucket: bucket,
+          Key: key,
+        });
+        const response = (await client.send(command)) as {
+          ContentLength?: number;
+          ContentType?: string;
+        };
+        return {
+          size: response.ContentLength ?? 0,
+          mimeType: response.ContentType,
+        };
+      } catch (err) {
+        if (isNotFoundError(err)) {
+          return null;
+        }
+        throw err;
+      }
+    },
+
+    async put(
+      hash: string,
+      data: Uint8Array,
+      _metadata?: Record<string, unknown>,
+      options?: { partitionId?: string }
+    ): Promise<void> {
+      const key = getKey(hash, options?.partitionId);
+      const command = new commands.PutObjectCommand({
+        Bucket: bucket,
+        Key: key,
+        Body: data,
+        ContentLength: data.length,
+        ContentType: 'application/octet-stream',
+      });
+      await client.send(command);
+    },
+
+    async get(
+      hash: string,
+      options?: { partitionId?: string }
+    ): Promise<Uint8Array | null> {
+      const key = getKey(hash, options?.partitionId);
+      try {
+        const command = new commands.GetObjectCommand({
+          Bucket: bucket,
+          Key: key,
+        });
+        const response = (await client.send(command)) as {
+          Body?: { transformToByteArray(): Promise<Uint8Array> };
+        };
+        if (!response.Body) return null;
+        return response.Body.transformToByteArray();
+      } catch (err) {
+        if (isNotFoundError(err)) {
+          return null;
+        }
+        throw err;
+      }
+    },
+
+    async getStream(
+      hash: string,
+      options?: { partitionId?: string }
+    ): Promise<ReadableStream<Uint8Array> | null> {
+      const key = getKey(hash, options?.partitionId);
+      try {
+        const command = new commands.GetObjectCommand({
+          Bucket: bucket,
+          Key: key,
+        });
+        const response = (await client.send(command)) as {
+          Body?: { transformToWebStream(): ReadableStream<Uint8Array> };
+        };
+        if (!response.Body) return null;
+        return response.Body.transformToWebStream();
+      } catch (err) {
+        if (isNotFoundError(err)) {
+          return null;
+        }
+        throw err;
+      }
+    },
+  };
+}
+
+function isNotFoundError(err: unknown): boolean {
+  if (typeof err !== 'object' || err === null) return false;
+  const e = err as { name?: string; $metadata?: { httpStatusCode?: number } };
+  return (
+    e.name === 'NotFound' ||
+    e.name === 'NoSuchKey' ||
+    e.$metadata?.httpStatusCode === 404
+  );
+}
+
+function hexToBase64(hex: string): string {
+  const bytes = new Uint8Array(hex.length / 2);
+  for (let i = 0; i < bytes.length; i++) {
+    bytes[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16);
+  }
+
+  // Use Buffer if available (Node/Bun), otherwise manual base64
+  if (typeof Buffer !== 'undefined') {
+    return Buffer.from(bytes).toString('base64');
+  }
+
+  // Manual base64 encoding
+  const chars =
+    'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+  let result = '';
+  const len = bytes.length;
+  const remainder = len % 3;
+
+  for (let i = 0; i < len - remainder; i += 3) {
+    const a = bytes[i]!;
+    const b = bytes[i + 1]!;
+    const c = bytes[i + 2]!;
+    result +=
+      chars.charAt((a >> 2) & 0x3f) +
+      chars.charAt(((a << 4) | (b >> 4)) & 0x3f) +
+      chars.charAt(((b << 2) | (c >> 6)) & 0x3f) +
+      chars.charAt(c & 0x3f);
+  }
+
+  if (remainder === 1) {
+    const a = bytes[len - 1]!;
+    result += `${chars.charAt((a >> 2) & 0x3f) + chars.charAt((a << 4) & 0x3f)}==`;
+  } else if (remainder === 2) {
+    const a = bytes[len - 2]!;
+    const b = bytes[len - 1]!;
+    result +=
+      chars.charAt((a >> 2) & 0x3f) +
+      chars.charAt(((a << 4) | (b >> 4)) & 0x3f) +
+      chars.charAt((b << 2) & 0x3f) +
+      '=';
+  }
+
+  return result;
+}

package/src/schema.ts

@@ -5,6 +5,7 @@
  * - sync_commits: one row per committed push
  * - sync_changes: one row per emitted change, stamped with scopes
  * - sync_table_commits: commit routing index for fast pull
+ * - sync_scope_commits: scope routing index for scoped incremental pull
  */
 
 import type { SyncOp } from '@syncular/core';
@@ -32,6 +33,10 @@ export interface SyncCommitsTable {
   result_json: unknown | null;
   /** Number of emitted changes (denormalized for observability) */
   change_count: Generated<number>;
+  /** Hex SHA-256 digest of the canonical persisted commit payload */
+  commit_digest: Generated<string | null>;
+  /** Hex SHA-256 root chaining this commit to the previous partition root */
+  commit_chain_root: Generated<string | null>;
   /**
    * Tables affected by this commit (for realtime notifications).
    * Array of table names that had changes.
@@ -83,6 +88,15 @@ export interface SyncClientCursorsTable {
    * This is the intersection of requested scopes and allowed scopes.
    */
   effective_scopes: unknown;
+  /**
+   * Last active realtime subscription snapshot for websocket binary packs.
+   *
+   * This duplicates the protocol-level subscription id/table/scope/cursor/root
+   * shape from the latest successful pull so a restarted realtime server can
+   * rebuild scoped binary notifications before the client performs another
+   * HTTP pull.
+   */
+  realtime_subscriptions: unknown | null;
   /** Last update timestamp */
   updated_at: Generated<string>;
 }
@@ -105,7 +119,11 @@ export interface SyncSnapshotChunksTable {
   row_cursor: string;
   /** Snapshot row limit used to produce this chunk */
   row_limit: number;
-  /** Row encoding (e.g. 'json-row-frame-v1') */
+  /** Cursor after this chunk; null means no next row for this table */
+  next_row_cursor?: string | null;
+  /** 1 when this chunk reaches the end of the table snapshot */
+  is_last_page?: number;
+  /** Row encoding (e.g. 'binary-table-v1') */
   encoding: string;
   /** Compression algorithm (e.g. 'gzip') */
   compression: string;
@@ -123,6 +141,108 @@ export interface SyncSnapshotChunksTable {
   expires_at: string;
 }
 
+/**
+ * Cached scoped snapshot artifacts.
+ *
+ * Artifacts are larger immutable bootstrap payloads, such as scoped SQLite
+ * snapshot files, addressed by a verified manifest and stored in object
+ * storage via blob_hash.
+ */
+export interface SyncSnapshotArtifactsTable {
+  /** Opaque artifact id */
+  artifact_id: string;
+  /** Logical partition key (tenant / demo / workspace) */
+  partition_id: string;
+  /** Effective scope key this artifact belongs to */
+  scope_key: string;
+  /** Stable subscription id this artifact serves */
+  subscription_id: string;
+  /** App table represented by the artifact */
+  table: string;
+  /** Artifact format (e.g. 'sqlite-snapshot-v1') */
+  artifact_kind: string;
+  /** App schema/cache semantic version */
+  schema_version: string;
+  /** Snapshot as-of commit sequence */
+  as_of_commit_seq: number;
+  /** Snapshot row cursor key (empty string represents null) */
+  row_cursor: string;
+  /** Snapshot row limit used to produce this artifact */
+  row_limit: number;
+  /** Number of rows included in the artifact */
+  row_count: number;
+  /** Cursor after this artifact; null means no next row for this table */
+  next_row_cursor?: string | null;
+  /** 1 when this artifact starts at the table's first page */
+  is_first_page: number;
+  /** 1 when this artifact reaches the end of the table snapshot */
+  is_last_page: number;
+  /** Artifact compression algorithm */
+  compression: string;
+  /** Hex-encoded sha256 of artifact body bytes */
+  sha256: string;
+  /** Byte length of artifact body bytes */
+  byte_length: number;
+  /** Hex digest of the scoped artifact manifest */
+  manifest_digest: string;
+  /** Normalized JSON feature set */
+  feature_set_json: string;
+  /** Canonical manifest JSON */
+  manifest_json: string;
+  /** Reference to blob storage */
+  blob_hash: string;
+  /** Created timestamp */
+  created_at: Generated<string>;
+  /** Expiration timestamp (server may delete after this) */
+  expires_at: string;
+}
+
+/**
+ * Shared encrypted CRDT update log.
+ *
+ * One physical table stores updates for every encrypted CRDT field. The logical
+ * stream is keyed by partition_id + stream_id, where stream_id is derived from
+ * app_table + row_id + field_name.
+ */
+export interface SyncCrdtUpdatesTable {
+  seq: Generated<number>;
+  partition_id: string;
+  stream_id: string;
+  app_table: string;
+  row_id: string;
+  field_name: string;
+  update_id: string;
+  actor_id: string | null;
+  client_id: string | null;
+  key_id: string;
+  ciphertext: string;
+  scopes: unknown;
+  created_at: Generated<string>;
+}
+
+/**
+ * Shared encrypted CRDT checkpoints.
+ *
+ * Checkpoints contain encrypted full Yjs/Yrs state snapshots and allow old
+ * update-log rows to be garbage-collected after retention rules are satisfied.
+ */
+export interface SyncCrdtCheckpointsTable {
+  seq: Generated<number>;
+  partition_id: string;
+  stream_id: string;
+  app_table: string;
+  row_id: string;
+  field_name: string;
+  checkpoint_id: string;
+  covers_seq: number;
+  actor_id: string | null;
+  client_id: string | null;
+  key_id: string;
+  ciphertext: string;
+  scopes: unknown;
+  created_at: Generated<string>;
+}
+
 /**
  * Index table: which commits affect which tables.
  *
@@ -136,6 +256,20 @@ export interface SyncTableCommitsTable {
   commit_seq: number;
 }
 
+/**
+ * Index table: which commits affect which table/scope route keys.
+ *
+ * Used to avoid scanning table-level commits that cannot match a scoped
+ * subscription. JSON scope filtering still validates exact multi-scope matches.
+ */
+export interface SyncScopeCommitsTable {
+  /** Logical partition key (tenant / demo / workspace) */
+  partition_id: string;
+  table: string;
+  scope_key: string;
+  commit_seq: number;
+}
+
 /**
  * Database interface for sync infrastructure tables
  * Merge this with your app's database interface
@@ -144,8 +278,12 @@ export interface SyncCoreDb {
   sync_commits: SyncCommitsTable;
   sync_changes: SyncChangesTable;
   sync_client_cursors: SyncClientCursorsTable;
+  sync_crdt_updates: SyncCrdtUpdatesTable;
+  sync_crdt_checkpoints: SyncCrdtCheckpointsTable;
   sync_table_commits: SyncTableCommitsTable;
+  sync_scope_commits: SyncScopeCommitsTable;
   sync_snapshot_chunks: SyncSnapshotChunksTable;
+  sync_snapshot_artifacts: SyncSnapshotArtifactsTable;
 }
 
 /**
@@ -156,6 +294,8 @@ export interface SyncCommitRow {
   actor_id: string;
   created_at: string;
   result_json: unknown | null;
+  commit_digest: string | null;
+  commit_chain_root: string | null;
 }
 
 /**