@syncular/server 0.15.23 → 0.15.24

package/dist/storage.d.ts

@@ -186,15 +186,23 @@ export interface StorageTransaction {
      */
     scanRowsByIndex?(query: IndexRowScanQuery): Promise<StoredRow[]>;
     /**
-     * Serialize candidate-state validation for this partition before any row
-     * read/write. Required at runtime when `commitValidator` is configured.
+     * Serialize every push apply for this partition before any operation read,
+     * validation, merge, or write. The push layer re-checks idempotency only
+     * after this resolves and retains the lock through terminal-result commit.
+     * Missing support fails closed before an app-row mutation.
+     */
+    lockPartitionForPush?(): Promise<void>;
+    /**
+     * @deprecated Implement `lockPartitionForPush`. Kept as a compatibility
+     * bridge for custom adapters whose existing implementation already locks
+     * the complete partition from before candidate reads through commit.
      */
     lockPartitionForCommitValidation?(): Promise<void>;
     /**
-     * §6.8 rejection finalization while the validation serialization lock is
-     * still held: discard every candidate write, persist the rejected
-     * idempotency result, and finish the transaction atomically. Required when
-     * `commitValidator` is configured so a concurrent duplicate cannot rerun it.
+     * Rejection finalization while the push-apply serialization lock is still
+     * held: discard every candidate write, persist the rejected idempotency
+     * result, and finish the transaction atomically. Required for every push so
+     * a concurrent duplicate cannot rerun operations, validators, or merges.
      */
     commitRejectedPushResult?(clientId: string, clientCommitId: string, result: StoredPushResult): Promise<void>;
     upsertRow(table: string, row: StoredRow, context?: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.15.23",
+  "version": "0.15.24",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -53,7 +53,7 @@
     "!dist/**/*.test.d.ts"
   ],
   "dependencies": {
-    "@syncular/core": "0.15.23"
+    "@syncular/core": "0.15.24"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/d1-storage.ts

@@ -31,11 +31,11 @@
  * reading `max_commit_seq` live and buffering the `+1` write. Under a single
  * Worker request this is exact. Two concurrent pushes to one partition need an
  * external serialization point (normally a per-partition Durable Object); a
- * realtime notifier alone does not serialize HTTP writes. A stateless
- * HTTP-only deployment SHOULD front same-partition writes with a coordinating
- * primitive (a DO or a Queue). For §6.8 whole-commit validation this becomes
- * mandatory and fail-closed: the coordinator must explicitly set
- * `commitValidationSerialized`, because D1 cannot provide the required lock.
+ * realtime notifier alone does not serialize HTTP writes. Every deployment
+ * that accepts D1 pushes MUST front same-partition sync rounds with a
+ * coordinating primitive (a DO or a Queue). The adapter fails closed unless
+ * that coordinator explicitly sets `pushApplySerialized`, because D1 cannot
+ * provide the required pre-operation lock.
  * This mirrors PostgreSQL's per-partition row lock, achieved by placement
  * rather than a lock D1 does not expose.
  */
@@ -143,12 +143,12 @@ class D1Transaction implements StorageTransaction {
   readonly #db: D1Database;
   readonly #partition: string;
   readonly #resolveTable: (name: string) => CompiledTable;
-  readonly #commitValidationSerialized: boolean;
+  readonly #pushApplySerialized: boolean;
   readonly #buffer: BufferedStatement[] = [];
   #open = true;
   /** Live snapshot of `max_commit_seq`, advanced within this transaction. */
   #maxCommitSeq: number | undefined;
-  #commitValidationCheckpoint: number | undefined;
+  #pushApplyCheckpoint: number | undefined;
   #lastApplicationOpIndex: number | undefined;
   /**
    * Read-your-own-writes overlay (§6.2 needs `getRow` to see buffered writes
@@ -161,12 +161,12 @@ class D1Transaction implements StorageTransaction {
     db: D1Database,
     partition: string,
     resolveTable: (name: string) => CompiledTable,
-    commitValidationSerialized: boolean,
+    pushApplySerialized: boolean,
   ) {
     this.#db = db;
     this.#partition = partition;
     this.#resolveTable = resolveTable;
-    this.#commitValidationSerialized = commitValidationSerialized;
+    this.#pushApplySerialized = pushApplySerialized;
   }
 
   #assertOpen(): void {
@@ -312,16 +312,16 @@ class D1Transaction implements StorageTransaction {
       .slice(0, query.limit);
   }
 
-  async lockPartitionForCommitValidation(): Promise<void> {
+  async lockPartitionForPush(): Promise<void> {
     this.#assertOpen();
-    if (!this.#commitValidationSerialized) {
+    if (!this.#pushApplySerialized) {
       throw new Error(
-        'D1 whole-commit validation requires externally serialized partition writes',
+        'D1 push apply requires externally serialized partition writes',
       );
     }
     // D1 has no interactive lock. The caller explicitly asserted that every
     // write for this partition is already serialized (normally by its DO).
-    this.#commitValidationCheckpoint = this.#buffer.length;
+    this.#pushApplyCheckpoint = this.#buffer.length;
   }
 
   async commitRejectedPushResult(
@@ -330,11 +330,9 @@ class D1Transaction implements StorageTransaction {
     result: StoredPushResult,
   ): Promise<void> {
     this.#assertOpen();
-    const checkpoint = this.#commitValidationCheckpoint;
+    const checkpoint = this.#pushApplyCheckpoint;
     if (checkpoint === undefined) {
-      throw new Error(
-        'whole-commit rejection requires its validation checkpoint',
-      );
+      throw new Error('push rejection requires its apply checkpoint');
     }
     this.#buffer.length = checkpoint;
     this.#pending.clear();
@@ -558,16 +556,22 @@ class D1Transaction implements StorageTransaction {
 
   async commit(): Promise<void> {
     this.#assertOpen();
-    this.#open = false;
-    if (this.#buffer.length === 0) return;
+    if (this.#buffer.length === 0) {
+      this.#open = false;
+      return;
+    }
     const statements = this.#buffer.map((entry) =>
       this.#db.prepare(entry.sql).bind(...entry.params),
     );
     // One atomic D1 batch — the §6.4 all-or-nothing commit.
     try {
       await this.#db.batch(statements);
+      this.#open = false;
     } catch (error) {
       if (isD1ConstraintError(error)) {
+        // D1 batches are atomic. Keep this logical transaction open so the
+        // push layer can discard its buffered candidates and persist the
+        // terminal rejection while the external partition queue is retained.
         throw new StorageConstraintError(error, this.#lastApplicationOpIndex);
       }
       throw error;
@@ -590,23 +594,30 @@ const D1_MAX_BIND_PARAMS = 100;
 export interface D1ServerStorageOptions {
   /**
    * Assert that all writes for a partition reach this storage serially.
-   * Required for §6.8 because D1 exposes no interactive transaction lock.
-   * Set this only inside a per-partition Durable Object or equivalent
-   * coordinator; the default fails closed when a commit validator is used.
+   * Required for every push because D1 exposes no interactive transaction
+   * lock. Set this only inside an explicit per-partition request queue,
+   * Durable Object, or equivalent coordinator; the default fails closed.
+   */
+  readonly pushApplySerialized?: boolean;
+  /**
+   * @deprecated Use `pushApplySerialized`. This alias remains valid only
+   * because the old assertion already promised that every partition write,
+   * not merely validator callbacks, was externally serialized.
    */
   readonly commitValidationSerialized?: boolean;
 }
 
 export class D1ServerStorage implements ServerStorage {
   readonly #db: D1Database;
-  readonly #commitValidationSerialized: boolean;
+  readonly #pushApplySerialized: boolean;
   /** Set by `ensureSchema`: app-table lookup for the relational row store. */
   #tables: ReadonlyMap<string, CompiledTable> | undefined;
   #schemaVersion: number | undefined;
 
   constructor(db: D1Database, options: D1ServerStorageOptions = {}) {
     this.#db = db;
-    this.#commitValidationSerialized =
+    this.#pushApplySerialized =
+      options.pushApplySerialized === true ||
       options.commitValidationSerialized === true;
   }
 
@@ -769,7 +780,7 @@ export class D1ServerStorage implements ServerStorage {
       this.#db,
       partition,
       (name) => this.table(name),
-      this.#commitValidationSerialized,
+      this.#pushApplySerialized,
     );
   }
 

package/src/postgres-storage.ts

@@ -439,7 +439,7 @@ class PostgresTransaction implements StorageTransaction {
   #partition: string;
   #resolveTable: (name: string) => CompiledTable;
   #open = true;
-  #commitValidationSavepoint = false;
+  #pushApplySavepoint = false;
   /** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
   #resolve: () => void;
   #reject: (error: unknown) => void;
@@ -521,7 +521,7 @@ class PostgresTransaction implements StorageTransaction {
     );
   }
 
-  async lockPartitionForCommitValidation(): Promise<void> {
+  async lockPartitionForPush(): Promise<void> {
     this.#assertOpen();
     await this.#client.query(
       `INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
@@ -532,8 +532,8 @@ class PostgresTransaction implements StorageTransaction {
       'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
       [this.#partition],
     );
-    await this.#client.query('SAVEPOINT syncular_commit_validation_candidate');
-    this.#commitValidationSavepoint = true;
+    await this.#client.query('SAVEPOINT syncular_push_candidate');
+    this.#pushApplySavepoint = true;
   }
 
   async commitRejectedPushResult(
@@ -542,18 +542,12 @@ class PostgresTransaction implements StorageTransaction {
     result: StoredPushResult,
   ): Promise<void> {
     this.#assertOpen();
-    if (!this.#commitValidationSavepoint) {
-      throw new Error(
-        'whole-commit rejection requires its validation savepoint',
-      );
+    if (!this.#pushApplySavepoint) {
+      throw new Error('push rejection requires its apply savepoint');
     }
-    await this.#client.query(
-      'ROLLBACK TO SAVEPOINT syncular_commit_validation_candidate',
-    );
-    await this.#client.query(
-      'RELEASE SAVEPOINT syncular_commit_validation_candidate',
-    );
-    this.#commitValidationSavepoint = false;
+    await this.#client.query('ROLLBACK TO SAVEPOINT syncular_push_candidate');
+    await this.#client.query('RELEASE SAVEPOINT syncular_push_candidate');
+    this.#pushApplySavepoint = false;
     await this.putPushResult(clientId, clientCommitId, result);
     await this.commit();
   }

package/src/push.ts

@@ -806,35 +806,6 @@ function idempotencyCacheMissFrame(
   };
 }
 
-async function persistRejectedPushResult(
-  storage: SyncRequestContext['storage'],
-  partition: string,
-  clientId: string,
-  clientCommitId: string,
-  stored: StoredPushResult,
-): Promise<{ readonly stored: StoredPushResult; readonly replayed: boolean }> {
-  const rejectionTx = await storage.begin(partition);
-  try {
-    await rejectionTx.putPushResult(clientId, clientCommitId, stored);
-    await rejectionTx.commit();
-  } catch (error) {
-    await rejectionTx.rollback();
-    throw error;
-  }
-  const canonical = await storage.getPushResult(
-    partition,
-    clientId,
-    clientCommitId,
-  );
-  if (canonical === undefined) {
-    throw new Error('push rejection finalization did not persist an outcome');
-  }
-  return {
-    stored: canonical,
-    replayed: canonical.cacheIdentity !== stored.cacheIdentity,
-  };
-}
-
 export interface AppliedCommitEvent {
   readonly commit: StoredCommit;
 }
@@ -899,48 +870,49 @@ export async function processPushCommitWithTrace(
   const validators = ctx.validators;
   const commitValidator = ctx.commitValidator;
   const tx = await storage.begin(partition);
+  const lockPartitionForPush =
+    tx.lockPartitionForPush?.bind(tx) ??
+    tx.lockPartitionForCommitValidation?.bind(tx);
   const commitRejectedPushResult = tx.commitRejectedPushResult?.bind(tx);
   try {
-    if (commitValidator !== undefined) {
-      if (
-        tx.lockPartitionForCommitValidation === undefined ||
-        commitRejectedPushResult === undefined
-      ) {
-        throw new Error(
-          'storage transaction does not support atomic whole-commit validation finalization',
-        );
-      }
-      await tx.lockPartitionForCommitValidation();
-      // The optimistic lookup above may have raced another request for the
-      // same idempotency key. Re-check after acquiring partition serialization
-      // so a concurrent duplicate never reruns the aggregate validator.
-      try {
-        const serializedPersisted = await storage.getPushResult(
-          partition,
-          clientId,
+    if (
+      lockPartitionForPush === undefined ||
+      commitRejectedPushResult === undefined
+    ) {
+      throw new Error(
+        'storage transaction does not support serialized push apply and atomic rejection finalization',
+      );
+    }
+    await lockPartitionForPush();
+    // The optimistic lookup above may have raced another delivery. Re-check
+    // only after acquiring partition serialization and before any operation
+    // read, validation, merge, or staged write.
+    try {
+      const serializedPersisted = await storage.getPushResult(
+        partition,
+        clientId,
+        frame.clientCommitId,
+      );
+      if (serializedPersisted !== undefined) {
+        await tx.rollback();
+        return processedPushCommit(
           frame.clientCommitId,
+          serializedPersisted,
+          true,
         );
-        if (serializedPersisted !== undefined) {
-          await tx.rollback();
-          return processedPushCommit(
-            frame.clientCommitId,
-            serializedPersisted,
-            true,
-          );
-        }
-      } catch (error) {
-        if (
-          error instanceof SyncError &&
-          error.code === 'sync.idempotency_cache_miss'
-        ) {
-          await tx.rollback();
-          return {
-            frame: idempotencyCacheMissFrame(frame.clientCommitId, error),
-            replayed: false,
-          };
-        }
-        throw error;
       }
+    } catch (error) {
+      if (
+        error instanceof SyncError &&
+        error.code === 'sync.idempotency_cache_miss'
+      ) {
+        await tx.rollback();
+        return {
+          frame: idempotencyCacheMissFrame(frame.clientCommitId, error),
+          replayed: false,
+        };
+      }
+      throw error;
     }
     const results: PushOperationResult[] = [];
     const changes: NewChange[] = [];
@@ -993,32 +965,24 @@ export async function processPushCommitWithTrace(
         status: 'rejected',
         results: [terminated],
       });
-      if (commitValidator !== undefined) {
-        // Discard candidate rows and persist the rejection while retaining the
-        // same partition lock. This closes the duplicate-request race between
-        // rollback and the durable idempotency outcome.
-        if (commitRejectedPushResult === undefined) {
-          throw new Error(
-            'storage transaction lost whole-commit rejection finalization support',
-          );
-        }
-        await commitRejectedPushResult(clientId, frame.clientCommitId, stored);
-      } else {
-        await tx.rollback();
-        const canonical = await persistRejectedPushResult(
-          storage,
-          partition,
-          clientId,
-          frame.clientCommitId,
-          stored,
-        );
-        return processedPushCommit(
-          frame.clientCommitId,
-          canonical.stored,
-          canonical.replayed,
+      // Discard candidates and persist the rejection while retaining the same
+      // partition lock. There is no unlock gap in which a duplicate can rerun.
+      await commitRejectedPushResult(clientId, frame.clientCommitId, stored);
+      const canonical = await storage.getPushResult(
+        partition,
+        clientId,
+        frame.clientCommitId,
+      );
+      if (canonical === undefined) {
+        throw new Error(
+          'push rejection finalization did not persist an outcome',
         );
       }
-      return processedPushCommit(frame.clientCommitId, stored, false);
+      return processedPushCommit(
+        frame.clientCommitId,
+        canonical,
+        canonical.cacheIdentity !== stored.cacheIdentity,
+      );
     }
 
     const commitSeq = await tx.appendCommit({
@@ -1045,7 +1009,6 @@ export async function processPushCommitWithTrace(
     }
     return processedPushCommit(frame.clientCommitId, stored, false);
   } catch (error) {
-    await tx.rollback();
     if (error instanceof StorageConstraintError) {
       const stored = newStoredPushResult(createdAtMs, {
         status: 'rejected',
@@ -1059,19 +1022,30 @@ export async function processPushCommitWithTrace(
           },
         ],
       });
-      const canonical = await persistRejectedPushResult(
-        storage,
+      if (commitRejectedPushResult === undefined) {
+        await tx.rollback();
+        throw new Error(
+          'storage transaction lost atomic push rejection finalization support',
+        );
+      }
+      await commitRejectedPushResult(clientId, frame.clientCommitId, stored);
+      const canonical = await storage.getPushResult(
         partition,
         clientId,
         frame.clientCommitId,
-        stored,
       );
+      if (canonical === undefined) {
+        throw new Error(
+          'push rejection finalization did not persist an outcome',
+        );
+      }
       return processedPushCommit(
         frame.clientCommitId,
-        canonical.stored,
-        canonical.replayed,
+        canonical,
+        canonical.cacheIdentity !== stored.cacheIdentity,
       );
     }
+    await tx.rollback();
     throw error;
   }
 }

package/src/realtime.ts

@@ -30,53 +30,26 @@ import {
   type ScopeMap,
   type WakeReason,
 } from '@syncular/core';
-import type {
-  LeaseConfig,
-  ResolveScopes,
-  ServerLimits,
-  SyncRequestContext,
-} from './context';
+import type { SyncRequestContext, SyncServerConfig } from './context';
 import { RESOLVER_OUTAGE } from './context';
 import { SyncError, syncError } from './errors';
 import { emitEvent, type SyncularServerEvents } from './events';
 import { createSyncResponseStream } from './handler';
-import type { ServerSchema } from './schema';
 import { type CompiledSchema, compileSchema } from './schema';
 import {
   computeEffective,
   matchesEffective,
   type ResolvedScopes,
 } from './scopes';
-import type { SegmentStore } from './segment-store';
-import type { SegmentUrlConfig } from './signed-url';
 import type { ServerStorage, StoredCommit } from './storage';
-import type { CommitValidator, ValidatorRegistry } from './validate';
 
-export interface RealtimeHubConfig {
-  readonly schema: ServerSchema;
-  readonly storage: ServerStorage;
-  readonly resolveScopes: ResolveScopes;
-  /** §6.7 validators used by sync rounds carried over this socket. */
-  readonly validators?: ValidatorRegistry;
-  /** §6.8 whole-commit validator shared with HTTP sync rounds. */
-  readonly commitValidator?: CommitValidator;
-  readonly clock?: () => number;
+/**
+ * Realtime adds fanout/presence tuning to the canonical sync-server config;
+ * socket rounds must never have a narrower push/pull capability set than HTTP.
+ */
+export interface RealtimeHubConfig extends Omit<SyncServerConfig, 'realtime'> {
   /** Deltas larger than this become `delta-too-large` wake-ups (§8.2). */
   readonly maxDeltaBytes?: number;
-  /** Optional structured-events sink (`realtime.*` events). */
-  readonly events?: SyncularServerEvents;
-  /**
-   * Segment store for sync rounds over the socket (§8.7). Without it a
-   * socket round fails loudly with an in-band ERROR — provide the same
-   * store the HTTP binding uses (one handler, two framings).
-   */
-  readonly segments?: SegmentStore;
-  /** Request limits for socket rounds; defaults match the HTTP binding. */
-  readonly limits?: Partial<ServerLimits>;
-  readonly signedUrls?: SegmentUrlConfig;
-  /** §7.3 auth leases for socket sync rounds (§8.7) — same config the
-   * HTTP binding uses, so rounds over the socket are lease-aware too. */
-  readonly leases?: LeaseConfig;
   /**
    * §8.6 presence: cap on the serialized size (bytes) of a published
    * presence document. An over-cap publish is rejected loudly to the
@@ -494,9 +467,11 @@ export class RealtimeSession {
    * round's request byte stream (§8.7). Synchronous entry — assembly and
    * violation detection happen inline so a pipelined chunk arriving
    * while a response streams is caught deterministically; the round
-   * itself runs async once the request is complete.
+   * itself runs async once the request is complete. The returned promise, when
+   * present, resolves only after response streaming and registration refresh;
+   * coordinated hosts await it to retain their partition FIFO through commit.
    */
-  handleBinary(bytes: Uint8Array): void {
+  handleBinary(bytes: Uint8Array): Promise<void> | undefined {
     if (bytes.length === 0) return;
     if (bytes[0] !== REALTIME_TAG_ROUND) {
       // §8.7: client→server tags other than 0x01 — a broken client.
@@ -531,7 +506,7 @@ export class RealtimeSession {
     }
     const token = Symbol('round');
     this.#activeRound = token;
-    void this.#runRound(done.message.slice(), token);
+    return this.#runRound(done.message.slice(), token);
   }
 
   /** Drive the shared handler and stream the response back (§8.7). */
@@ -914,7 +889,10 @@ export class RealtimeHub {
    * the same shape the HTTP adapter builds, so the round drives the
    * SAME handler with zero semantic divergence.
    */
-  requestContext(session: RealtimeSession): SyncRequestContext {
+  requestContextFor(identity: {
+    readonly partition: string;
+    readonly actorId: string;
+  }): SyncRequestContext {
     const segments = this.#config.segments;
     if (segments === undefined) {
       // Fail loud (§8.7): a hub serving socket rounds needs the same
@@ -925,12 +903,21 @@ export class RealtimeHub {
       );
     }
     return {
-      partition: session.partition,
-      actorId: session.actorId,
+      partition: identity.partition,
+      actorId: identity.actorId,
       schema: this.#config.schema,
       storage: this.#config.storage,
       segments,
       resolveScopes: this.#config.resolveScopes,
+      ...(this.#config.blobs !== undefined
+        ? { blobs: this.#config.blobs }
+        : {}),
+      ...(this.#config.maxBlobBytes !== undefined
+        ? { maxBlobBytes: this.#config.maxBlobBytes }
+        : {}),
+      ...(this.#config.crdtMergers !== undefined
+        ? { crdtMergers: this.#config.crdtMergers }
+        : {}),
       ...(this.#config.validators !== undefined
         ? { validators: this.#config.validators }
         : {}),
@@ -946,6 +933,15 @@ export class RealtimeHub {
       ...(this.#config.signedUrls !== undefined
         ? { signedUrls: this.#config.signedUrls }
         : {}),
+      ...(this.#config.blobSignedUrls !== undefined
+        ? { blobSignedUrls: this.#config.blobSignedUrls }
+        : {}),
+      ...(this.#config.blobUploadUrls !== undefined
+        ? { blobUploadUrls: this.#config.blobUploadUrls }
+        : {}),
+      ...(this.#config.sqliteImageBuilder !== undefined
+        ? { sqliteImageBuilder: this.#config.sqliteImageBuilder }
+        : {}),
       ...(this.#config.leases !== undefined
         ? { leases: this.#config.leases }
         : {}),
@@ -956,6 +952,10 @@ export class RealtimeHub {
     };
   }
 
+  requestContext(session: RealtimeSession): SyncRequestContext {
+    return this.requestContextFor(session);
+  }
+
   /**
    * Register a connected socket (§8.1): load the client's last pull's
    * subscription list, resolve + intersect scopes, send `hello`.