@syncular/server 0.32.0 → 0.33.0

package/src/handler.ts

@@ -526,11 +526,12 @@ async function* streamResponse(
   }
   try {
     // Push half (§6): one PUSH_RESULT per PUSH_COMMIT, in request order.
+    const currentSchema = compileSchema(ctx.schema);
     let acceptedThrough = 0;
     for (const push of plan.pushes) {
       const processed = await processPushCommitWithTrace(
         ctx,
-        compileSchema(ctx.schema),
+        currentSchema,
         plan.resolved,
         plan.header.clientId,
         push,
@@ -543,20 +544,36 @@ async function* streamResponse(
       const frame = {
         ...processed.frame,
         results: processed.frame.results.map((result) => {
-          if (result.status !== 'conflict') return result;
-          const operation = push.operations[result.opIndex]!;
+          if (result.status !== 'conflict' || schema === currentSchema) {
+            return result;
+          }
+          const operation = push.operations[result.opIndex];
+          // §2.3: a replay returns the persisted result without rebuilding
+          // operations, so a reused clientCommitId with changed operations
+          // leaves no table to project the stored conflict row through.
+          if (operation === undefined) {
+            throw syncError(
+              'sync.invalid_request',
+              'replayed clientCommitId carries changed operations',
+              JSON.stringify({
+                clientCommitId: push.clientCommitId,
+                opIndex: result.opIndex,
+              }),
+            );
+          }
           const target = schema.tables.get(operation.table)!;
-          const current = compileSchema(ctx.schema).tables.get(
-            operation.table,
-          )!;
+          const bitmapBytes = Math.ceil(target.columns.length / 8);
           return {
             ...result,
-            serverRow: projectRowPayload(current, target, result.serverRow),
+            serverRow: projectRowPayload(
+              currentSchema.tables.get(operation.table)!,
+              target,
+              result.serverRow,
+            ),
             conflictColumns: result.conflictColumns
-              .slice(0, Math.ceil(target.columns.length / 8))
+              .slice(0, bitmapBytes)
               .map((byte, i) =>
-                i === Math.ceil(target.columns.length / 8) - 1 &&
-                target.columns.length % 8 !== 0
+                i === bitmapBytes - 1 && target.columns.length % 8 !== 0
                   ? byte & ((1 << (target.columns.length % 8)) - 1)
                   : byte,
               ),
@@ -700,6 +717,17 @@ async function* streamResponse(
       updatedAtMs: clockOf(ctx)(),
       subscriptions,
     });
+    if (
+      ctx.realtime?.notifySubscriptions !== undefined &&
+      JSON.stringify(
+        (previous?.subscriptions ?? []).map((s) => [s.id, s.table, s.scopes]),
+      ) !== JSON.stringify(subscriptions.map((s) => [s.id, s.table, s.scopes]))
+    ) {
+      await ctx.realtime.notifySubscriptions(
+        ctx.partition,
+        plan.header.clientId,
+      );
+    }
   } catch (error) {
     if (error instanceof SyncError) {
       if (report !== undefined) {

package/src/index.ts

@@ -11,6 +11,8 @@ export * from './admin';
 export * from './authoritative-query';
 export * from './blob-handlers';
 export * from './blob-store';
+export * from './bucket-blob-store';
+export * from './bucket-segment-store';
 export * from './content-encoding';
 export * from './context';
 export * from './crdt-merger';
@@ -65,8 +67,10 @@ export {
 } from './reactions';
 export * from './realtime';
 export * from './restore';
+export * from './object-bucket';
 export * from './relational-rows';
 export * from './s3-blob-store';
+export * from './s3-object-bucket';
 export * from './s3-segment-store';
 export * from './schema';
 export * from './scopes';

package/src/object-bucket.ts

@@ -0,0 +1,83 @@
+/**
+ * The object-bucket seam: the handful of object-store operations the durable
+ * segment store (§5.1) and blob store (§5.9.2) actually perform, expressed
+ * without an HTTP transport.
+ *
+ * Two adapters implement it — `S3ObjectBucket` (`s3-object-bucket.ts`,
+ * hand-rolled SigV4 over `fetch`) and the R2 binding adapter in
+ * `@syncular/server-workers` (`r2-stores.ts`, the Worker's `env.BUCKET`) — so
+ * one implementation of the store logic (`bucket-segment-store.ts`,
+ * `bucket-blob-store.ts`) runs over both. The key layouts are the same on
+ * either transport, so a bucket can be served through the S3 API for one host
+ * and the binding for another.
+ *
+ * Everything here is structural: an adapter may be any object with these
+ * methods, and Cloudflare's `R2Bucket` is assignable to the declaration in
+ * `r2-stores.ts` without a dependency on `@cloudflare/workers-types`.
+ */
+
+/** One stored object, readable once. Mirrors `R2ObjectBody`. */
+export interface BucketObject {
+  /** The body, streamed; `null` only if the backend served none. */
+  readonly body: ReadableStream<Uint8Array<ArrayBuffer>> | null;
+  /** Backend ETag of the stored object (opaque, used as a CAS token). */
+  readonly etag: string | undefined;
+  /** Stored size in bytes, when the backend reports one. */
+  readonly size: number | undefined;
+  /** User metadata: the `x-amz-meta-*` headers S3-side, `customMetadata` on R2. */
+  readonly metadata: Readonly<Record<string, string>>;
+  arrayBuffer(): Promise<ArrayBuffer>;
+  text(): Promise<string>;
+}
+
+/** One stored object's description without its body. Mirrors `R2Object`. */
+export interface BucketObjectHead {
+  readonly etag: string | undefined;
+  readonly size: number | undefined;
+  readonly metadata: Readonly<Record<string, string>>;
+}
+
+export interface BucketWriteOptions {
+  readonly body: Uint8Array;
+  readonly contentType: string;
+  /** User metadata; values must be ASCII (both backends require it). */
+  readonly metadata?: Readonly<Record<string, string>> | undefined;
+  /**
+   * Create-only write (`If-None-Match: *`, R2 `etagDoesNotMatch: '*'`). The
+   * content-addressed bytes object is written this way so a second publisher
+   * of identical bytes is a no-op and a hash collision cannot overwrite.
+   */
+  readonly createOnly?: boolean | undefined;
+  /**
+   * Compare-and-swap write (`If-Match`, R2 `etagMatches`): the write lands
+   * only if the stored ETag still equals this. The read-modify-write of the
+   * segment record and of the stats accumulator depend on it.
+   */
+  readonly ifMatch?: string | undefined;
+}
+
+/** One page of keys under a prefix. `cursor` is set when more remain. */
+export interface BucketListPage {
+  readonly keys: readonly string[];
+  readonly cursor: string | undefined;
+}
+
+export interface ObjectBucket {
+  /** The object plus its bytes, or `undefined` when the key is absent. */
+  get(key: string): Promise<BucketObject | undefined>;
+  head(key: string): Promise<BucketObjectHead | undefined>;
+  /**
+   * Write one object. Returns `false` when the `createOnly` / `ifMatch`
+   * precondition failed, so the caller can re-read and retry; a backend that
+   * ignores conditional writes would instead silently lose a concurrent
+   * update, which is why the return value is not optional.
+   */
+  put(key: string, options: BucketWriteOptions): Promise<boolean>;
+  /** Delete one object. Deleting an absent key succeeds (idempotent). */
+  delete(key: string): Promise<void>;
+  /**
+   * List keys under a prefix, one page at a time. The blob orphan sweep
+   * (§5.9.2) is the only caller: everything on the hot path is a GET/HEAD.
+   */
+  list(prefix: string, cursor?: string): Promise<BucketListPage>;
+}

package/src/operations-realtime.ts

@@ -271,5 +271,12 @@ export function composeRealtimeNotifiers(
         notifiers.map((notifier) => notifier.notifyCommit(partition, commit)),
       );
     },
+    notifySubscriptions: async (partition, clientId) => {
+      await Promise.all(
+        notifiers.map((notifier) =>
+          notifier.notifySubscriptions?.(partition, clientId),
+        ),
+      );
+    },
   };
 }

package/src/operations.ts

@@ -21,7 +21,11 @@ import {
 import { reportError, SyncError, syncError } from './errors';
 import { processPushOperationsWithTrace } from './push';
 import { compileSchema } from './schema';
-import { authorizeWrite, type ResolvedScopes } from './scopes';
+import {
+  authorizeWrite,
+  renderScopeValue,
+  type ResolvedScopes,
+} from './scopes';
 import type {
   AuthoritativeQueryRequest,
   AuthoritativeQueryResult,
@@ -534,11 +538,11 @@ function commandOperations(
       }
       return mutation.values[column.name] ?? null;
     });
-    const rowId = values[table.primaryKeyIndex];
-    if (typeof rowId !== 'string' || rowId.length === 0) {
+    const rowId = renderScopeValue(values[table.primaryKeyIndex]);
+    if (rowId === undefined) {
       throw syncError(
         'operation.invalid_request',
-        'command upsert requires a non-empty string primary key',
+        'command upsert requires a non-empty primary key',
       );
     }
     return {

package/src/push.ts

@@ -503,6 +503,7 @@ async function applyOperation(
         if (
           present[i] &&
           !crdtIndexes.has(i) &&
+          i !== table.primaryKeyIndex &&
           effectiveVersion(i) > op.baseVersion
         ) {
           marked.push(i);
@@ -978,7 +979,20 @@ async function enforceReferences(
   const operations: ValidateCommitOperation[] = [];
   const originByIndex: number[] = [];
 
+  const finalByRow = new Map<
+    string,
+    { readonly first: StagedWrite; readonly last: StagedWrite }
+  >();
   for (const write of staged) {
+    const rowKey = `${write.table.name}\u0000${write.rowId}`;
+    const existing = finalByRow.get(rowKey);
+    finalByRow.set(rowKey, {
+      first: existing?.first ?? write,
+      last: write,
+    });
+  }
+
+  for (const { last: write } of finalByRow.values()) {
     if (write.op !== 'upsert' || write.values === undefined) continue;
     for (const reference of write.table.references) {
       const value = write.values[reference.columnIndex];
@@ -1028,9 +1042,9 @@ async function enforceReferences(
     readonly rowId: string;
     readonly originOpIndex: number;
   }> = [];
-  for (const write of staged) {
-    if (write.op === 'delete' && write.stored !== undefined) {
-      schedule(write.table, write.rowId, write.opIndex, scheduled, queue);
+  for (const { first, last } of finalByRow.values()) {
+    if (last.op === 'delete' && first.stored !== undefined) {
+      schedule(last.table, last.rowId, last.opIndex, scheduled, queue);
     }
   }
 
@@ -1040,6 +1054,13 @@ async function enforceReferences(
     const key = `${item.table.name}\u0000${item.rowId}`;
     if (processed.has(key)) continue;
     processed.add(key);
+    const parentPkColumn = item.table.columns[item.table.primaryKeyIndex];
+    const parentKeyValue: RowValue =
+      parentPkColumn?.type === 'integer'
+        ? Number(item.rowId)
+        : parentPkColumn?.type === 'boolean'
+          ? item.rowId === 'true'
+          : item.rowId;
     for (const reference of item.table.referencedBy) {
       if (tx.scanRowsByIndex === undefined) {
         throw new Error(
@@ -1052,7 +1073,7 @@ async function enforceReferences(
         const page = await tx.scanRowsByIndex({
           table: reference.table,
           index: reference.index,
-          values: [item.rowId],
+          values: [parentKeyValue],
           afterRowId,
           limit: 1_000,
         });
@@ -1081,6 +1102,21 @@ async function enforceReferences(
         );
       }
       for (const childRow of children) {
+        if (reference.onDelete === 'CASCADE') {
+          if (
+            !schedule(
+              child,
+              childRow.rowId,
+              item.originOpIndex,
+              scheduled,
+              queue,
+            )
+          ) {
+            continue;
+          }
+        } else if (scheduled.has(`${child.name}\u0000${childRow.rowId}`)) {
+          continue;
+        }
         if (originByIndex.length >= cascadeLimit) {
           return {
             kind: 'terminate',
@@ -1091,11 +1127,6 @@ async function enforceReferences(
             ),
           };
         }
-        if (
-          !schedule(child, childRow.rowId, item.originOpIndex, scheduled, queue)
-        ) {
-          continue;
-        }
         const syntheticIndex = clientOpCount + originByIndex.length;
         const storedValues = decodeRow(child.columns, childRow.payload);
         if (reference.onDelete === 'CASCADE') {

package/src/realtime.ts

@@ -23,6 +23,7 @@ import {
   encodeMessage,
   encodePresenceError,
   encodePresenceFanout,
+  isSupportedProtocolWireVersion,
   MessageStreamScanner,
   PROTOCOL_WIRE_VERSION,
   type PresenceKind,
@@ -278,6 +279,8 @@ export class RealtimeSession {
   #registrationGeneration = 0;
   /** Generation of the most recent refresh whose result was applied. */
   #registrationApplied = 0;
+  /** Highest sequence that required a catch-up wake for this session. */
+  #wakeRequiredThroughSeq: number;
 
   constructor(
     hub: RealtimeHub,
@@ -301,6 +304,7 @@ export class RealtimeSession {
     this.cursor = cursor;
     this.lastKnownSeq = latestSeq;
     this.wakePending = cursor < latestSeq;
+    this.#wakeRequiredThroughSeq = cursor < latestSeq ? latestSeq : 0;
     this.registrations = registrations;
     this.openedAtMs = clock();
     this.#send = options.send;
@@ -336,9 +340,9 @@ export class RealtimeSession {
     // proves those commits no longer need socket notifications, so the next
     // notification must be adjacent to the acknowledged cursor.
     this.lastKnownSeq = Math.max(this.lastKnownSeq, this.cursor);
-    if (this.cursor >= this.lastKnownSeq) this.wakePending = false;
+    if (this.cursor >= this.#wakeRequiredThroughSeq) this.wakePending = false;
     // §8.2: acks update the client cursor record without an HTTP pull.
-    this.#persistCursor();
+    this.#persistCursor(cursor);
   }
 
   /**
@@ -571,7 +575,7 @@ export class RealtimeSession {
     try {
       const requestedWireVersion =
         (requestBytes[4] ?? 0) | ((requestBytes[5] ?? 0) << 8);
-      if (requestedWireVersion === 1 || requestedWireVersion === 2) {
+      if (isSupportedProtocolWireVersion(requestedWireVersion)) {
         this.wireVersion = requestedWireVersion;
       }
       let stream: AsyncIterable<Uint8Array> | undefined;
@@ -688,6 +692,15 @@ export class RealtimeSession {
    * deltas immediately; it also survives a newer round-end refresh that
    * failed, because that round applied nothing.
    */
+  /**
+   * A pull outside this connection changed the client's subscriptions. A
+   * round in flight on the connection reloads them at its end anyway.
+   */
+  async refreshAfterPull(): Promise<void> {
+    if (this.#activeRound !== undefined) return;
+    await this.refreshRegistrations(false);
+  }
+
   async refreshRegistrations(failClosed: boolean): Promise<void> {
     const generation = ++this.#registrationGeneration;
     let registrations: Registration[];
@@ -721,7 +734,7 @@ export class RealtimeSession {
     this.#hub.disconnect(this);
   }
 
-  #persistCursor(): void {
+  #persistCursor(cursor: number): void {
     // §8.2: start the write now and keep it on the control tail so `drain()`
     // can await it. Two acks may write concurrently: storage folds them with
     // `MAX(cursor, ?)`, so neither order loses the later cursor.
@@ -731,7 +744,7 @@ export class RealtimeSession {
         this.clientId,
         this.actorId,
         this.logEpoch,
-        this.cursor,
+        cursor,
         this.#clock(),
       );
     })();
@@ -767,6 +780,10 @@ export class RealtimeSession {
 
   sendWake(reason: WakeReason): void {
     this.wakePending = true;
+    this.#wakeRequiredThroughSeq = Math.max(
+      this.#wakeRequiredThroughSeq,
+      this.lastKnownSeq,
+    );
     this.#sendSafe(
       JSON.stringify({
         event: 'sync',
@@ -1096,58 +1113,13 @@ export class RealtimeHub {
         'socket sync rounds require a segment store on the realtime hub (§8.7)',
       );
     }
+    // The whole config, so a new `SyncServerConfig` field reaches socket and
+    // hub-served HTTP rounds without being listed here.
     return {
+      ...this.#config,
+      segments,
       partition: identity.partition,
       actorId: identity.actorId,
-      schema: this.#config.schema,
-      ...(this.#config.schemaWindow !== undefined
-        ? { schemaWindow: this.#config.schemaWindow }
-        : {}),
-      storage: this.#config.storage,
-      segments,
-      resolveScopes: this.#config.resolveScopes,
-      ...(this.#config.blobs !== undefined
-        ? { blobs: this.#config.blobs }
-        : {}),
-      ...(this.#config.maxBlobBytes !== undefined
-        ? { maxBlobBytes: this.#config.maxBlobBytes }
-        : {}),
-      ...(this.#config.crdtMergers !== undefined
-        ? { crdtMergers: this.#config.crdtMergers }
-        : {}),
-      ...(this.#config.validators !== undefined
-        ? { validators: this.#config.validators }
-        : {}),
-      ...(this.#config.commitValidator !== undefined
-        ? { commitValidator: this.#config.commitValidator }
-        : {}),
-      ...(this.#config.clock !== undefined
-        ? { clock: this.#config.clock }
-        : {}),
-      ...(this.#config.limits !== undefined
-        ? { limits: this.#config.limits }
-        : {}),
-      ...(this.#config.signedUrls !== undefined
-        ? { signedUrls: this.#config.signedUrls }
-        : {}),
-      ...(this.#config.blobSignedUrls !== undefined
-        ? { blobSignedUrls: this.#config.blobSignedUrls }
-        : {}),
-      ...(this.#config.blobUploadUrls !== undefined
-        ? { blobUploadUrls: this.#config.blobUploadUrls }
-        : {}),
-      ...(this.#config.sqliteImageBuilder !== undefined
-        ? { sqliteImageBuilder: this.#config.sqliteImageBuilder }
-        : {}),
-      ...(this.#config.leases !== undefined
-        ? { leases: this.#config.leases }
-        : {}),
-      ...(this.#config.events !== undefined
-        ? { events: this.#config.events }
-        : {}),
-      ...(this.#config.onError !== undefined
-        ? { onError: this.#config.onError }
-        : {}),
       realtime: this,
     };
   }
@@ -1211,7 +1183,9 @@ export class RealtimeHub {
       this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES,
       storage,
       registry.logEpoch,
-      record?.wireVersion ?? PROTOCOL_WIRE_VERSION,
+      record !== undefined && isSupportedProtocolWireVersion(record.wireVersion)
+        ? record.wireVersion
+        : PROTOCOL_WIRE_VERSION,
       this.#config.events,
     );
     this.#sessions.add(session);
@@ -1292,6 +1266,22 @@ export class RealtimeHub {
     }
   }
 
+  /**
+   * RealtimeNotifier: a pull changed a client's subscription list (§8.7).
+   * Its sockets reload their registrations with the round-end semantics, so
+   * an unreadable record keeps the previous registrations.
+   */
+  async notifySubscriptions(
+    partition: string,
+    clientId: string,
+  ): Promise<void> {
+    for (const session of [...this.#sessions]) {
+      if (session.partition !== partition || session.clientId !== clientId)
+        continue;
+      await session.refreshAfterPull();
+    }
+  }
+
   /**
    * Host-initiated, fail-closed registration refresh (§8.7). A host calls this
    * after it changes an actor's membership or connection state, so the