@syncular/server 0.32.1 → 0.34.0

package/src/object-bucket.ts

@@ -0,0 +1,83 @@
+/**
+ * The object-bucket seam: the handful of object-store operations the durable
+ * segment store (§5.1) and blob store (§5.9.2) actually perform, expressed
+ * without an HTTP transport.
+ *
+ * Two adapters implement it — `S3ObjectBucket` (`s3-object-bucket.ts`,
+ * hand-rolled SigV4 over `fetch`) and the R2 binding adapter in
+ * `@syncular/server-workers` (`r2-stores.ts`, the Worker's `env.BUCKET`) — so
+ * one implementation of the store logic (`bucket-segment-store.ts`,
+ * `bucket-blob-store.ts`) runs over both. The key layouts are the same on
+ * either transport, so a bucket can be served through the S3 API for one host
+ * and the binding for another.
+ *
+ * Everything here is structural: an adapter may be any object with these
+ * methods, and Cloudflare's `R2Bucket` is assignable to the declaration in
+ * `r2-stores.ts` without a dependency on `@cloudflare/workers-types`.
+ */
+
+/** One stored object, readable once. Mirrors `R2ObjectBody`. */
+export interface BucketObject {
+  /** The body, streamed; `null` only if the backend served none. */
+  readonly body: ReadableStream<Uint8Array<ArrayBuffer>> | null;
+  /** Backend ETag of the stored object (opaque, used as a CAS token). */
+  readonly etag: string | undefined;
+  /** Stored size in bytes, when the backend reports one. */
+  readonly size: number | undefined;
+  /** User metadata: the `x-amz-meta-*` headers S3-side, `customMetadata` on R2. */
+  readonly metadata: Readonly<Record<string, string>>;
+  arrayBuffer(): Promise<ArrayBuffer>;
+  text(): Promise<string>;
+}
+
+/** One stored object's description without its body. Mirrors `R2Object`. */
+export interface BucketObjectHead {
+  readonly etag: string | undefined;
+  readonly size: number | undefined;
+  readonly metadata: Readonly<Record<string, string>>;
+}
+
+export interface BucketWriteOptions {
+  readonly body: Uint8Array;
+  readonly contentType: string;
+  /** User metadata; values must be ASCII (both backends require it). */
+  readonly metadata?: Readonly<Record<string, string>> | undefined;
+  /**
+   * Create-only write (`If-None-Match: *`, R2 `etagDoesNotMatch: '*'`). The
+   * content-addressed bytes object is written this way so a second publisher
+   * of identical bytes is a no-op and a hash collision cannot overwrite.
+   */
+  readonly createOnly?: boolean | undefined;
+  /**
+   * Compare-and-swap write (`If-Match`, R2 `etagMatches`): the write lands
+   * only if the stored ETag still equals this. The read-modify-write of the
+   * segment record and of the stats accumulator depend on it.
+   */
+  readonly ifMatch?: string | undefined;
+}
+
+/** One page of keys under a prefix. `cursor` is set when more remain. */
+export interface BucketListPage {
+  readonly keys: readonly string[];
+  readonly cursor: string | undefined;
+}
+
+export interface ObjectBucket {
+  /** The object plus its bytes, or `undefined` when the key is absent. */
+  get(key: string): Promise<BucketObject | undefined>;
+  head(key: string): Promise<BucketObjectHead | undefined>;
+  /**
+   * Write one object. Returns `false` when the `createOnly` / `ifMatch`
+   * precondition failed, so the caller can re-read and retry; a backend that
+   * ignores conditional writes would instead silently lose a concurrent
+   * update, which is why the return value is not optional.
+   */
+  put(key: string, options: BucketWriteOptions): Promise<boolean>;
+  /** Delete one object. Deleting an absent key succeeds (idempotent). */
+  delete(key: string): Promise<void>;
+  /**
+   * List keys under a prefix, one page at a time. The blob orphan sweep
+   * (§5.9.2) is the only caller: everything on the hot path is a GET/HEAD.
+   */
+  list(prefix: string, cursor?: string): Promise<BucketListPage>;
+}

package/src/operations-realtime.ts

@@ -271,5 +271,12 @@ export function composeRealtimeNotifiers(
         notifiers.map((notifier) => notifier.notifyCommit(partition, commit)),
       );
     },
+    notifySubscriptions: async (partition, clientId) => {
+      await Promise.all(
+        notifiers.map((notifier) =>
+          notifier.notifySubscriptions?.(partition, clientId),
+        ),
+      );
+    },
   };
 }

package/src/operations.ts

@@ -8,11 +8,12 @@ import {
   encodeRemoteOperationResponse,
   encodeSparseRow,
   type PushOperation,
+  type RemoteOperationRequest,
   type RemoteOperationResponse,
   type RowValue,
   type ScopeMap,
 } from '@syncular/core';
-import type { SyncRequestContext } from './context';
+import type { ResolveScopes, SyncRequestContext } from './context';
 import {
   REMOTE_COMMAND_CLIENT_ID_PREFIX,
   RESOLVER_OUTAGE,
@@ -33,7 +34,11 @@ import type {
 } from './storage';
 import type { StorageTransaction } from './storage';
 import { transactionQuery } from './storage-query';
-import { toValidateRow, type ValidateRow } from './validate';
+import {
+  toValidateRow,
+  ValidationRejection,
+  type ValidateRow,
+} from './validate';
 
 export interface RemoteQueryDependency {
   readonly table: string;
@@ -683,10 +688,41 @@ export class RemoteOperationRegistry {
   }
 }
 
+/**
+ * The registry and identity one remote operation request runs with
+ * (REMOTE.md §5.1). `actorId` is the commit's actor and defaults to the
+ * authenticated actor; `resolveScopes` replaces the configured resolver for
+ * this request.
+ */
+export interface RemoteOperationBinding {
+  readonly registry: RemoteOperationRegistry;
+  readonly actorId?: string;
+  readonly resolveScopes?: ResolveScopes;
+}
+
+/**
+ * Chooses the binding of one decoded request (REMOTE.md §5.1). `ctx` carries
+ * the authenticated actor; the resolver runs after the client-id binding
+ * check and may throw a `SyncError` or a `ValidationRejection`.
+ */
+export type RemoteOperationResolver = (
+  request: RemoteOperationRequest,
+  ctx: SyncRequestContext,
+) => RemoteOperationBinding | Promise<RemoteOperationBinding>;
+
 function encodeRemoteOperationError(
   error: unknown,
   fallbackCode: 'operation.invalid_request' | 'operation.execution_failed',
 ): Uint8Array {
+  if (error instanceof ValidationRejection) {
+    return encodeRemoteOperationResponse({
+      revision: 1,
+      kind: 'error',
+      code: error.code,
+      message: error.message,
+      retryable: false,
+    });
+  }
   const sync =
     error instanceof SyncError
       ? error
@@ -707,8 +743,8 @@ function encodeRemoteOperationError(
 
 export async function handleRemoteOperation(
   bytes: Uint8Array,
-  ctx: SyncRequestContext,
-  registry: RemoteOperationRegistry,
+  authenticated: SyncRequestContext,
+  operations: RemoteOperationRegistry | RemoteOperationResolver,
 ): Promise<Uint8Array> {
   let request;
   try {
@@ -728,6 +764,7 @@ export async function handleRemoteOperation(
   } catch (error) {
     return encodeRemoteOperationError(error, 'operation.invalid_request');
   }
+  let ctx = authenticated;
   try {
     await touchAuthenticatedPartition(ctx);
     if (request.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
@@ -752,6 +789,16 @@ export async function handleRemoteOperation(
     ) {
       throw syncError('operation.invalid_request');
     }
+    let registry = operations;
+    if (typeof registry === 'function') {
+      const binding = await registry(request, ctx);
+      registry = binding.registry;
+      ctx = {
+        ...ctx,
+        actorId: binding.actorId ?? ctx.actorId,
+        resolveScopes: binding.resolveScopes ?? ctx.resolveScopes,
+      };
+    }
     const operation = registry.get(request.operationId);
     if (operation === undefined) {
       throw syncError('operation.unknown');
@@ -772,7 +819,7 @@ export async function handleRemoteOperation(
       ),
     );
   } catch (error) {
-    if (!(error instanceof SyncError)) {
+    if (!(error instanceof SyncError || error instanceof ValidationRejection)) {
       reportError(error, ctx.onError, 'operations');
     }
     return encodeRemoteOperationError(error, 'operation.execution_failed');

package/src/realtime.ts

@@ -692,6 +692,15 @@ export class RealtimeSession {
    * deltas immediately; it also survives a newer round-end refresh that
    * failed, because that round applied nothing.
    */
+  /**
+   * A pull outside this connection changed the client's subscriptions. A
+   * round in flight on the connection reloads them at its end anyway.
+   */
+  async refreshAfterPull(): Promise<void> {
+    if (this.#activeRound !== undefined) return;
+    await this.refreshRegistrations(false);
+  }
+
   async refreshRegistrations(failClosed: boolean): Promise<void> {
     const generation = ++this.#registrationGeneration;
     let registrations: Registration[];
@@ -1104,67 +1113,13 @@ export class RealtimeHub {
         'socket sync rounds require a segment store on the realtime hub (§8.7)',
       );
     }
+    // The whole config, so a new `SyncServerConfig` field reaches socket and
+    // hub-served HTTP rounds without being listed here.
     return {
+      ...this.#config,
+      segments,
       partition: identity.partition,
       actorId: identity.actorId,
-      schema: this.#config.schema,
-      ...(this.#config.schemaWindow !== undefined
-        ? { schemaWindow: this.#config.schemaWindow }
-        : {}),
-      storage: this.#config.storage,
-      segments,
-      resolveScopes: this.#config.resolveScopes,
-      ...(this.#config.blobs !== undefined
-        ? { blobs: this.#config.blobs }
-        : {}),
-      ...(this.#config.maxBlobBytes !== undefined
-        ? { maxBlobBytes: this.#config.maxBlobBytes }
-        : {}),
-      ...(this.#config.crdtMergers !== undefined
-        ? { crdtMergers: this.#config.crdtMergers }
-        : {}),
-      ...(this.#config.validators !== undefined
-        ? { validators: this.#config.validators }
-        : {}),
-      ...(this.#config.commitValidator !== undefined
-        ? { commitValidator: this.#config.commitValidator }
-        : {}),
-      ...(this.#config.reactionPlanner !== undefined
-        ? { reactionPlanner: this.#config.reactionPlanner }
-        : {}),
-      ...(this.#config.checkpoints !== undefined
-        ? { checkpoints: this.#config.checkpoints }
-        : {}),
-      ...(this.#config.clock !== undefined
-        ? { clock: this.#config.clock }
-        : {}),
-      ...(this.#config.limits !== undefined
-        ? { limits: this.#config.limits }
-        : {}),
-      ...(this.#config.signedUrls !== undefined
-        ? { signedUrls: this.#config.signedUrls }
-        : {}),
-      ...(this.#config.blobSignedUrls !== undefined
-        ? { blobSignedUrls: this.#config.blobSignedUrls }
-        : {}),
-      ...(this.#config.blobUploadUrls !== undefined
-        ? { blobUploadUrls: this.#config.blobUploadUrls }
-        : {}),
-      ...(this.#config.sqliteImageBuilder !== undefined
-        ? { sqliteImageBuilder: this.#config.sqliteImageBuilder }
-        : {}),
-      ...(this.#config.leases !== undefined
-        ? { leases: this.#config.leases }
-        : {}),
-      ...(this.#config.events !== undefined
-        ? { events: this.#config.events }
-        : {}),
-      ...(this.#config.onError !== undefined
-        ? { onError: this.#config.onError }
-        : {}),
-      ...(this.#config.mapError !== undefined
-        ? { mapError: this.#config.mapError }
-        : {}),
       realtime: this,
     };
   }
@@ -1311,6 +1266,22 @@ export class RealtimeHub {
     }
   }
 
+  /**
+   * RealtimeNotifier: a pull changed a client's subscription list (§8.7).
+   * Its sockets reload their registrations with the round-end semantics, so
+   * an unreadable record keeps the previous registrations.
+   */
+  async notifySubscriptions(
+    partition: string,
+    clientId: string,
+  ): Promise<void> {
+    for (const session of [...this.#sessions]) {
+      if (session.partition !== partition || session.clientId !== clientId)
+        continue;
+      await session.refreshAfterPull();
+    }
+  }
+
   /**
    * Host-initiated, fail-closed registration refresh (§8.7). A host calls this
    * after it changes an actor's membership or connection state, so the

package/src/relational-rows.ts

@@ -709,20 +709,17 @@ export function prefetchRowsSql(table: CompiledTable): string {
  * SQLite/D1: the newest `commit_seq` in the inverted change-scope index for
  * one variable's values, as one index-only max per value (a `(partition,
  * tbl, var, value)` prefix), so the cost is independent of the table's
- * history. Bind order: [partition, tbl, var, value] per value.
+ * history. The values arrive as one JSON array, so the statement has no
+ * compound SELECT and four bound parameters for any value count (D1 caps a
+ * compound SELECT at 5 terms and a statement at 100 parameters).
+ * Params: [partition, tbl, var, values as JSON array text].
  */
-export function latestScopedChangeSeqSql(valueCount: number): string {
-  const perValue: string[] = [];
-  for (let i = 0; i < valueCount; i++) {
-    perValue.push(
-      'SELECT (SELECT max(commit_seq) FROM sync_change_scopes WHERE partition=? AND tbl=? AND var=? AND value=?) AS seq',
-    );
-  }
-  return `SELECT max(seq) AS seq FROM (${perValue.join(' UNION ALL ')})`;
-}
+export const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = `SELECT max((SELECT max(commit_seq) FROM sync_change_scopes
+           WHERE partition=? AND tbl=? AND var=? AND value=v.value)) AS seq
+     FROM json_each(?) AS v`;
 
 /**
- * PostgreSQL set-based form of `latestScopedChangeSeqSql` for the
+ * PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
  * subscriptions of one table in a pull. `$2` is a JSON array of
  * `{i, var, vals}` records. Params: [partition, specs (JSON text), tbl].
  */