@proteinjs/db-driver-spanner 1.12.3 → 1.14.0

package/src/SpannerDriver.ts

@@ -19,10 +19,18 @@ import { SpannerSchemaMetadata } from './SpannerSchemaMetadata';
  * Google Spanner driver for ProteinJs Db
  */
 export class SpannerDriver implements DbDriver {
-  private static SPANNER: Spanner;
-  private static SPANNER_INSTANCE: Instance;
-  private static SPANNER_DB: Database;
+  private static SPANNER?: Spanner;
+  private static SPANNER_INSTANCE?: Instance;
+  private static SPANNER_DB?: Database;
   private static LIVENESS_MONITOR: SpannerLivenessMonitor;
+  /**
+   * Channel-death recycle accounting (2026-08-06 overnight wedge). Ops are tagged with the
+   * generation of the client they started on; only same-generation outcomes touch the counter,
+   * so a burst of deadline failures from the OLD (recycled) channel can never spuriously
+   * recycle the fresh one.
+   */
+  private static CLIENT_GENERATION = 0;
+  private static CONSECUTIVE_DEADLINE_FAILURES = 0;
   private logger = new Logger({ name: this.constructor.name });
   private config: SpannerConfig;
   public getTable: ((name: string) => Table<any>) | undefined;
@@ -75,7 +83,10 @@ export class SpannerDriver implements DbDriver {
 
   private getSpannerDb(): Database {
     if (!SpannerDriver.SPANNER_DB) {
-      SpannerDriver.SPANNER_DB = this.getSpannerInstance().database(this.config.databaseName);
+      SpannerDriver.SPANNER_DB = this.getSpannerInstance().database(
+        this.config.databaseName,
+        this.config.sessionPoolOptions
+      );
       SpannerDriver.LIVENESS_MONITOR = new SpannerLivenessMonitor(SpannerDriver.SPANNER_DB).start();
     }
 
@@ -174,13 +185,17 @@ export class SpannerDriver implements DbDriver {
 
     try {
       this.logger.debug({ message: `Executing query`, obj: { sql, params: namedParams } });
-      const [rows] = await this.logIfStalled(
+      const [rows] = await this.withDeadline(
         'spanner query',
         sql,
         runner.run({
           sql,
           params: namedParams?.params,
           types: namedParams?.types,
+          // The gRPC deadline is what actually cancels the RPC on a dead channel: the stream
+          // errors, the library ends the snapshot, and the borrowed session RETURNS to the
+          // pool. The withDeadline race alone would fail the caller but leak the session.
+          gaxOptions: { timeout: this.operationDeadlineMs() },
         })
       );
       const durationMs = Number(process.hrtime.bigint() - startTime) / 1_000_000;
@@ -214,14 +229,22 @@ export class SpannerDriver implements DbDriver {
     }
 
     // Stalls in the transaction wrapper itself (session acquisition / begin / commit) happen
-    // OUTSIDE executeDml's instrumentation — wrap the whole round trip too.
-    return await this.logIfStalled(
+    // OUTSIDE executeDml's instrumentation — wrap the whole round trip too. Every await inside
+    // the run function is deadline-bounded (dml, commit, rollback): the run function therefore
+    // ALWAYS settles, which is what makes runTransactionAsync's own `finally` release the
+    // transaction's session back to the pool on a dead channel.
+    return await this.withDeadline(
       'spanner dml transaction',
       '(runTransactionAsync)',
       this.getSpannerDb().runTransactionAsync(async (transaction) => {
-        const rowCount = await this.executeDml(generateStatement, transaction);
-        await transaction.commit();
-        return rowCount;
+        try {
+          const rowCount = await this.executeDml(generateStatement, transaction);
+          await this.commit(transaction);
+          return rowCount;
+        } catch (error) {
+          await this.rollbackQuietly(transaction);
+          throw error;
+        }
       })
     );
   }
@@ -241,13 +264,16 @@ export class SpannerDriver implements DbDriver {
 
     try {
       this.logger.debug({ message: `Executing dml`, obj: { sql, params: namedParams } });
-      const [rowCount] = await this.logIfStalled(
+      const [rowCount] = await this.withDeadline(
         'spanner dml',
         sql,
         runner.runUpdate({
           sql,
           params: namedParams?.params,
           types: namedParams?.types,
+          // gRPC deadline: cancels the RPC on a dead channel so the transaction's run function
+          // settles and the library releases the session (see runDml).
+          gaxOptions: { timeout: this.operationDeadlineMs() },
         })
       );
       const durationMs = Number(process.hrtime.bigint() - startTime) / 1_000_000;
@@ -273,36 +299,91 @@ export class SpannerDriver implements DbDriver {
    * @returns the return of the `fn`
    */
   async runTransaction<T>(fn: (transaction: Transaction) => Promise<T>): Promise<T> {
-    return await this.logIfStalled(
+    return await this.withDeadline(
       'spanner transaction',
       '(runTransactionAsync)',
       this.getSpannerDb().runTransactionAsync(async (transaction) => {
-        const result = await fn(transaction);
-        await transaction.commit();
-        return result;
+        try {
+          const result = await fn(transaction);
+          await this.commit(transaction);
+          return result;
+        } catch (error) {
+          await this.rollbackQuietly(transaction);
+          throw error;
+        }
       })
     );
   }
 
   /**
-   * Stall diagnostics (2026-07-10 flow-hang investigation): background flow tasks intermittently
-   * wedge between model calls with every model-layer guard silent — the remaining awaits on that
-   * path are Spanner ops, and this client has NO acquire/read timeouts (a wedged session/gRPC
-   * stream waits forever). Logs ops still pending at 30s and 120s — with the op and statement —
-   * then keeps waiting: pure diagnosis, zero behavior change.
+   * Deadline-bounded commit: a commit hanging on a dead channel would otherwise keep the
+   * transaction's run function pending forever, and with it the session (runTransactionAsync
+   * only releases the session once the run function settles).
+   */
+  private async commit(transaction: Transaction): Promise<void> {
+    await this.withDeadline(
+      'spanner commit',
+      '(commit)',
+      transaction.commit({ gaxOptions: { timeout: this.operationDeadlineMs() } })
+    );
+  }
+
+  /**
+   * Release a transaction whose work errored. The client library's runner does NOT roll back on
+   * non-retryable errors — it just rethrows — so a thrown `fn` (e.g. an application rollback, or
+   * a failed statement) left the read-write transaction OPEN until session timeout. Real Spanner
+   * tolerates that (locks expire); the emulator serializes on its single read-write transaction,
+   * so one leaked transaction blocks every subsequent DDL with FAILED_PRECONDITION ("a
+   * read-write transaction is already in progress") — poisoning whole test runs. Rollback of an
+   * already-invalid transaction (e.g. ABORTED, about to be retried by the runner with a fresh
+   * transaction, or one whose commit was already called) is expected to fail; that failure is
+   * logged at debug and swallowed so the ORIGINAL error — the one that carries retry semantics —
+   * always propagates. Deadline-bounded like commit: an unbounded rollback on a dead channel
+   * would keep the run function (and its session) pending forever.
+   */
+  private async rollbackQuietly(transaction: Transaction): Promise<void> {
+    try {
+      await this.withDeadline(
+        'spanner rollback',
+        '(rollback)',
+        transaction.rollback({ timeout: this.operationDeadlineMs() })
+      );
+    } catch (rollbackError: any) {
+      this.logger.debug({ message: `Rollback after transaction error failed`, obj: { rollbackError } });
+    }
+  }
+
+  /**
+   * The one choke point every query/dml/transaction op awaits through. Two jobs:
+   *
+   * 1. Stall diagnostics (2026-07-10 flow-hang investigation): logs ops still pending at 30s
+   *    and 120s with the op and statement.
+   * 2. OP DEADLINE (2026-08-06 overnight wedge): an op that has not settled by
+   *    `operationDeadlineMs` (default 60s) FAILS with an error naming the deadline instead of
+   *    hanging forever on a dead gRPC channel. This race covers hangs the gRPC deadline cannot
+   *    see (session-pool acquisition, library internals before the RPC starts); the
+   *    wire-level cancellation + session return is the gRPC deadline attached per call
+   *    (`gaxOptions.timeout`, same value) — see the call sites. Deadline failures feed the
+   *    channel-death recycle counter; any success resets it.
    */
-  private logIfStalled<T>(op: string, sql: string, promise: PromiseLike<T>, timeoutMs = 30_000): Promise<T> {
+  private withDeadline<T>(op: string, sql: string, promise: PromiseLike<T>): Promise<T> {
+    // Pool gauge (P4a): every op passes through here, so this is where waiting-on-the-pool
+    // becomes visible (throttled inside the monitor). The monitor exists by now — all ops
+    // require getSpannerDb() first.
+    SpannerDriver.LIVENESS_MONITOR.logPoolPressure();
+    const deadlineMs = this.operationDeadlineMs();
+    const generation = SpannerDriver.CLIENT_GENERATION;
     let settled = false;
     const logStall = (afterMs: number) =>
       this.logger.error({
         message: `Spanner op stalled: ${op}`,
-        obj: { afterMs, sql: String(sql).slice(0, 200) },
+        obj: { afterMs, sql: String(sql).slice(0, 200), pool: SpannerDriver.LIVENESS_MONITOR.poolStats() },
       });
     const t1 = setTimeout(() => {
       if (!settled) {
-        logStall(timeoutMs);
+        logStall(30_000);
       }
-    }, timeoutMs);
+    }, 30_000);
     const t2 = setTimeout(() => {
       if (!settled) {
         logStall(120_000);
@@ -310,13 +391,119 @@ export class SpannerDriver implements DbDriver {
     }, 120_000);
     t1.unref?.();
     t2.unref?.();
-    return Promise.resolve(promise).finally(() => {
-      settled = true;
-      clearTimeout(t1);
-      clearTimeout(t2);
+    return new Promise<T>((resolve, reject) => {
+      const clear = () => {
+        settled = true;
+        clearTimeout(t1);
+        clearTimeout(t2);
+        clearTimeout(deadlineTimer);
+      };
+      const deadlineTimer = setTimeout(() => {
+        if (settled) {
+          return;
+        }
+        clear();
+        this.logger.error({
+          message: `Spanner op exceeded its ${deadlineMs}ms deadline: ${op} — failing the op`,
+          obj: { sql: String(sql).slice(0, 200), pool: SpannerDriver.LIVENESS_MONITOR.poolStats() },
+        });
+        this.recordDeadlineFailure(generation);
+        // Deliberately no grpc `code` on this error: the liveness monitor's probe/exit
+        // escalation stays owned by genuine grpc errors; hang-shaped death is owned by the
+        // recycle counter.
+        reject(
+          new Error(
+            `Spanner op exceeded its ${deadlineMs}ms deadline: ${op} (configure via SpannerConfig.operationDeadlineMs)`
+          )
+        );
+      }, deadlineMs);
+      deadlineTimer.unref?.();
+      Promise.resolve(promise).then(
+        (value) => {
+          if (settled) {
+            return; // deadline already failed the caller; the library handles the late settle
+          }
+          clear();
+          this.recordOpSuccess(generation);
+          resolve(value);
+        },
+        (error) => {
+          if (settled) {
+            return;
+          }
+          clear();
+          // A grpc DEADLINE_EXCEEDED (code 4) is the same dead-channel signal arriving via the
+          // per-call gRPC deadline — classification for the one counter, not a second path.
+          if (error?.code === 4) {
+            this.recordDeadlineFailure(generation);
+          }
+          reject(error);
+        }
+      );
     });
   }
 
+  // ── Channel-death recycle (one owner for the process-wide client) ─────────
+
+  private recordOpSuccess(generation: number): void {
+    if (generation !== SpannerDriver.CLIENT_GENERATION) {
+      return; // outcome from a recycled client says nothing about the fresh channel
+    }
+    SpannerDriver.CONSECUTIVE_DEADLINE_FAILURES = 0;
+  }
+
+  private recordDeadlineFailure(generation: number): void {
+    if (generation !== SpannerDriver.CLIENT_GENERATION) {
+      return;
+    }
+    SpannerDriver.CONSECUTIVE_DEADLINE_FAILURES += 1;
+    if (SpannerDriver.CONSECUTIVE_DEADLINE_FAILURES < this.deadlineFailuresBeforeRecycle()) {
+      return;
+    }
+    // Reset BEFORE recycling: the counter and generation swap happen in this same synchronous
+    // frame, so a burst of ops all timing out together triggers exactly one recycle.
+    SpannerDriver.CONSECUTIVE_DEADLINE_FAILURES = 0;
+    this.recycleClient();
+  }
+
+  /**
+   * Drop the process-wide Spanner client so the next op builds a fresh client/channel — the
+   * categorical cure for a dead gRPC channel (Mac sleep, NAT drop) that deadlines every op.
+   * The old liveness monitor is stopped so its probe/exit escalation can't kill the process
+   * for a channel we just abandoned; LIVENESS_MONITOR itself stays pointed at it (stopped)
+   * until getSpannerDb() installs the new client's monitor, keeping in-flight ops' error
+   * paths callable.
+   */
+  private recycleClient(): void {
+    this.logger.error({
+      message: `Recycling Spanner client after ${this.deadlineFailuresBeforeRecycle()} consecutive op-deadline failures (dead gRPC channel suspected) — a fresh client/channel will be created on the next op`,
+    });
+    SpannerDriver.CLIENT_GENERATION += 1;
+    const oldMonitor = SpannerDriver.LIVENESS_MONITOR;
+    const oldDb = SpannerDriver.SPANNER_DB;
+    const oldSpanner = SpannerDriver.SPANNER;
+    SpannerDriver.SPANNER = undefined;
+    SpannerDriver.SPANNER_INSTANCE = undefined;
+    SpannerDriver.SPANNER_DB = undefined;
+    oldMonitor?.stop();
+    // Best-effort teardown of the old client: session deletes are themselves RPCs on the very
+    // channel we believe is dead — the recycle must not depend on them succeeding.
+    void oldDb?.close().catch(() => undefined);
+    try {
+      oldSpanner?.close();
+    } catch {
+      // already torn down
+    }
+  }
+
+  private operationDeadlineMs(): number {
+    return this.config.operationDeadlineMs ?? 60_000;
+  }
+
+  private deadlineFailuresBeforeRecycle(): number {
+    return this.config.deadlineFailuresBeforeRecycle ?? 3;
+  }
+
   /**
    * Execute a schema write operation.
    */

package/src/SpannerLivenessMonitor.ts

@@ -1,39 +1,101 @@
-import { Database } from '@google-cloud/spanner';
+import { Database, SessionPool } from '@google-cloud/spanner';
 import { Logger } from '@proteinjs/logger';
 
 /** grpc status codes that indicate connectivity trouble rather than an application error */
 const CONNECTIVITY_GRPC_CODES = [4 /* DEADLINE_EXCEEDED */, 14 /* UNAVAILABLE */];
 
+/**
+ * Restart-requested exit code — the supervision contract shared with @proteinjs/build's
+ * serve-package (ServePackageSupervisor.RESTART_REQUEST_EXIT_CODE): a supervised process
+ * exiting with this code is respawned with bounded backoff instead of being treated as a plain
+ * failure (which the supervisor mirrors, i.e. stays down). Orchestrators that restart on any
+ * nonzero exit (systemd, Kubernetes) treat it like any other failure code, so it is safe
+ * everywhere. Hard-coded by design: the contract crosses a process boundary, like a signal.
+ */
+const RESTART_REQUEST_EXIT_CODE = 86;
+
+/** Session pool gauge (P4a): the four numbers that make pool exhaustion observable. */
+export type SpannerSessionPoolStats = {
+  size: number;
+  available: number;
+  borrowed: number;
+  totalWaiters: number;
+};
+
 export class SpannerLivenessMonitor {
   private static readonly PROBE_SQL = 'SELECT 1';
   private static readonly PROBE_TIMEOUT_MS = 10_000;
   /** delay before each attempt; 5 attempts spanning ~2 min (fast failures) to ~4.5 min (30s-deadline failures) */
   private static readonly PROBE_DELAYS_MS = [0, 5_000, 15_000, 30_000, 60_000];
+  /** waiters > 0 is the wedge signature — warn on sight, but at most once per interval per process */
+  private static readonly POOL_PRESSURE_WARN_INTERVAL_MS = 10_000;
   private logger = new Logger({ name: this.constructor.name });
   private checkInFlight = false;
+  private stopped = false;
+  private lastPoolPressureWarnMs = Number.NEGATIVE_INFINITY;
 
   constructor(private db: Database) {}
 
   /** Attach the single 'error' listener; called once when the Database singleton is created. */
   start(): this {
     this.db.on('error', (error: any) => {
+      if (this.stopped) {
+        return;
+      }
       this.logger.warn({
         message: `Spanner session pool emitted a background error; verifying db connectivity`,
-        obj: { code: error?.code, errorDetails: error?.details ?? String(error) },
+        obj: { code: error?.code, errorDetails: error?.details ?? String(error), pool: this.poolStats() },
       });
       void this.verifyLiveness();
     });
     return this;
   }
 
+  /**
+   * Retire this monitor when its client is recycled (channel-death recycle in SpannerDriver):
+   * a stopped monitor must never escalate to process exit for a channel the driver has already
+   * abandoned — the replacement client gets a fresh monitor.
+   */
+  stop(): void {
+    this.stopped = true;
+  }
+
   /** Called from driver catch blocks; probes only for connectivity-shaped errors. */
   reportError(error: any): void {
-    if (!CONNECTIVITY_GRPC_CODES.includes(error?.code)) {
+    if (this.stopped || !CONNECTIVITY_GRPC_CODES.includes(error?.code)) {
       return;
     }
     void this.verifyLiveness();
   }
 
+  poolStats(): SpannerSessionPoolStats {
+    const pool = (this.db as unknown as { pool_: SessionPool }).pool_;
+    return { size: pool.size, available: pool.available, borrowed: pool.borrowed, totalWaiters: pool.totalWaiters };
+  }
+
+  /**
+   * Warn whenever operations are queued waiting on the session pool (the silent-wedge
+   * signature — with an infinite acquireTimeout a wedged pool otherwise produces no signal at
+   * all). Called by the driver as ops are issued; throttled so a contended burst emits one
+   * line per interval, not one per queued op.
+   */
+  logPoolPressure(nowMs = Date.now()): void {
+    const stats = this.poolStats();
+    if (stats.totalWaiters === 0) {
+      return;
+    }
+
+    if (nowMs - this.lastPoolPressureWarnMs < SpannerLivenessMonitor.POOL_PRESSURE_WARN_INTERVAL_MS) {
+      return;
+    }
+
+    this.lastPoolPressureWarnMs = nowMs;
+    this.logger.warn({
+      message: `Spanner session pool under pressure: operations are waiting for a session`,
+      obj: stats,
+    });
+  }
+
   // --- helpers last ---
 
   private async verifyLiveness(): Promise<void> {
@@ -44,6 +106,9 @@ export class SpannerLivenessMonitor {
     try {
       for (let attempt = 0; attempt < SpannerLivenessMonitor.PROBE_DELAYS_MS.length; attempt++) {
         await this.sleep(SpannerLivenessMonitor.PROBE_DELAYS_MS[attempt]);
+        if (this.stopped) {
+          return; // client recycled mid-cycle — this channel's fate no longer matters
+        }
         try {
           await this.probe();
           this.logger.info({ message: `Db connectivity verified`, obj: { attempt: attempt + 1 } });
@@ -55,8 +120,11 @@ export class SpannerLivenessMonitor {
           });
         }
       }
+      if (this.stopped) {
+        return;
+      }
       this.logger.error({
-        message: `Db unreachable after sustained probing; exiting so supervision can restart into a valid state`,
+        message: `Db unreachable after sustained probing; exiting restart-requested (code ${RESTART_REQUEST_EXIT_CODE}) so supervision respawns into a valid state`,
       });
       this.exit();
     } finally {
@@ -76,6 +144,6 @@ export class SpannerLivenessMonitor {
   }
 
   private exit(): void {
-    process.exit(1);
+    process.exit(RESTART_REQUEST_EXIT_CODE);
   }
 }

package/test/ServiceUpdateVerbs.test.ts

@@ -0,0 +1,188 @@
+import { SpannerDriver } from '@proteinjs/db-driver-spanner';
+import {
+  ArrayMembershipUpdate,
+  computeArrayMembershipOps,
+  Db,
+  isTable,
+  PreservedPath,
+  ReferenceArray,
+  Table,
+  tableByName,
+} from '@proteinjs/db';
+import { TransactionContext } from '@proteinjs/db-transaction-context';
+import { getDropTestTable } from './util/getDropTestTable';
+import { SpannerEmulatorProvisioner } from './util/SpannerEmulatorProvisioner';
+import {
+  ServiceVerbsDoc,
+  ServiceVerbsDocTable,
+  serviceVerbsScopeContext as scope,
+} from './util/serviceUpdateVerbsTestTables';
+import '../generated/test/index';
+
+/**
+ * The RMW update verbs (`updateArrayMembership`, `updatePreserving`) over the DbService RPC
+ * path, against real Spanner transaction semantics.
+ *
+ * The service path differs from bespoke server code in three load-bearing ways, each pinned here:
+ * - ONE long-lived `Db` instance serves every request (ServiceRouter builds its executor map
+ *   once), so the verbs' self-wrapped transactions must not couple concurrent callers through
+ *   instance state.
+ * - args cross a serialization boundary: `Table` as `{ tableName }` resolved via `tableByName`,
+ *   the op payloads (`ArrayMembershipUpdate`, `PreservedPath`) as plain JSON.
+ * - authorization is the `TableServiceAuth.canAccess(methodName, args)` gate plus scoped/column
+ *   query injection inside the verb's read-modify-write — an out-of-scope row must behave as
+ *   nonexistent.
+ */
+
+const spannerDriver = new SpannerDriver({
+  projectId: 'proteinjs-test',
+  instanceName: 'proteinjs-test',
+  databaseName: 'test',
+});
+
+describe('DbService RMW update verbs (updateArrayMembership / updatePreserving)', () => {
+  const docTable = new ServiceVerbsDocTable() as Table<ServiceVerbsDoc>;
+  const dropTable = getDropTestTable(spannerDriver);
+  // Service-singleton shaped: one shared instance, default table resolution (tableByName).
+  const db = new Db<ServiceVerbsDoc>(spannerDriver, undefined, new TransactionContext());
+
+  /**
+   * The wire forms the Serializer produces for DbService args: `Table` crosses as its name and is
+   * resolved via `tableByName` server-side (TableSerializer); the op payloads are plain JSON.
+   */
+  const overWire = (args: any[]) =>
+    args.map((arg) => (isTable(arg) ? tableByName((arg as Table<any>).name) : JSON.parse(JSON.stringify(arg))));
+
+  /** Invoke like ServiceExecutor does: deserialize wire args, run the canAccess gate, call the method. */
+  const rpc = async (methodName: string, args: any[]) => {
+    const wireArgs = overWire(args);
+    if (!db.serviceMetadata!.auth!.canAccess!(methodName, wireArgs)) {
+      throw new Error(`User not authorized to run service: DbService.${methodName}`);
+    }
+    return await (db as any)[methodName](...wireArgs);
+  };
+
+  const memberIds = async (id: string) => {
+    const row = await db.get(docTable, { id });
+    return row?.members?._ids;
+  };
+
+  beforeAll(async () => {
+    await SpannerEmulatorProvisioner.ensureProvisioned({
+      projectId: 'proteinjs-test',
+      instanceName: 'proteinjs-test',
+      databaseName: 'test',
+    });
+    await dropTable(docTable);
+    await spannerDriver.getTableManager().loadTable(docTable);
+  }, 60000);
+
+  afterAll(async () => {
+    await dropTable(docTable);
+    await SpannerEmulatorProvisioner.release();
+  }, 30000);
+
+  beforeEach(() => {
+    scope.current = 'scope-a';
+  });
+
+  test('membership ops apply against committed truth, not the caller list snapshot', async () => {
+    const doc = await db.insert(docTable, {
+      title: 'd1',
+      members: new ReferenceArray(docTable.name, ['a', 'b', 'c']),
+    });
+    // Another writer's committed wholesale update adds `d` — the caller's snapshot is now stale.
+    await db.update(docTable, { id: doc.id, members: new ReferenceArray(docTable.name, ['a', 'b', 'c', 'd']) });
+
+    // The caller computed remove(b) against the stale base [a, b, c].
+    const update: ArrayMembershipUpdate = {
+      recordId: doc.id,
+      columnPropertyName: 'members',
+      ops: computeArrayMembershipOps(['a', 'b', 'c'], ['a', 'c']),
+    };
+    expect(await rpc('updateArrayMembership', [docTable, update])).toBe(1);
+
+    // Both writers' effects survive; a wholesale write of the stale list would have erased `d`.
+    expect(await memberIds(doc.id)).toEqual(['a', 'c', 'd']);
+  });
+
+  test('a scoped caller cannot touch another scope’s row via updateArrayMembership', async () => {
+    const doc = await db.insert(docTable, { title: 'd2', members: new ReferenceArray(docTable.name, ['a', 'b']) });
+
+    scope.current = 'scope-b';
+    const removeA: ArrayMembershipUpdate = {
+      recordId: doc.id,
+      columnPropertyName: 'members',
+      ops: [{ op: 'remove', id: 'a' }],
+    };
+    expect(await rpc('updateArrayMembership', [docTable, removeA])).toBe(0);
+
+    scope.current = 'scope-a';
+    expect(await memberIds(doc.id)).toEqual(['a', 'b']);
+
+    // The owning scope can perform the identical op.
+    expect(await rpc('updateArrayMembership', [docTable, removeA])).toBe(1);
+    expect(await memberIds(doc.id)).toEqual(['b']);
+  });
+
+  test('a scoped caller cannot touch another scope’s row via updatePreserving', async () => {
+    const doc = await db.insert(docTable, { title: 'd3', body: { content: 'v1', style: { color: 'blue' } } });
+
+    scope.current = 'scope-b';
+    const preserve: PreservedPath[] = [{ columnPropertyName: 'body', paths: ['content'], whenType: 'string' }];
+    const foreignWrite = { id: doc.id, body: { content: 'hijack', style: { color: 'red' } } };
+    expect(await rpc('updatePreserving', [docTable, foreignWrite, preserve])).toBe(0);
+
+    scope.current = 'scope-a';
+    const row = await db.get(docTable, { id: doc.id });
+    expect(row.body).toEqual({ content: 'v1', style: { color: 'blue' } });
+  });
+
+  test('updatePreserving over the service preserves the committed sub-path while writing owned paths', async () => {
+    const doc = await db.insert(docTable, { title: 'd4', body: { content: 'v1', style: { color: 'blue' } } });
+    // The owning writer's committed text save; the structural writer's payload still carries v1.
+    await db.update(docTable, { id: doc.id, body: { content: 'v2', style: { color: 'blue' } } });
+
+    const preserve: PreservedPath[] = [{ columnPropertyName: 'body', paths: ['content'], whenType: 'string' }];
+    const stalePayload = { id: doc.id, body: { content: 'v1', style: { color: 'red' } } };
+    expect(await rpc('updatePreserving', [docTable, stalePayload, preserve])).toBe(1);
+
+    const row = await db.get(docTable, { id: doc.id });
+    expect(row.body).toEqual({ content: 'v2', style: { color: 'red' } });
+  });
+
+  test('concurrent RPCs on the shared service instance get isolated transactions', async () => {
+    const doc = await db.insert(docTable, { title: 'd5', members: new ReferenceArray(docTable.name, ['seed']) });
+
+    // The service singleton serves concurrent requests; each self-wrapped RMW transaction must be
+    // isolated (no cross-request transaction bleed through shared instance state), and every
+    // membership add must land. Issuance is STAGGERED so later requests arrive while an earlier
+    // request's transaction is open — the shape that makes shared instance state bleed: a
+    // simultaneous burst would let every call check the (still unset) transaction state before
+    // any transaction opens, masking the coupling.
+    const addedIds = ['m1', 'm2', 'm3', 'm4', 'm5', 'm6'];
+    const inFlight: Promise<unknown>[] = [];
+    for (const id of addedIds) {
+      inFlight.push(
+        rpc('updateArrayMembership', [
+          docTable,
+          { recordId: doc.id, columnPropertyName: 'members', ops: [{ op: 'add', id, afterId: 'seed' }] },
+        ])
+      );
+      await new Promise((resolve) => setTimeout(resolve, 5));
+    }
+    await Promise.all(inFlight);
+
+    expect([...((await memberIds(doc.id)) ?? [])].sort()).toEqual([...addedIds, 'seed'].sort());
+  }, 60000);
+
+  test('updateArrayMembership rejects a non-ReferenceArrayColumn target', async () => {
+    const doc = await db.insert(docTable, { title: 'd6' });
+    await expect(
+      rpc('updateArrayMembership', [
+        docTable,
+        { recordId: doc.id, columnPropertyName: 'title', ops: [{ op: 'add', id: 'x', afterId: null }] },
+      ])
+    ).rejects.toThrow('requires a ReferenceArrayColumn');
+  });
+});