@powersync/service-module-mongodb-storage 0.22.0 → 0.23.0

package/test/src/concurrent_writers.test.ts

@@ -0,0 +1,599 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { test_utils } from '@powersync/service-core-tests';
+import { execFile } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+import { describe, expect, test, vi } from 'vitest';
+import { PersistedBatch } from '../../src/storage/implementation/common/PersistedBatch.js';
+import { MongoOpIdAllocator } from '../../src/storage/implementation/MongoOpIdAllocator.js';
+import { MongoPersistedReplicationStream } from '../../src/storage/implementation/MongoPersistedReplicationStream.js';
+import { mongoTestStorageFactoryGenerator } from '../../src/utils/test-utils.js';
+import { env } from './env.js';
+
+const factoryGen = mongoTestStorageFactoryGenerator({ url: env.MONGO_TEST_URL, isCI: env.CI });
+const rules = `bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM items
+`;
+
+async function openStream(factory: Awaited<ReturnType<typeof factoryGen.factory>>, version: number) {
+  const stream = await factory.updateSyncRules(updateSyncRulesFromYaml(rules, { storageVersion: version }));
+  const bucketStorage = await test_utils.getTestStorage(factory, stream);
+  const writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+  const table = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, stream.replicationStreamId);
+  await writer.markAllSnapshotDone('1/1');
+  return { stream, bucketStorage, writer, table };
+}
+
+async function insert(writer: storage.BucketStorageBatch, table: storage.SourceTable, id: string) {
+  await writer.save({
+    sourceTable: table,
+    tag: storage.SaveOperationTag.INSERT,
+    after: { id, description: id },
+    afterReplicaId: test_utils.rid(id)
+  });
+}
+
+describe.each([1, 2, 4])('concurrent writers v%s', (version) => {
+  test('requires a lease to create a writer, including after another lease is released', async () => {
+    await using factory = await factoryGen.factory();
+    const stream = await factory.updateSyncRules(updateSyncRulesFromYaml(rules, { storageVersion: version }));
+    const unleased = factory.getInstance(stream);
+    await expect(unleased.createWriter(test_utils.BATCH_OPTIONS)).rejects.toThrow('replication lease is required');
+    const lock = await stream.lock();
+    try {
+      const leased = factory.getInstance(stream, { replicationLock: lock });
+      await using writer = await leased.createWriter(test_utils.BATCH_OPTIONS);
+      await lock.release();
+      await expect(writer.commit('1/2')).rejects.toThrow('Replication lock released');
+      await expect(leased.createWriter(test_utils.BATCH_OPTIONS)).rejects.toThrow('Replication lock released');
+      await expect(unleased.createWriter(test_utils.BATCH_OPTIONS)).rejects.toThrow('replication lease is required');
+    } finally {
+      await lock.release();
+    }
+  });
+
+  test('combines the final flush and checkpoint into one transaction', async () => {
+    const monitored = mongoTestStorageFactoryGenerator({
+      url: env.MONGO_TEST_URL,
+      isCI: env.CI,
+      monitorCommands: true
+    });
+    await using factory = await monitored.factory();
+    const a = await openStream(factory, version);
+    await using writer = a.writer;
+    // Warm up activation, the reservation and error clearing before measuring.
+    await insert(writer, a.table, 'warmup');
+    await writer.commit('1/2');
+    const commands: string[] = [];
+    const listener = (event: mongo.CommandStartedEvent) => {
+      if (event.command.autocommit === false) {
+        commands.push(event.commandName);
+      }
+    };
+    factory.db.client.on('commandStarted', listener);
+    try {
+      await insert(writer, a.table, 'separate');
+      await writer.flush();
+      await writer.commit('1/3');
+      const separate = [...commands];
+      commands.length = 0;
+      await insert(writer, a.table, 'combined');
+      await writer.commit('1/4');
+      expect(separate.filter((name) => name === 'commitTransaction')).toHaveLength(2);
+      expect(commands.filter((name) => name === 'commitTransaction')).toHaveLength(1);
+      expect(separate.length - commands.length).toBe(version < 3 ? 2 : 3);
+      expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(3n);
+    } finally {
+      factory.db.client.off('commandStarted', listener);
+    }
+  });
+
+  test('only checkpoints the last piece of a split flush', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, version);
+    await using writer = a.writer;
+    await insert(writer, a.table, 'warmup');
+    await writer.commit('1/2');
+    // Force a split after each row without needing a transaction-sized fixture.
+    const split = vi.spyOn(PersistedBatch.prototype, 'shouldFlushTransaction').mockReturnValue(true);
+    const flush = PersistedBatch.prototype.flush;
+    const observed: bigint[] = [];
+    const inspect = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementation(async function (
+      this: PersistedBatch,
+      ...args
+    ) {
+      observed.push((await a.bucketStorage.getCheckpoint()).checkpoint);
+      return flush.apply(this, args);
+    });
+    try {
+      await insert(writer, a.table, 'first');
+      await insert(writer, a.table, 'last');
+      await writer.commit('1/3');
+      expect(observed).toEqual([1n, 1n]);
+      expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(3n);
+    } finally {
+      inspect.mockRestore();
+      split.mockRestore();
+    }
+  });
+
+  test('retries the final rows and checkpoint together after a transaction conflict', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, version);
+    await using writer = a.writer;
+    await insert(writer, a.table, 'warmup');
+    await writer.commit('1/2');
+    const withTransaction = mongo.ClientSession.prototype.withTransaction;
+    let attempts = 0;
+    const retry = vi.spyOn(mongo.ClientSession.prototype, 'withTransaction').mockImplementationOnce(function (
+      this: mongo.ClientSession,
+      callback,
+      options
+    ) {
+      return withTransaction.call(
+        this,
+        async (session) => {
+          const result = await callback(session);
+          // Neither the final rows nor their checkpoint have committed yet.
+          expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(1n);
+          expect(writer.last_flushed_op).toBe(1n);
+          attempts += 1;
+          if (attempts === 1) {
+            throw new mongo.MongoServerError({
+              message: 'retry checkpoint',
+              code: 112,
+              errorLabels: ['TransientTransactionError']
+            });
+          }
+          return result;
+        },
+        options
+      );
+    });
+    try {
+      await insert(writer, a.table, 'retried');
+      await writer.commit('1/3');
+      expect(attempts).toBe(2);
+      expect(writer.last_flushed_op).toBe(2n);
+      expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(2n);
+    } finally {
+      retry.mockRestore();
+    }
+  });
+
+  test('a blocked combined checkpoint preserves the head for a new writer', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, version);
+    await using writer = a.writer;
+    await writer.markAllSnapshotDone('1/9');
+    await insert(writer, a.table, 'blocked');
+    expect(await writer.commit('1/2')).toMatchObject({ checkpointBlocked: true, checkpointCreated: false });
+    expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(0n);
+    await using resumed = await a.bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    await resumed.commit('1/9');
+    expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(writer.last_flushed_op);
+    expect(writer.last_flushed_op).toBe(1n);
+  });
+
+  test('same-stream writers queue FIFO before starting transactions', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, version);
+    await using first = a.writer;
+    await insert(first, a.table, 'warmup');
+    await first.commit('1/2');
+    // Separate storage instances must still share the lease's queue.
+    const otherStorage = factory.getInstance(a.stream);
+    await using checkpointWriter = await otherStorage.createWriter(test_utils.BATCH_OPTIONS);
+    await using last = await otherStorage.createWriter(test_utils.BATCH_OPTIONS);
+    const lastTable = await test_utils.resolveTestTable(
+      last,
+      'items',
+      ['id'],
+      factoryGen,
+      a.stream.replicationStreamId
+    );
+    await insert(first, a.table, 'first');
+    await insert(last, lastTable, 'last');
+
+    const entered = Promise.withResolvers<void>();
+    const release = Promise.withResolvers<void>();
+    const queued = Promise.withResolvers<void>();
+    const mutex = (a.stream as MongoPersistedReplicationStream).current_lock!.writerMutex;
+    const exclusiveLock = mutex.exclusiveLock;
+    let admissions = 0;
+    const admission = vi.spyOn(mutex, 'exclusiveLock').mockImplementation(function (callback) {
+      const result = exclusiveLock.call(mutex, callback);
+      if (++admissions === 3) queued.resolve();
+      return result;
+    });
+    const transactions = vi.spyOn(mongo.ClientSession.prototype, 'withTransaction');
+    const flush = PersistedBatch.prototype.flush;
+    const stalled = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementationOnce(async function (
+      this: PersistedBatch,
+      ...args
+    ) {
+      entered.resolve();
+      await release.promise;
+      return flush.apply(this, args);
+    });
+    const pending: Promise<unknown>[] = [];
+    try {
+      pending.push(first.flush());
+      await entered.promise;
+      // The empty checkpoint must run before the later flush.
+      pending.push(checkpointWriter.commit('1/3'));
+      pending.push(last.flush());
+      await queued.promise;
+      expect(transactions).toHaveBeenCalledTimes(1);
+      release.resolve();
+      await Promise.all(pending);
+      expect(first.last_flushed_op).toBe(2n);
+      expect(last.last_flushed_op).toBe(3n);
+      expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(2n);
+    } finally {
+      release.resolve();
+      await Promise.allSettled(pending);
+      stalled.mockRestore();
+      transactions.mockRestore();
+      admission.mockRestore();
+    }
+  });
+
+  test('another stream publishes while a transaction is stalled, without touching its reserved IDs', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, version);
+    await using writerA = a.writer;
+    // An independent factory/client has no shared allocator or in-process coordination.
+    await using other = await factoryGen.factory({ doNotClear: true });
+    const b = await openStream(other, version);
+    await using writerB = b.writer;
+    const entered = Promise.withResolvers<void>();
+    const release = Promise.withResolvers<void>();
+    const flush = PersistedBatch.prototype.flush;
+    const stalled = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementationOnce(async function (
+      this: PersistedBatch,
+      ...args
+    ) {
+      entered.resolve();
+      await release.promise;
+      return flush.apply(this, args);
+    });
+    await insert(writerA, a.table, 'a');
+    const publishingA = writerA.flush();
+    try {
+      await entered.promise;
+      await insert(writerB, b.table, 'b');
+      await writerB.commit('1/2');
+      expect((await b.bucketStorage.getCheckpoint()).checkpoint).toBeGreaterThan(1n);
+      expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(0n);
+    } finally {
+      release.resolve();
+      await publishingA;
+      stalled.mockRestore();
+    }
+    expect(writerA.last_flushed_op).toBe(1n);
+    // A can keep using its lower range: B's checkpoint belongs to another stream.
+    await insert(writerA, a.table, 'a2');
+    await writerA.flush();
+    expect(writerA.last_flushed_op).toBe(2n);
+  });
+
+  test('another process can reserve and checkpoint while this process holds a stream transaction', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, version);
+    await using writerA = a.writer;
+    const b = await openStream(factory, version);
+    await b.writer.dispose();
+    await test_utils.releaseTestStorageLease(factory, b.stream.replicationStreamId);
+    const entered = Promise.withResolvers<void>();
+    const release = Promise.withResolvers<void>();
+    const flush = PersistedBatch.prototype.flush;
+    const stalled = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementationOnce(async function (
+      this: PersistedBatch,
+      ...args
+    ) {
+      entered.resolve();
+      await release.promise;
+      return flush.apply(this, args);
+    });
+    await insert(writerA, a.table, 'a');
+    const pending = writerA.flush();
+    try {
+      await entered.promise;
+      const child = await promisify(execFile)(
+        process.execPath,
+        [
+          fileURLToPath(new URL('./helpers/concurrentWriter.mjs', import.meta.url)),
+          env.MONGO_TEST_URL,
+          `${b.stream.replicationStreamId}`
+        ],
+        { timeout: 10_000 }
+      ).catch((error) => {
+        throw new Error(`${error.message}\n${error.stdout}\n${error.stderr}`);
+      });
+      expect(child.stdout).toMatch(/checkpoint=[1-9][0-9]+/);
+      expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(0n);
+    } finally {
+      release.resolve();
+      await pending;
+      stalled.mockRestore();
+    }
+  }, 15_000);
+
+  test('reuses one reservation across writers and flushes and retries rolled-back writes', async () => {
+    await using factory = await factoryGen.factory();
+    const { writer: batch, table, bucketStorage } = await openStream(factory, version);
+    await using writer = batch;
+    const reservations = vi.spyOn(MongoOpIdAllocator.prototype, 'reserve');
+    const flush = PersistedBatch.prototype.flush;
+    const retry = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementationOnce(async function (
+      this: PersistedBatch,
+      ...args
+    ) {
+      await flush.apply(this, args);
+      throw new mongo.MongoServerError({ message: 'retry', code: 112, errorLabels: ['TransientTransactionError'] });
+    });
+    try {
+      for (let i = 0; i < 3; i++) {
+        await insert(writer, table, `${i}`);
+        await writer.flush();
+      }
+      await writer.commit('1/2');
+      expect(reservations).toHaveBeenCalledTimes(1);
+      expect((await bucketStorage.getCheckpoint()).checkpoint).toBe(3n);
+      await using nextWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      const nextTable = await test_utils.resolveTestTable(nextWriter, 'items', ['id'], factoryGen, 1);
+      await insert(nextWriter, nextTable, 'next');
+      await nextWriter.commit('1/3');
+      expect((await bucketStorage.getCheckpoint()).checkpoint).toBe(4n);
+      expect(reservations).toHaveBeenCalledTimes(1);
+      expect((await factory.db.op_id_sequence.findOne({ _id: 'main' }))!.op_id).toBeGreaterThan(3n);
+    } finally {
+      reservations.mockRestore();
+      retry.mockRestore();
+    }
+  });
+
+  test.each(['during flush', 'after flush'])('graceful cancellation %s preserves snapshot progress', async (when) => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, version);
+    await a.writer.dispose();
+    const abort = new AbortController();
+    await using writer = await a.bucketStorage.createWriter({ ...test_utils.BATCH_OPTIONS, signal: abort.signal });
+    const table = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, a.stream.replicationStreamId);
+    const flush = PersistedBatch.prototype.flush;
+    const interrupted = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementationOnce(async function (
+      this: PersistedBatch,
+      ...args
+    ) {
+      const result = await flush.apply(this, args);
+      if (when === 'during flush') {
+        abort.abort();
+      }
+      return result;
+    });
+    try {
+      await insert(writer, table, 'page');
+      await writer.flush();
+      if (when === 'after flush') {
+        abort.abort();
+      }
+      await writer.updateTableProgress(table, { replicatedCount: 1, totalEstimatedCount: 1 });
+      const persisted = await writer.getSourceTableStatus(table);
+      expect(persisted?.snapshotStatus?.replicatedCount).toBe(1);
+      await writer.commit('1/2');
+      expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(1n);
+    } finally {
+      interrupted.mockRestore();
+    }
+  });
+
+  test('recovers flushed operations through existing version-specific fields', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, version);
+    await using writer = a.writer;
+    await insert(writer, a.table, 'checkpointed');
+    await writer.commit('1/2');
+    await insert(writer, a.table, 'flushed');
+    await writer.flush();
+    const document = await factory.db.sync_rules.findOne({ _id: a.stream.replicationStreamId });
+    expect(document).not.toHaveProperty('writer_transaction');
+    if (version < 3) {
+      expect(document).not.toHaveProperty('last_persisted_op');
+      expect(document).toMatchObject({ last_checkpoint: 1n, keepalive_op: '2' });
+    } else {
+      expect(document).toHaveProperty('last_persisted_op', 2n);
+    }
+    await using other = await factoryGen.factory({ doNotClear: true });
+    const stream = (await other.getReplicatingReplicationStreams())[0];
+    await test_utils.releaseTestStorageLease(factory, a.stream.replicationStreamId);
+    const bucketStorage = await test_utils.getTestStorage(other, stream);
+    await using resumed = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    await resumed.commit('1/3');
+    expect((await bucketStorage.getCheckpoint()).checkpoint).toBe(2n);
+    if (version < 3) {
+      expect(await factory.db.sync_rules.findOne({ _id: stream.replicationStreamId })).toMatchObject({
+        last_checkpoint: 2n,
+        keepalive_op: null
+      });
+    }
+  });
+
+  test('fencing changes the heartbeat even when it is ahead of the clock', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, version);
+    await using writer = a.writer;
+    const future = new Date(Date.now() + 60_000);
+    await factory.db.sync_rules.updateOne(
+      { _id: a.stream.replicationStreamId },
+      { $set: { last_keepalive_ts: future } }
+    );
+    await writer.setResumeLsn('1/2');
+    const document = await factory.db.sync_rules.findOne({ _id: a.stream.replicationStreamId });
+    expect(document!.last_keepalive_ts!.getTime()).toBe(future.getTime() + 1);
+    expect(document).not.toHaveProperty('writer_transaction');
+  });
+
+  test('independent writers of the same stream never publish behind its durable head', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, version);
+    await using writerA = a.writer;
+    await insert(writerA, a.table, 'a');
+    await writerA.flush();
+    await using other = await factoryGen.factory({ doNotClear: true });
+    const reloaded = (await other.getReplicatingReplicationStreams()).find(
+      (s) => s.replicationStreamId === a.stream.replicationStreamId
+    )!;
+    await using writerB = await other
+      .getInstance(reloaded, { replicationLock: (a.stream as MongoPersistedReplicationStream).current_lock! })
+      .createWriter(test_utils.BATCH_OPTIONS);
+    const tableB = await test_utils.resolveTestTable(writerB, 'items', ['id'], factoryGen, 1);
+    await insert(writerB, tableB, 'b');
+    await writerB.flush();
+    const higher = writerB.last_flushed_op!;
+    await insert(writerA, a.table, 'c');
+    await writerA.flush();
+    expect(writerA.last_flushed_op).toBeGreaterThan(higher);
+    await writerA.commit('1/2');
+    expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(writerA.last_flushed_op);
+  });
+
+  test('lease takeover fences stale row, metadata, and checkpoint writes', async () => {
+    await using factory = await factoryGen.factory();
+    const initial = await openStream(factory, version);
+    await initial.writer.dispose();
+    const stream = initial.stream as MongoPersistedReplicationStream;
+    await test_utils.releaseTestStorageLease(factory, stream.replicationStreamId);
+    const first = await stream.lock();
+    await using firstLifetime = { [Symbol.asyncDispose]: () => first.release() };
+    await using oldWriter = await factory.getInstance(stream).createWriter(test_utils.BATCH_OPTIONS);
+    await using oldCheckpointWriter = await factory.getInstance(stream).createWriter(test_utils.BATCH_OPTIONS);
+    const oldTable = await test_utils.resolveTestTable(oldWriter, 'items', ['id'], factoryGen, 1);
+    await insert(oldWriter, oldTable, 'old');
+    await oldWriter.flush();
+    const abandonedEnd = (await factory.db.op_id_sequence.findOne({ _id: 'main' }))!.op_id;
+    await factory.db.sync_rules.updateOne(
+      { _id: stream.replicationStreamId },
+      { $set: { 'lock.expires_at': new Date(0) } }
+    );
+    await using other = await factoryGen.factory({ doNotClear: true });
+    const nextStream = (await other.getReplicatingReplicationStreams())[0];
+    const next = await nextStream.lock();
+    await using nextLifetime = { [Symbol.asyncDispose]: () => next.release() };
+    await using nextWriter = await other.getInstance(nextStream).createWriter(test_utils.BATCH_OPTIONS);
+    const nextTable = await test_utils.resolveTestTable(nextWriter, 'items', ['id'], factoryGen, 1);
+    await insert(nextWriter, nextTable, 'new');
+    await nextWriter.commit('1/2');
+    expect(nextWriter.last_flushed_op).toBeGreaterThan(abandonedEnd);
+    await insert(oldWriter, oldTable, 'stale');
+    await expect(oldWriter.flush()).rejects.toThrow('no longer owns');
+    await expect(oldWriter.setResumeLsn('9/9')).rejects.toThrow('no longer owns');
+    await expect(oldWriter.markAllSnapshotDone('9/9')).rejects.toThrow('no longer owns');
+    await expect(oldWriter.markTableSnapshotRequired(oldTable)).rejects.toThrow('no longer owns');
+    await expect(oldWriter.commit('9/9')).rejects.toThrow('no longer owns');
+    // No pending rows: this exercises the checkpoint fence, not the flush fence.
+    await expect(oldCheckpointWriter.commit('9/9')).rejects.toThrow('no longer owns');
+    await first.release();
+    expect((await factory.db.sync_rules.findOne({ _id: stream.replicationStreamId }))!.lock?.id).toBe(
+      (next as typeof first).lock_id
+    );
+  });
+
+  test.each([false, true])('extends a nearly exhausted range (fallback: %s)', async (fallback) => {
+    await using factory = await factoryGen.factory();
+    const { writer: batch, table, bucketStorage } = await openStream(factory, version);
+    await using writer = batch;
+    const allocator = factory.getOpIdAllocator(
+      bucketStorage.replicationStream,
+      bucketStorage.replicationStream.current_lock!
+    );
+    await allocator.reserve();
+    allocator.committed(65_535n);
+    // Simulate capacity becoming insufficient after the pre-transaction check.
+    // The fallback must still abort and retry safely in that case.
+    const capacity = vi.spyOn(allocator, 'ensureCapacity');
+    if (fallback) {
+      capacity.mockResolvedValueOnce(undefined);
+    }
+    const sequences = vi.spyOn(allocator, 'sequence');
+    try {
+      await insert(writer, table, 'first');
+      await insert(writer, table, 'second');
+      await writer.commit('1/2');
+      expect(writer.last_flushed_op).toBe(65_537n);
+      expect((await bucketStorage.getCheckpoint()).checkpoint).toBe(65_537n);
+      expect(sequences).toHaveBeenCalledTimes(fallback ? 2 : 1);
+    } finally {
+      capacity.mockRestore();
+      sequences.mockRestore();
+    }
+  });
+});
+
+describe('operation ID reservations', () => {
+  test('initializes and reserves disjoint ranges concurrently in one command each', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, 4);
+    await using writer = a.writer;
+    const db = factory.db.versioned(a.bucketStorage.replicationStream.getStorageConfig());
+    const allocators = Array.from({ length: 4 }, () => new MongoOpIdAllocator(db));
+    const initialize = vi.spyOn(db.op_id_sequence, 'updateOne');
+    const reserve = vi.spyOn(db.op_id_sequence, 'findOneAndUpdate');
+    try {
+      await Promise.all(allocators.map((allocator) => allocator.reserve()));
+      const firstIds = allocators.map((allocator) => allocator.sequence(0n).next()).sort((a, b) => Number(a - b));
+      expect(firstIds).toEqual([1n, 65_537n, 131_073n, 196_609n]);
+      expect(initialize).not.toHaveBeenCalled();
+      expect(reserve).toHaveBeenCalledTimes(4);
+    } finally {
+      initialize.mockRestore();
+      reserve.mockRestore();
+    }
+  });
+
+  test('refills below 16k remaining IDs, preserving the tail across disjoint ranges', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, 4);
+    await using writer = a.writer;
+    const db = factory.db.versioned(a.bucketStorage.replicationStream.getStorageConfig());
+    const allocator = factory.getOpIdAllocator(
+      a.bucketStorage.replicationStream,
+      a.bucketStorage.replicationStream.current_lock!
+    );
+    await allocator.ensureCapacity();
+    allocator.committed(49_152n);
+    await allocator.ensureCapacity();
+    expect((await db.op_id_sequence.findOne({ _id: 'main' }))!.op_id).toBe(65_536n);
+
+    // Another stream's reservation creates a gap that must not count as capacity.
+    await new MongoOpIdAllocator(db).reserve();
+    allocator.committed(49_153n);
+    await Promise.all([allocator.ensureCapacity(), allocator.ensureCapacity()]);
+    expect((await db.op_id_sequence.findOne({ _id: 'main' }))!.op_id).toBe(196_608n);
+    expect(allocator.sequence(49_153n).next()).toBe(49_154n);
+    const sequence = allocator.sequence(65_535n);
+    expect(sequence.next()).toBe(65_536n);
+    expect(sequence.next()).toBe(131_073n);
+    await allocator.ensureCapacity();
+    expect((await db.op_id_sequence.findOne({ _id: 'main' }))!.op_id).toBe(196_608n);
+  });
+
+  test('refuses a reservation that would overflow without changing the watermark', async () => {
+    await using factory = await factoryGen.factory();
+    const a = await openStream(factory, 4);
+    await using writer = a.writer;
+    const max = (1n << 63n) - 1n;
+    await factory.db.op_id_sequence.insertOne({ _id: 'main', op_id: max - 65_536n });
+    const allocator = factory.getOpIdAllocator(
+      a.bucketStorage.replicationStream,
+      a.bucketStorage.replicationStream.current_lock!
+    );
+    await allocator.reserve();
+    expect(allocator.sequence(max - 1n).next()).toBe(max);
+    await expect(allocator.reserve()).rejects.toThrow('Operation ID sequence exhausted');
+    expect((await factory.db.op_id_sequence.findOne({ _id: 'main' }))!.op_id).toBe(max);
+  });
+});

package/test/src/helpers/concurrentWriter.mjs

@@ -0,0 +1,29 @@
+import { storage } from '@powersync/service-core';
+import { test_utils } from '@powersync/service-core-tests';
+import { mongoTestStorageFactoryGenerator } from '../../../dist/utils/test-utils.js';
+
+// A separate process exercises the database protocol without sharing any mutex,
+// allocator, MongoClient, or lease object with the test runner.
+async function main() {
+  const factoryGen = mongoTestStorageFactoryGenerator({ url: process.argv[2], isCI: true });
+  await using factory = await factoryGen.factory({ doNotClear: true });
+  const streamId = Number(process.argv[3]);
+  const stream = (await factory.getReplicatingReplicationStreams()).find((s) => s.replicationStreamId === streamId);
+  const lock = await stream.lock();
+  await using lockLifetime = { [Symbol.asyncDispose]: () => lock.release() };
+  const bucketStorage = factory.getInstance(stream, { replicationLock: lock });
+  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+  const table = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, streamId);
+  await writer.save({
+    sourceTable: table,
+    tag: storage.SaveOperationTag.INSERT,
+    after: { id: 'child', description: 'child' },
+    afterReplicaId: test_utils.rid('child')
+  });
+  await writer.commit('1/2');
+  console.log(`checkpoint=${(await bucketStorage.getCheckpoint()).checkpoint}`);
+}
+
+await main();
+// Workspace test utilities start background timers. Exit after all resources are disposed.
+process.exit(0);

package/test/src/helpers/s3TestFactory.ts

@@ -4,7 +4,7 @@ import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 import { randomUUID } from 'node:crypto';
 import { MemoryObjectStorage } from './MemoryObjectStorage.js';
 
-const minioStorages: S3ObjectStorage[] = [];
+const rustfsStorages: S3ObjectStorage[] = [];
 
 export interface S3TestFactoryOptions {
   url: string;
@@ -26,24 +26,24 @@ function createTestStorageSuite(options: S3TestFactoryOptions, objectStorage: Ob
 
 /**
  * Creates an ObjectStorage instance for S3 tests.
- * Set MINIO_ENDPOINT to switch all S3 tests from MemoryObjectStorage
- * to a real MinIO/S3 endpoint.
- *   MINIO_ENDPOINT=http://localhost:9000
+ * Set RUSTFS_ENDPOINT to switch all S3 tests from MemoryObjectStorage
+ * to a real RustFS/S3 endpoint.
+ *   RUSTFS_ENDPOINT=http://localhost:9000
  */
 export function createS3TestStorageSuite(options: S3TestFactoryOptions) {
-  const minioEndpoint = process.env.MINIO_ENDPOINT;
+  const rustfsEndpoint = process.env.RUSTFS_ENDPOINT;
   let objectStorage: ObjectStorage;
-  if (minioEndpoint) {
+  if (rustfsEndpoint) {
     const s3 = new S3ObjectStorage({
       bucket: 'powersync-s3-test',
       region: 'us-east-1',
       prefix: `test-${process.pid}-${randomUUID()}`,
-      endpoint: minioEndpoint,
+      endpoint: rustfsEndpoint,
       forcePathStyle: true,
-      accessKeyId: process.env.MINIO_ACCESS_KEY ?? 'minioadmin',
-      secretAccessKey: process.env.MINIO_SECRET_KEY ?? 'minioadmin'
+      accessKeyId: process.env.RUSTFS_ACCESS_KEY ?? 'rustfsadmin',
+      secretAccessKey: process.env.RUSTFS_SECRET_KEY ?? 'rustfsadmin'
     });
-    minioStorages.push(s3);
+    rustfsStorages.push(s3);
     objectStorage = s3;
   } else {
     objectStorage = new MemoryObjectStorage();
@@ -52,9 +52,9 @@ export function createS3TestStorageSuite(options: S3TestFactoryOptions) {
   return createTestStorageSuite(options, objectStorage);
 }
 
-/** Remove all objects created by MinIO-backed suites in the current test. */
+/** Remove all objects created by RustFS-backed suites in the current test. */
 export async function cleanupS3TestStorage(): Promise<void> {
-  for (const storage of minioStorages.splice(0)) {
+  for (const storage of rustfsStorages.splice(0)) {
     await storage.deletePrefix('bucket-data/');
   }
 }