@powersync/service-module-mongodb-storage 0.21.0 → 0.22.0

package/test/src/storage_compacting.test.ts

@@ -7,6 +7,7 @@ import { CompactionLease } from '@module/storage/implementation/v3/CompactionLea
 import { BucketDataDocumentV3 } from '@module/storage/implementation/v3/models.js';
 import { ObjectStorageError } from '@module/storage/implementation/v3/object-storage/ObjectStorage.js';
 import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
+import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 import { replicaIdToSubkey } from '@module/utils/util.js';
 import { logger as defaultLogger } from '@powersync/lib-services-framework';
 import {
@@ -19,6 +20,8 @@ import {
 import { bucketRequest, compactActive, register, test_utils } from '@powersync/service-core-tests';
 import * as bson from 'bson';
 import { describe, expect, test, vi } from 'vitest';
+import { env } from './env.js';
+import { MemoryObjectStorage } from './helpers/MemoryObjectStorage.js';
 import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
 function makeOp(
@@ -460,7 +463,7 @@ bucket_definitions:
 
   test('aborting scheduled compaction does not reschedule the remaining batch', async () => {
     const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
-    const buckets = ['first[]', 'second[]', 'third[]'];
+    const buckets = Array.from({ length: 8 }, (_, index) => `bucket${index}[]`);
     const documents = buckets.map((bucket) =>
       serializeBucketData(bucket, [makeOp(2, bucket, bucket, { ...ctx, bucket }, sourceTableId)])
     );
@@ -499,8 +502,11 @@ bucket_definitions:
     const states = await bucketStateCollection.find({ '_id.b': { $in: buckets } }).toArray();
     const completed = states.filter((state) => state.compacted_state != null);
     const remaining = states.filter((state) => state.compacted_state == null);
-    expect(completed).toHaveLength(1);
-    expect(remaining).toHaveLength(2);
+    // Already-started buckets may finish after abort, but the queued remainder
+    // must not be claimed or rescheduled.
+    expect(completed.length).toBeGreaterThanOrEqual(1);
+    expect(completed.length).toBeLessThanOrEqual(2);
+    expect(remaining.length).toBeGreaterThanOrEqual(6);
     for (const state of remaining) {
       expect(state.next_compact_check).toEqual(new Date(0));
       expect(state.compact_lease).toBeUndefined();
@@ -1637,8 +1643,8 @@ bucket_definitions:
 describe('Streaming compactor', () => {
   const BUCKET = 'global[]';
 
-  async function setupV3() {
-    await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
+  async function setupV3(factoryGenerator = INITIALIZED_MONGO_STORAGE_FACTORY) {
+    await using factory = await factoryGenerator.factory();
     const syncRules = await factory.updateSyncRules(
       updateSyncRulesFromYaml(
         `
@@ -1769,6 +1775,252 @@ bucket_definitions:
     });
   });
 
+  test.each([
+    { payloadSize: 100, workers: undefined, abort: false },
+    { payloadSize: 450_000, workers: undefined, abort: false },
+    { payloadSize: 450_000, workers: 4, abort: false },
+    { payloadSize: 100, workers: 1, abort: false },
+    { payloadSize: 450_000, workers: 4, abort: true }
+  ])('chunk compaction shares configured workers across runs: %o', async ({ payloadSize, workers, abort }) => {
+    const concurrency = workers ?? 4;
+    const objectStorage = new MemoryObjectStorage();
+    const generator = mongoTestStorageFactoryGenerator({
+      url: env.MONGO_TEST_URL,
+      isCI: env.CI,
+      objectStorage,
+      chunkCompactionConcurrency: workers,
+      inlineThresholdBytes: 0
+    });
+    const { bucketStorage, syncRules, collection, bucketStateCollection, ctx, sourceTableId, db } =
+      await setupV3(generator);
+    const bucketCount = 12;
+    for (let i = 0; i < bucketCount; i++) {
+      const bucket = `global[${i}]`;
+      const documents = [1, 2].map((id) =>
+        serializeBucketData(bucket, [makeOp(id, String(id), 'x'.repeat(payloadSize), ctx, sourceTableId)])
+      );
+      await collection.insertMany(documents);
+      await bucketStateCollection.insertOne({
+        _id: { d: ctx.definitionId, b: bucket },
+        last_op: 2n,
+        next_compact_check: new Date(0),
+        first_uncompacted_write: new Date(0),
+        bucket_stats: {
+          count: 2,
+          bytes: BigInt(documents.reduce((sum, doc) => sum + doc.size, 0)),
+          chunks: 2
+        }
+      });
+    }
+
+    const gate = Promise.withResolvers<void>();
+    const originalPut = objectStorage.put.bind(objectStorage);
+    let uploads = 0;
+    const put = vi.spyOn(objectStorage, 'put').mockImplementation(async (...args) => {
+      uploads++;
+      await gate.promise;
+      await originalPut(...args);
+    });
+    const controller = new AbortController();
+    let settled = false;
+    const runs = Promise.allSettled(
+      Array.from({ length: 2 }, () =>
+        bucketStorage.factory
+          .getInstance(syncRules)
+          .compactInitialReplication({ maxOpId: 2n, signal: controller.signal })
+      )
+    ).finally(() => {
+      settled = true;
+    });
+    try {
+      await vi.waitFor(() => expect(uploads).toBe(concurrency));
+      // Let other ready workers reach admission; the held uploads must prevent
+      // any additional payloads from reaching S3, across both compactor instances.
+      await new Promise((resolve) => setTimeout(resolve, 50));
+      expect(uploads).toBe(concurrency);
+      expect(await bucketStateCollection.countDocuments({ compact_lease: { $exists: true } })).toBeLessThanOrEqual(
+        concurrency
+      );
+      expect(await collection.countDocuments()).toBe(bucketCount * 2);
+      if (abort) {
+        controller.abort();
+        await new Promise((resolve) => setTimeout(resolve, 50));
+        expect(settled).toBe(false);
+      }
+    } finally {
+      gate.resolve();
+      await runs;
+      put.mockRestore();
+    }
+
+    const results = await runs;
+    if (abort) {
+      expect(results.every((result) => result.status === 'rejected')).toBe(true);
+      expect(await bucketStateCollection.countDocuments({ compact_lease: { $exists: true } })).toBe(0);
+      // Cancellation must return all worker slots.
+      expect(await bucketStorage.compactInitialReplication({ maxOpId: 2n })).toEqual({ buckets: bucketCount });
+    } else {
+      expect(results.every((result) => result.status === 'fulfilled')).toBe(true);
+      expect(results.reduce((sum, result) => sum + (result.status === 'fulfilled' ? result.value.buckets : 0), 0)).toBe(
+        bucketCount
+      );
+    }
+
+    const documents = await collection.find().toArray();
+    expect(documents).toHaveLength(bucketCount);
+    expect(documents.every((doc) => doc.count === 2 && doc.checksum === 21n && doc.storage_ref != null)).toBe(true);
+    expect(
+      await bucketStateCollection.countDocuments({ 'compacted_state.op_id': 2n, compact_lease: { $exists: false } })
+    ).toBe(bucketCount);
+    const usage = await db.objectStorageUsage.find({ '_id.g': ctx.replicationStreamId }).toArray();
+    const activeBytes = usage.reduce((sum, doc) => sum + BigInt(doc.definitions[ctx.definitionId] ?? 0), 0n);
+    expect(activeBytes).toBe(documents.reduce((sum, doc) => sum + BigInt(doc.storage_ref!.file_size), 0n));
+  });
+
+  test.each([2, 3])('failed chunk worker stops queued work and drains siblings (%s workers)', async (workerCount) => {
+    const { bucketStorage } = await setupV3();
+    const compactor = bucketStorage.createMongoCompactor({}) as any;
+    const failure = new Error('worker failed');
+    const failing = Promise.withResolvers<void>();
+    const draining = Promise.withResolvers<void>();
+    const started: number[] = [];
+    let settled = false;
+    // The factory has two slots. With three workers, one also waits for a slot
+    // when its sibling fails; it must release that slot without starting work.
+    const run = compactor
+      .runChunkCompactionWorkers(
+        [0, 1, 2, 3, 4],
+        Array.from({ length: workerCount }, () => compactor.objectStorageUsage),
+        async (bucket: number) => {
+          started.push(bucket);
+          if (bucket === 0) await failing.promise;
+          if (bucket === 1) await draining.promise;
+        }
+      )
+      .then(
+        () => undefined,
+        (error: unknown) => error
+      )
+      .finally(() => {
+        settled = true;
+      });
+    try {
+      await vi.waitFor(() => expect(started).toEqual([0, 1]));
+      failing.reject(failure);
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      expect(started).toEqual([0, 1]);
+      expect(settled).toBe(false);
+    } finally {
+      failing.resolve();
+      draining.resolve();
+      await run;
+    }
+    expect(await run).toBe(failure);
+    expect(started).toEqual([0, 1]);
+    expect(bucketStorage.factory.chunkCompactionSlots.getValue()).toBe(2);
+  });
+
+  test('unforced chunk workers drain before full compaction and reclassify aged buckets on the next scan', async () => {
+    const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
+    const buckets = ['chunk1[]', 'chunk2[]', 'promoted[]', 'full1[]', 'full2[]'];
+    for (const bucket of buckets) {
+      const documents = Array.from({ length: 8 }, (_, index) =>
+        serializeBucketData(bucket, [makeOp(index + 1, String(index), 'data', { ...ctx, bucket }, sourceTableId)])
+      );
+      await collection.insertMany(documents);
+      await bucketStateCollection.insertOne({
+        _id: { d: ctx.definitionId, b: bucket },
+        last_op: 8n,
+        next_compact_check: new Date(0),
+        first_uncompacted_write: bucket.startsWith('full') ? new Date(0) : new Date(),
+        bucket_stats: { count: 8, bytes: BigInt(documents.reduce((sum, doc) => sum + doc.size, 0)), chunks: 8 }
+      });
+    }
+
+    const originalClaim = CompactionLease.claim.bind(CompactionLease);
+    const jobStartedAt = new Date();
+    let serverTime = jobStartedAt;
+    let promoted = false;
+    const claim = vi.spyOn(CompactionLease, 'claim').mockImplementation(async (...args) => {
+      if (!promoted && (args[1]._id as { b: string })?.b === 'promoted[]') {
+        promoted = true;
+        // Advance the server clock past the full-compaction interval without
+        // changing bucket state. A fixed scan timestamp would retry forever.
+        serverTime = new Date(jobStartedAt.getTime() + 3 * 60 * 60 * 1000);
+      }
+      const lease = await originalClaim(...args);
+      if (lease != null) {
+        Object.defineProperty(lease, 'startedAt', { value: serverTime });
+      }
+      return lease;
+    });
+
+    const compactor = bucketStorage.createMongoCompactor({ maxOpId: 8n });
+    const internal = compactor as any;
+    let timeReads = 0;
+    const clock = vi.spyOn(internal, 'readCompactionTime').mockImplementation(async () => {
+      if (++timeReads > 5) {
+        throw new Error('Compaction did not advance after the bucket aged');
+      }
+      return serverTime;
+    });
+    const scan = vi.spyOn(internal, 'findScheduledBucketBatch');
+    const originalChunks = internal.compactSingleBucketChunks.bind(compactor);
+    const originalFull = internal.compactSingleBucketFully.bind(compactor);
+    const gate = Promise.withResolvers<void>();
+    let activeChunks = 0;
+    let activeFull = 0;
+    let maxActiveFull = 0;
+    const fullBuckets: string[] = [];
+    const chunks = vi.spyOn(internal, 'compactSingleBucketChunks').mockImplementation(async (...args) => {
+      activeChunks++;
+      try {
+        await gate.promise;
+        return await originalChunks(...args);
+      } finally {
+        activeChunks--;
+      }
+    });
+    const full = vi.spyOn(internal, 'compactSingleBucketFully').mockImplementation(async (...args) => {
+      expect(activeChunks).toBe(0);
+      activeFull++;
+      maxActiveFull = Math.max(maxActiveFull, activeFull);
+      fullBuckets.push((args[0] as any).state._id.b);
+      try {
+        return await originalFull(...args);
+      } finally {
+        activeFull--;
+      }
+    });
+    const run = compactor.compact();
+    try {
+      await vi.waitFor(() => expect(activeChunks).toBe(2));
+      expect(full).not.toHaveBeenCalled();
+    } finally {
+      gate.resolve();
+      try {
+        await run;
+        expect(timeReads).toBe(3); // Job start, first batch, and promoted bucket's batch.
+        expect(scan).toHaveBeenCalledTimes(3); // Includes the final empty scan.
+        for (const [cutoff] of scan.mock.calls) {
+          expect(cutoff).toEqual(jobStartedAt);
+        }
+      } finally {
+        clock.mockRestore();
+        scan.mockRestore();
+        claim.mockRestore();
+        chunks.mockRestore();
+        full.mockRestore();
+      }
+    }
+    expect(await run).toBe(5);
+    expect(promoted).toBe(true);
+    expect(maxActiveFull).toBe(1);
+    expect(fullBuckets.sort()).toEqual(['full1[]', 'full2[]', 'promoted[]']);
+    expect(await bucketStateCollection.countDocuments({ compact_lease: { $exists: true } })).toBe(0);
+    expect(await collection.countDocuments()).toBe(5);
+  });
+
   test('capped chunk compaction repairs stale bucket stats after a committed first merge', async () => {
     const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
     const operations = [

package/test/src/storage_s3_reading.test.ts

@@ -2,7 +2,7 @@ import { DeleteObjectsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { bucketRequest, test_utils } from '@powersync/service-core-tests';
 import * as bson from 'bson';
-import { describe, expect, test } from 'vitest';
+import { describe, expect, test, vi } from 'vitest';
 import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
 import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
 import { hydrateBucketDataDocuments } from '../../src/storage/implementation/v3/object-storage/BucketDataObjectStorage.js';
@@ -576,6 +576,52 @@ describe('S3 object storage reads', () => {
     expect(maxActiveOperations).toBe(4);
   });
 
+  test('drains sibling downloads before reporting a hydration failure', async () => {
+    const objectStorage = new MemoryObjectStorage();
+    const gate = Promise.withResolvers<void>();
+    const failure = new Error('download failed');
+    const documents = ['failed', 'pending'].map((path, index) => ({
+      _id: { b: 'bucket', o: BigInt(index + 1) },
+      min_op: BigInt(index + 1),
+      checksum: 0n,
+      count: 0,
+      size: 1,
+      storage_ref: { path, file_size: 1 }
+    }));
+    const get = vi.spyOn(objectStorage, 'get').mockImplementation(async (path) => {
+      if (path === 'failed') {
+        throw failure;
+      }
+      await gate.promise;
+      return {
+        data: bson.serialize({ ops: [] }),
+        metadata: { contentType: 'application/bson', contentEncoding: null }
+      };
+    });
+    let settled = false;
+    const result = hydrateBucketDataDocuments(documents, objectStorage, {}).then(
+      () => {
+        settled = true;
+        return undefined;
+      },
+      (error) => {
+        settled = true;
+        return error;
+      }
+    );
+    try {
+      await vi.waitFor(() => expect(get).toHaveBeenCalledTimes(2));
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      expect(settled).toBe(false);
+    } finally {
+      gate.resolve();
+      await result;
+      get.mockRestore();
+    }
+    expect(await result).toBe(failure);
+    expect(documents[1]).toHaveProperty('ops', []);
+  });
+
   test('aborts active object downloads', async () => {
     const objectStorage = new MemoryObjectStorage();
     const controller = new AbortController();