@powersync/service-module-mongodb-storage 0.18.2 → 0.19.0

package/test/src/helpers/s3TestFactory.ts

@@ -0,0 +1,69 @@
+import { ObjectStorage } from '@module/storage/implementation/v3/object-storage/ObjectStorage.js';
+import { S3ObjectStorage } from '@module/storage/implementation/v3/object-storage/S3ObjectStorage.js';
+import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { randomUUID } from 'node:crypto';
+import { MemoryObjectStorage } from './MemoryObjectStorage.js';
+
+const minioStorages: S3ObjectStorage[] = [];
+
+export interface S3TestFactoryOptions {
+  url: string;
+  isCI: boolean;
+  inlineThresholdBytes?: number;
+}
+
+function createTestStorageSuite(options: S3TestFactoryOptions, objectStorage: ObjectStorage) {
+  return {
+    objectStorage,
+    factoryGen: mongoTestStorageFactoryGenerator({
+      url: options.url,
+      isCI: options.isCI,
+      objectStorage,
+      inlineThresholdBytes: options.inlineThresholdBytes ?? 0
+    })
+  };
+}
+
+/**
+ * Creates an ObjectStorage instance for S3 tests.
+ * Set MINIO_ENDPOINT to switch all S3 tests from MemoryObjectStorage
+ * to a real MinIO/S3 endpoint.
+ *   MINIO_ENDPOINT=http://localhost:9000
+ */
+export function createS3TestStorageSuite(options: S3TestFactoryOptions) {
+  const minioEndpoint = process.env.MINIO_ENDPOINT;
+  let objectStorage: ObjectStorage;
+  if (minioEndpoint) {
+    const s3 = new S3ObjectStorage({
+      bucket: 'powersync-s3-test',
+      region: 'us-east-1',
+      prefix: `test-${process.pid}-${randomUUID()}`,
+      endpoint: minioEndpoint,
+      forcePathStyle: true,
+      accessKeyId: process.env.MINIO_ACCESS_KEY ?? 'minioadmin',
+      secretAccessKey: process.env.MINIO_SECRET_KEY ?? 'minioadmin'
+    });
+    minioStorages.push(s3);
+    objectStorage = s3;
+  } else {
+    objectStorage = new MemoryObjectStorage();
+  }
+
+  return createTestStorageSuite(options, objectStorage);
+}
+
+/** Remove all objects created by MinIO-backed suites in the current test. */
+export async function cleanupS3TestStorage(): Promise<void> {
+  for (const storage of minioStorages.splice(0)) {
+    await storage.deletePrefix('bucket-data/');
+  }
+}
+
+/** Creates an explicitly memory-backed suite for tests that inspect stored objects. */
+export function createMemoryS3TestStorageSuite(options: S3TestFactoryOptions) {
+  const objectStorage = new MemoryObjectStorage();
+  return {
+    ...createTestStorageSuite(options, objectStorage),
+    objectStorage
+  };
+}

package/test/src/object_storage_lifecycle.test.ts

@@ -0,0 +1,59 @@
+import { createHash } from 'node:crypto';
+import { describe, expect, test } from 'vitest';
+import { ObjectStorageLifecycle } from '../../src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js';
+import { MemoryObjectStorage } from './helpers/MemoryObjectStorage.js';
+
+describe('ObjectStorageLifecycle', () => {
+  test('escapes bucket names as path segments', () => {
+    const lifecycle = new ObjectStorageLifecycle(null as any, 12, null as any);
+
+    const path = lifecycle.allocatePath('definition', 'bucket/with spaces?and%escapes', 1n, 2n);
+
+    expect(path).toMatch(/^bucket-data\/12\/definition\/bucket%2Fwith%20spaces%3Fand%25escapes\/1-2-[0-9a-f-]+\.bson$/);
+  });
+
+  test('hashes bucket path segments longer than 128 bytes', () => {
+    const lifecycle = new ObjectStorageLifecycle(null as any, 12, null as any);
+    const bucket = '💾/long bucket parameter/'.repeat(20);
+    const hash = createHash('sha256').update(bucket).digest('base64url');
+
+    const path = lifecycle.allocatePath('definition', bucket, 1n, 2n);
+
+    expect(path).toMatch(new RegExp(`^bucket-data/12/definition/sha256-${hash}/1-2-[0-9a-f-]+\\.bson$`));
+  });
+
+  test('deletePrefix removes only objects under the exact prefix', async () => {
+    const objectStorage = new MemoryObjectStorage();
+    const lifecycle = new ObjectStorageLifecycle(null as any, 1, objectStorage);
+    for (const path of [
+      'bucket-data/1/definition/object-1.bson',
+      'bucket-data/12/definition/object-2.bson',
+      'bucket-data/1-other/definition/object-3.bson'
+    ]) {
+      await objectStorage.put(path, new Uint8Array(), {
+        contentType: 'application/bson',
+        contentEncoding: null
+      });
+    }
+
+    await expect(lifecycle.deletePrefix(lifecycle.streamPrefix())).resolves.toEqual({ objectCount: 1 });
+    expect([...objectStorage.store.keys()]).toEqual([
+      'bucket-data/12/definition/object-2.bson',
+      'bucket-data/1-other/definition/object-3.bson'
+    ]);
+  });
+
+  test('definition prefixes do not match ids with the same string prefix', async () => {
+    const objectStorage = new MemoryObjectStorage();
+    const lifecycle = new ObjectStorageLifecycle(null as any, 7, objectStorage);
+    for (const path of ['bucket-data/7/1/object-1.bson', 'bucket-data/7/12/object-2.bson']) {
+      await objectStorage.put(path, new Uint8Array(), {
+        contentType: 'application/bson',
+        contentEncoding: null
+      });
+    }
+
+    await lifecycle.deletePrefix(lifecycle.definitionPrefix('1'));
+    expect([...objectStorage.store.keys()]).toEqual(['bucket-data/7/12/object-2.bson']);
+  });
+});

package/test/src/setup.ts

@@ -1,6 +1,7 @@
 import { container } from '@powersync/lib-services-framework';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
-import { beforeAll, beforeEach } from 'vitest';
+import { afterEach, beforeAll, beforeEach } from 'vitest';
+import { cleanupS3TestStorage } from './helpers/s3TestFactory.js';
 
 beforeAll(async () => {
   // Executes for every test file
@@ -10,3 +11,7 @@ beforeAll(async () => {
 beforeEach(async () => {
   METRICS_HELPER.resetMetrics();
 });
+
+afterEach(async () => {
+  await cleanupS3TestStorage();
+});

package/test/src/storage.test.ts

@@ -1,6 +1,8 @@
 import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
-import { register } from '@powersync/service-core-tests';
-import { describe } from 'vitest';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { register, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test } from 'vitest';
 import { env } from './env.js';
 import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
@@ -17,6 +19,218 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
 
   describe(`Mongo Sync Bucket Storage - Checkpoints - v${storageVersion}`, () =>
     register.registerDataStorageCheckpointTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion }));
+
+  describe(`Mongo Sync Bucket Storage - write checkpoint metadata - v${storageVersion}`, () => {
+    test('uses checkpoint_requested_at as the client-requested checkpoint marker', async () => {
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+bucket_definitions:
+  global:
+    data: []
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+
+      // user1 has no existing row, so this covers updateMany with upsert.
+      // The initial request stores checkpoint id 42 at source head 5/0.
+      const requestedResult = await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '5/0' }, checkpoint_request_id: 42n }
+      ]);
+      expect(requestedResult.writeCheckpoints.get('user1')).toEqual(42n);
+      expect(requestedResult.shouldAdvance).toBe(true);
+
+      const requested = await factory.db.write_checkpoints.findOne({ user_id: 'user1' });
+      expect(requested?.client_id).toEqual(42n);
+      expect(requested?.lsns).toEqual({ '1': '5/0' });
+      expect(requested?.processed_at_lsn).toBeNull();
+      expect(requested?.checkpoint_requested_at).toBeInstanceOf(Date);
+
+      // Request id 41 is lower than the stored id 42, so the entire stale
+      // request is ignored, including its newer 6/0 source head.
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '6/0' }, checkpoint_request_id: 41n }
+      ]);
+      const stale = await factory.db.write_checkpoints.findOne({ user_id: 'user1' });
+      expect(stale?.checkpoint_requested_at).toEqual(requested?.checkpoint_requested_at);
+
+      const expiredRequestedAt = new Date('2024-01-01T00:00:00.000Z');
+      await factory.db.write_checkpoints.updateOne(
+        { user_id: 'user1' },
+        { $set: { checkpoint_requested_at: expiredRequestedAt } }
+      );
+      // Although the previous incoming request was 41, the stored id is still
+      // 42. This is therefore an equal-id retry of the original request.
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '6/0' }, checkpoint_request_id: 42n }
+      ]);
+      const retried = await factory.db.write_checkpoints.findOne({ user_id: 'user1' });
+      // Retrying the current id refreshes its retention timestamp without
+      // replacing the original source head or resetting its processed state.
+      expect(retried?.checkpoint_requested_at).toBeInstanceOf(Date);
+      expect(retried!.checkpoint_requested_at!.getTime()).toBeGreaterThan(expiredRequestedAt.getTime());
+      expect(retried?.lsns).toEqual({ '1': '5/0' });
+      expect(retried?.processed_at_lsn).toBeNull();
+
+      await factory.db.write_checkpoints.updateOne(
+        { user_id: 'user1' },
+        { $set: { checkpoint_requested_at: expiredRequestedAt } }
+      );
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '6/0' }, checkpoint_request_id: 43n }
+      ]);
+      const advanced = await factory.db.write_checkpoints.findOne({ user_id: 'user1' });
+      // A greater id refreshes retention and advances the stored checkpoint.
+      expect(advanced?.checkpoint_requested_at).toBeInstanceOf(Date);
+      expect(advanced!.checkpoint_requested_at!.getTime()).toBeGreaterThan(expiredRequestedAt.getTime());
+      expect(advanced?.client_id).toEqual(43n);
+      expect(advanced?.lsns).toEqual({ '1': '6/0' });
+
+      await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '7/0' } }]);
+      const generated = await factory.db.write_checkpoints.findOne({ user_id: 'user1' });
+      // Generated checkpoints unset the field, keeping the document out of the
+      // partial checkpoint_requested_at index.
+      expect(generated).not.toBeNull();
+      expect(generated?.checkpoint_requested_at).toBeUndefined();
+
+      await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user2', heads: { '1': '8/0' }, checkpoint_request_id: 50n }
+      ]);
+      await factory.db.write_checkpoints.updateOne(
+        { user_id: 'user2' },
+        { $set: { checkpoint_requested_at: new Date('2024-01-01T00:00:00.000Z') } }
+      );
+      await bucketStorage.compact({
+        compactBuckets: [],
+        deleteCheckpointRequestsBefore: new Date('2024-02-01T00:00:00.000Z')
+      });
+      // The request is expired but still pending, so compaction must retain it.
+      await expect(factory.db.write_checkpoints.findOne({ user_id: 'user2' })).resolves.not.toBeNull();
+
+      // Advancing replication to the request head sets processed_at_lsn.
+      await using managedWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      await managedWriter.markAllSnapshotDone('1/1');
+      await managedWriter.keepalive('8/0');
+
+      await bucketStorage.compact({
+        compactBuckets: [],
+        deleteCheckpointRequestsBefore: new Date('2024-02-01T00:00:00.000Z')
+      });
+      // The request is now both expired and processed, so it can be removed.
+      await expect(factory.db.write_checkpoints.findOne({ user_id: 'user2' })).resolves.toBeNull();
+
+      bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      await writer.markAllSnapshotDone('1/1');
+      const customCheckpointRequestedAt = new Date('2024-01-01T00:00:00.000Z');
+      writer.addCustomWriteCheckpoint({
+        user_id: 'custom1',
+        checkpoint: 51n,
+        checkpoint_requested_at: customCheckpointRequestedAt
+      });
+      await writer.flush();
+      const customRequested = await factory.db.custom_write_checkpoints.findOne({ user_id: 'custom1' });
+      expect(customRequested?.checkpoint_requested_at).toEqual(customCheckpointRequestedAt);
+
+      writer.addCustomWriteCheckpoint({
+        user_id: 'custom1',
+        checkpoint: 52n
+      });
+      await writer.flush();
+      const customGenerated = await factory.db.custom_write_checkpoints.findOne({ user_id: 'custom1' });
+      expect(customGenerated).not.toBeNull();
+      expect(customGenerated?.checkpoint_requested_at).toBeUndefined();
+    });
+
+    /**
+     * It's extremely rare (but technically possible) to have duplicate write checkpoint records
+     * for a full user_id. This is normally not an issue for checkpoints requested from `write-checkpoint2.json`,
+     * but it can be for requests in the `sync/checkpoint-request` flow.
+     * That flow seeds the client with the current latest state (if available). We need to ensure that the client
+     * does NOT start at a lower bound if there are duplicate records.
+     */
+    test('updates duplicate managed rows on supplied checkpoint requests', async () => {
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+bucket_definitions:
+  global:
+    data: []
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+
+      // Replicate an extremely rare possibility where there are multiple records
+      await factory.db.write_checkpoints.insertMany([
+        {
+          _id: new mongo.ObjectId('000000000000000000000001'),
+          user_id: 'user1',
+          client_id: 50n,
+          lsns: { '1': '50/0' },
+          processed_at_lsn: '50/0'
+        },
+        {
+          _id: new mongo.ObjectId('000000000000000000000002'),
+          user_id: 'user1',
+          client_id: 42n,
+          lsns: { '1': '42/0' },
+          processed_at_lsn: null,
+          checkpoint_requested_at: new Date('2024-01-01T00:00:00.000Z')
+        }
+      ]);
+
+      // Simulate a client without a sequence state trying to seed its state.
+      const seeded = await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': '1/0' }, checkpoint_request_id: 1n }
+      ]);
+
+      // The result should be the current max state.
+      const seededCheckpoint = seeded.writeCheckpoints.get('user1')!;
+      expect(seededCheckpoint).toEqual(50n);
+      expect(seeded.shouldAdvance).toBe(false);
+
+      // Simulate a client using the above state for its next checkpoint request.
+      const nextCheckpoint = seededCheckpoint + 1n;
+      const incremented = await bucketStorage.createManagedWriteCheckpoints([
+        { user_id: 'user1', heads: { '1': `${nextCheckpoint}/0` }, checkpoint_request_id: nextCheckpoint }
+      ]);
+
+      // This should be accepted by the service.
+      expect(incremented.writeCheckpoints.get('user1')).toEqual(nextCheckpoint);
+      expect(incremented.shouldAdvance).toBe(true);
+
+      // The service should now reconcile all duplicates to the latest state.
+      // Duplicates can now also automatically be deleted when they expire.
+      const docs = await factory.db.write_checkpoints.find({ user_id: 'user1' }, { sort: { _id: 1 } }).toArray();
+      expect(
+        docs.map((doc) => ({
+          client_id: doc.client_id,
+          lsn: doc.lsns['1'],
+          processed_at_lsn: doc.processed_at_lsn,
+          is_checkpoint_request: doc.checkpoint_requested_at != null
+        }))
+      ).toEqual([
+        {
+          client_id: nextCheckpoint,
+          lsn: '51/0',
+          processed_at_lsn: null,
+          is_checkpoint_request: true
+        },
+        {
+          client_id: nextCheckpoint,
+          lsn: '51/0',
+          processed_at_lsn: null,
+          is_checkpoint_request: true
+        }
+      ]);
+    });
+  });
 }
 
 describe('Sync Bucket Validation', register.registerBucketValidationTests);