@powersync/service-module-mongodb-storage 0.20.0 → 0.22.0

package/test/src/chunk_compaction_config.test.ts

@@ -0,0 +1,38 @@
+import { MongoStorageConfig, normalizeChunkCompactionConcurrency } from '@module/types/types.js';
+import { describe, expect, test } from 'vitest';
+
+describe('chunk compaction concurrency configuration', () => {
+  test('defaults to two workers', () => {
+    expect(normalizeChunkCompactionConcurrency(undefined)).toBe(2);
+  });
+
+  test('defaults to four workers with object storage', () => {
+    expect(normalizeChunkCompactionConcurrency(undefined, true)).toBe(4);
+  });
+
+  test.each([false, true])('preserves an explicit override with object storage %s', (hasObjectStorage) => {
+    expect(normalizeChunkCompactionConcurrency(3, hasObjectStorage)).toBe(3);
+  });
+
+  test('decodes a configured worker count', () => {
+    const config = MongoStorageConfig.decode({
+      type: 'mongodb',
+      uri: 'mongodb://localhost:27017/powersync',
+      chunk_compaction_concurrency: 4
+    });
+    expect(normalizeChunkCompactionConcurrency(config.chunk_compaction_concurrency)).toBe(4);
+  });
+
+  test.each([1, 64])('accepts boundary worker count %s', (value) => {
+    expect(normalizeChunkCompactionConcurrency(value)).toBe(value);
+  });
+
+  test.each([0, -1, 1.5, 65, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1])(
+    'rejects invalid worker count %s',
+    (value) => {
+      expect(() => normalizeChunkCompactionConcurrency(value)).toThrow(
+        'storage.chunk_compaction_concurrency must be an integer between 1 and 64'
+      );
+    }
+  );
+});

package/test/src/cleanup-stopped-sync-configs.test.ts

@@ -324,7 +324,7 @@ streams:
   test('drops current_data when a table becomes event-only but is kept by a live event', async () => {
     await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
 
-    // The first config syncs `todos` data and also fires events for it.
+    // The first config syncs `todos` data and also evaluates the audit event for its rows.
     const first = await factory.updateSyncRules(
       updateSyncRulesFromYaml(
         `
@@ -360,15 +360,22 @@ event_definitions:
       },
       afterReplicaId: test_utils.rid('todo-1')
     });
+    const eventDefinitionId = [...resolved.tables[0].eventDefinitionIds!][0];
+    firstWriter.addCustomWriteCheckpoint({
+      user_id: 'user-1',
+      checkpoint: 1n,
+      event_id: eventDefinitionId
+    });
     await firstWriter.markAllSnapshotDone('1/1');
     await firstWriter.commit('1/1');
 
     const sourceTableId = resolved.tables[0].id as bson.ObjectId;
     const sourceRecordsCollection = db.sourceRecords(first.replicationStreamId, sourceTableId).collectionName;
     const ownerDefinitionId = first.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
-    expect(
-      (await db.sourceTables(first.replicationStreamId).findOne({ _id: sourceTableId }))?.bucket_data_source_ids
-    ).toEqual([ownerDefinitionId]);
+    const initialSourceTable = await db.sourceTables(first.replicationStreamId).findOne({ _id: sourceTableId });
+    expect(initialSourceTable?.bucket_data_source_ids).toEqual([ownerDefinitionId]);
+    expect(initialSourceTable?.event_definition_ids).toHaveLength(1);
+    const eventDefinitionIds = initialSourceTable!.event_definition_ids;
     // The prior snapshot is persisted in current_data.
     expect(await collectionExists(db, sourceRecordsCollection)).toBe(true);
 
@@ -414,13 +421,22 @@ event_definitions:
       sourceTablesDeleted: 0
     });
 
-    // The source table row survives (the live event still needs it), narrowed to empty
-    // memberships, but its now-unused current_data collection is dropped.
+    // The source table row survives (the live event still needs it), narrowed to empty data and
+    // parameter memberships, but its event membership remains and current_data is dropped.
     const sourceTable = await db.sourceTables(first.replicationStreamId).findOne({ _id: sourceTableId });
     expect(sourceTable).not.toBeNull();
     expect(sourceTable?.bucket_data_source_ids).toEqual([]);
     expect(sourceTable?.parameter_lookup_source_ids).toEqual([]);
+    expect(sourceTable?.event_definition_ids).toEqual(eventDefinitionIds);
     expect(await collectionExists(db, sourceRecordsCollection)).toBe(false);
+    await expect(
+      db
+        .customCheckpointRequests({
+          replicationStreamId: first.replicationStreamId,
+          eventId: eventDefinitionId
+        })
+        .findOne({ user_id: 'user-1' })
+    ).resolves.not.toBeNull();
 
     const streamDoc = (await db.sync_rules.findOne({ _id: first.replicationStreamId })) as ReplicationStreamDocumentV3;
     expect(streamDoc.sync_configs.map((config) => config.state)).toEqual([storage.SyncRuleState.ACTIVE]);
@@ -429,9 +445,9 @@ event_definitions:
   test('cleans up event-only source tables no longer triggered by a live sync config', async () => {
     await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
 
-    // The first config syncs `todos` and additionally fires events for `audit_log`. The
-    // `audit_log` table is referenced only by the event trigger, so its source table carries
-    // empty membership arrays (an event-only table).
+    // The first config syncs `todos` and additionally evaluates the audit event for `audit_log`. The
+    // `audit_log` table is referenced only by the event trigger, so its source table has only an
+    // event membership (an event-only table).
     const first = await factory.updateSyncRules(
       updateSyncRulesFromYaml(
         `
@@ -485,21 +501,27 @@ event_definitions:
       },
       afterReplicaId: test_utils.rid('audit-1')
     });
+    const auditEventDefinitionId = [...auditTable.eventDefinitionIds!][0];
+    firstWriter.addCustomWriteCheckpoint({
+      user_id: 'audit-user',
+      checkpoint: 1n,
+      event_id: auditEventDefinitionId
+    });
     await firstWriter.markAllSnapshotDone('1/1');
     await firstWriter.commit('1/1');
 
     const todosTableId = todosTable.id as bson.ObjectId;
     const auditTableId = auditTable.id as bson.ObjectId;
-    // The event-only table is persisted with empty membership arrays, so the membership filter
-    // never selects it.
+    // The event-only table has empty bucket and parameter memberships, plus its event id.
     const auditSourceTable = await db.sourceTables(first.replicationStreamId).findOne({ _id: auditTableId });
     expect(auditSourceTable?.bucket_data_source_ids).toEqual([]);
     expect(auditSourceTable?.parameter_lookup_source_ids).toEqual([]);
+    expect(auditSourceTable?.event_definition_ids).toHaveLength(1);
     const auditRecordsCollection = db.sourceRecords(first.replicationStreamId, auditTableId).collectionName;
 
     // The second (active) config keeps the same `by_owner` stream but drops the audit event.
-    // The shared bucket definition stays in use, so no bucket/parameter ids become unused and
-    // the membership-cleanup block is skipped entirely.
+    // The shared bucket definition stays in use, while the unused event id selects this table for
+    // membership cleanup.
     const second = await factory.updateSyncRules(
       updateSyncRulesFromYaml(
         `
@@ -537,6 +559,11 @@ streams:
     // The orphaned event-only table and its source records are removed...
     expect(await db.sourceTables(first.replicationStreamId).countDocuments({ _id: auditTableId })).toBe(0);
     expect(await collectionExists(db, auditRecordsCollection)).toBe(false);
+    const auditCheckpointCollection = db.customCheckpointRequests({
+      replicationStreamId: first.replicationStreamId,
+      eventId: auditEventDefinitionId
+    }).collectionName;
+    expect(await collectionExists(db, auditCheckpointCollection)).toBe(false);
     // ...while the data table still used by the live config is retained.
     expect(await db.sourceTables(first.replicationStreamId).countDocuments({ _id: todosTableId })).toBe(1);
 

package/test/src/compact-utils.test.ts

@@ -0,0 +1,226 @@
+import {
+  applyCompactionDelta,
+  applyStatsReplacement,
+  bucketStats,
+  chooseCompactionKind,
+  combineAdjacentStats,
+  combineChunkStats,
+  CompactIntervalConfig,
+  CompactionKind,
+  emptyBucketStats,
+  forcedCompactionKind,
+  fullCompactionCheckAt,
+  statsForDocument,
+  statsForDocuments
+} from '@module/storage/implementation/v3/compact-utils.js';
+import { BucketStateDocumentV3 } from '@module/storage/implementation/v3/models.js';
+import { describe, expect, test } from 'vitest';
+
+const INTERVAL_CONFIG: CompactIntervalConfig = {
+  minCompactChunkIntervalMs: 1_000,
+  minCompactFullIntervalMs: 10_000,
+  maxCompactFullIntervalMs: 100_000
+};
+
+function bucketState(overrides: Partial<BucketStateDocumentV3> = {}): BucketStateDocumentV3 {
+  return {
+    _id: { d: 'definition', b: 'bucket[]' },
+    last_op: 10n,
+    next_compact_check: new Date(0),
+    first_uncompacted_write: new Date(0),
+    bucket_stats: { count: 10, bytes: 100n, chunks: 1 },
+    ...overrides
+  };
+}
+
+function compactedState(overrides: Partial<NonNullable<BucketStateDocumentV3['compacted_state']>> = {}) {
+  return {
+    op_id: 5n,
+    checksum: 50n,
+    count: 5,
+    bytes: 50n,
+    chunks: 1,
+    at: new Date(0),
+    ...overrides
+  };
+}
+
+describe('V3 compact utilities', () => {
+  test.each([
+    {
+      name: 'uses the minimum interval before the first full compact',
+      state: bucketState(),
+      expected: new Date(10_000)
+    },
+    {
+      name: 'scales the interval by the uncompacted row ratio',
+      state: bucketState({
+        bucket_stats: { count: 100, bytes: 100n, chunks: 1 },
+        last_full_compact: { op_id: 90n, count: 90, puts: 90, at: new Date(0) }
+      }),
+      expected: new Date(100_000)
+    },
+    {
+      name: 'uses the maximum interval when there are no uncompacted rows',
+      state: bucketState({
+        bucket_stats: { count: 100, bytes: 100n, chunks: 1 },
+        last_full_compact: { op_id: 10n, count: 100, puts: 100, at: new Date(0) }
+      }),
+      expected: new Date(100_000)
+    }
+  ])('$name', ({ state, expected }) => {
+    expect(fullCompactionCheckAt(state, INTERVAL_CONFIG)).toEqual(expected);
+  });
+
+  test('rejects a scheduled bucket without an uncompacted write', () => {
+    expect(() => fullCompactionCheckAt(bucketState({ first_uncompacted_write: undefined }), INTERVAL_CONFIG)).toThrow(
+      'Scheduled V3 bucket bucket[] has no first uncompacted write'
+    );
+  });
+
+  test('chooses a full compact when its check is due', () => {
+    const decision = chooseCompactionKind(
+      bucketState({ bucket_stats: { count: 10, bytes: 100n, chunks: 10 } }),
+      new Date(10_000),
+      INTERVAL_CONFIG
+    );
+
+    expect(decision.kind).toBe(CompactionKind.Full);
+    expect(decision.nextCompactCheck).toEqual({
+      $min: [new Date(70_000), { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: 1_000 } }]
+    });
+  });
+
+  test('chooses a chunk compact after enough chunks and the minimum interval', () => {
+    const decision = chooseCompactionKind(
+      bucketState({
+        bucket_stats: { count: 10, bytes: 100n, chunks: 9 },
+        compacted_state: compactedState({ chunks: 1 })
+      }),
+      new Date(5_000),
+      INTERVAL_CONFIG
+    );
+
+    expect(decision.kind).toBe(CompactionKind.Chunks);
+  });
+
+  test('waits when the chunk interval has not elapsed', () => {
+    const decision = chooseCompactionKind(
+      bucketState({
+        bucket_stats: { count: 10, bytes: 100n, chunks: 9 },
+        compacted_state: compactedState({ chunks: 1, at: new Date(4_500) })
+      }),
+      new Date(5_000),
+      INTERVAL_CONFIG
+    );
+
+    expect(decision.kind).toBeNull();
+  });
+
+  test('schedules only the full check when too few chunks were added', () => {
+    const decision = chooseCompactionKind(
+      bucketState({ bucket_stats: { count: 7, bytes: 70n, chunks: 7 } }),
+      new Date(5_000),
+      INTERVAL_CONFIG
+    );
+
+    expect(decision).toEqual({ kind: null, nextCompactCheck: new Date(70_000) });
+  });
+
+  test.each([
+    {
+      name: 'does not force an unspecified kind',
+      forceKind: undefined,
+      compactedOpId: undefined,
+      maxOpIdCap: undefined,
+      expected: null
+    },
+    {
+      name: 'forces work without previous compacted state',
+      forceKind: CompactionKind.Chunks,
+      compactedOpId: undefined,
+      maxOpIdCap: undefined,
+      expected: CompactionKind.Chunks
+    },
+    {
+      name: 'skips work already compacted through the cap',
+      forceKind: CompactionKind.Chunks,
+      compactedOpId: 5n,
+      maxOpIdCap: 5n,
+      expected: null
+    },
+    {
+      name: 'skips work already compacted through the bucket head',
+      forceKind: CompactionKind.Chunks,
+      compactedOpId: 10n,
+      maxOpIdCap: undefined,
+      expected: null
+    },
+    {
+      name: 'forces work beyond the compacted state',
+      forceKind: CompactionKind.Chunks,
+      compactedOpId: 5n,
+      maxOpIdCap: 6n,
+      expected: CompactionKind.Chunks
+    }
+  ])('$name', ({ forceKind, compactedOpId, maxOpIdCap, expected }) => {
+    const state = bucketState({
+      compacted_state: compactedOpId == null ? undefined : compactedState({ op_id: compactedOpId })
+    });
+    expect(forcedCompactionKind(state, forceKind, { maxOpIdCap })).toBe(expected);
+  });
+
+  test('derives and combines bucket statistics', () => {
+    const state = bucketState({ bucket_stats: { count: 12, bytes: 120n, chunks: 3 } });
+    expect(bucketStats(state)).toEqual({ count: 12, bytes: 120n, chunks: 3 });
+    expect(emptyBucketStats()).toEqual({ count: 0, bytes: 0n, chunks: 0, checksum: 0 });
+    expect(statsForDocument({ count: 2, size: 20, checksum: 10n })).toEqual({
+      count: 2,
+      bytes: 20n,
+      chunks: 1,
+      checksum: 10
+    });
+    expect(
+      statsForDocuments([
+        { count: 2, size: 20, checksum: 10n },
+        { count: 3, size: 30, checksum: 20n }
+      ])
+    ).toEqual({ count: 5, bytes: 50n, chunks: 2, checksum: 30 });
+    expect(
+      combineAdjacentStats(
+        { count: 2, bytes: 20n, chunks: 1, checksum: 10 },
+        { count: 3, bytes: 30n, chunks: 2, checksum: 20 }
+      )
+    ).toEqual({ count: 5, bytes: 50n, chunks: 3, checksum: 30 });
+  });
+
+  test('replaces an overlapping chunk in cached statistics', () => {
+    expect(
+      combineChunkStats(
+        compactedState({ count: 10, bytes: 100n, chunks: 4, checksum: 100n }),
+        { count: 6, bytes: 60n, chunks: 2, checksum: 50 },
+        { count: 3, bytes: 30n, chunks: 1, checksum: 30 }
+      )
+    ).toEqual({ count: 13, bytes: 130n, chunks: 5, checksum: 120 });
+  });
+
+  test('applies compacted statistics as a delta to total bucket statistics', () => {
+    expect(
+      applyCompactionDelta(
+        { count: 20, bytes: 200n, chunks: 8 },
+        { count: 5, bytes: 50n, chunks: 3, checksum: 10 },
+        { count: 3, bytes: 30n, chunks: 1, checksum: 10 }
+      )
+    ).toEqual({ count: 18, bytes: 180n, chunks: 6 });
+  });
+
+  test('applies a stored range replacement to statistics including its checksum', () => {
+    expect(
+      applyStatsReplacement(
+        { count: 20, bytes: 200n, chunks: 8, checksum: 100 },
+        { count: 5, bytes: 50n, chunks: 3, checksum: 40 },
+        { count: 3, bytes: 30n, chunks: 1, checksum: 20 }
+      )
+    ).toEqual({ count: 18, bytes: 180n, chunks: 6, checksum: 80 });
+  });
+});

package/test/src/helpers/MemoryObjectStorage.ts

@@ -1,5 +1,6 @@
 import {
   ObjectStorage,
+  ObjectStorageOperationOptions,
   ObjectStoragePutMetadata
 } from '@module/storage/implementation/v3/object-storage/ObjectStorage.js';
 
@@ -9,13 +10,19 @@ export class MemoryObjectStorage implements ObjectStorage {
    */
   public readonly store = new Map<string, { data: Uint8Array; metadata: ObjectStoragePutMetadata }>();
 
-  async put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void> {
+  async put(
+    path: string,
+    data: Uint8Array,
+    metadata: ObjectStoragePutMetadata,
+    options?: ObjectStorageOperationOptions
+  ): Promise<void> {
+    options?.signal?.throwIfAborted();
     this.store.set(path, { data, metadata: metadata });
   }
 
   async get(
     path: string,
-    options?: { signal?: AbortSignal }
+    options?: ObjectStorageOperationOptions
   ): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }> {
     const signal = options?.signal;
     signal?.throwIfAborted();
@@ -26,7 +33,7 @@ export class MemoryObjectStorage implements ObjectStorage {
     return data;
   }
 
-  async *list(prefix: string, options?: { signal?: AbortSignal }): AsyncIterable<string> {
+  async *list(prefix: string, options?: ObjectStorageOperationOptions): AsyncIterable<string> {
     const signal = options?.signal;
     for (const path of this.store.keys()) {
       signal?.throwIfAborted();
@@ -36,18 +43,19 @@ export class MemoryObjectStorage implements ObjectStorage {
     }
   }
 
-  async delete(paths: string[]): Promise<void> {
+  async delete(paths: string[], options?: ObjectStorageOperationOptions): Promise<void> {
+    options?.signal?.throwIfAborted();
     for (const p of paths) {
       this.store.delete(p);
     }
   }
 
-  async deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }> {
+  async deletePrefix(prefix: string, options?: ObjectStorageOperationOptions): Promise<{ objectCount: number }> {
     const paths: string[] = [];
     for await (const path of this.list(prefix, options)) {
       paths.push(path);
     }
-    await this.delete(paths);
+    await this.delete(paths, options);
     return { objectCount: paths.length };
   }
 }