@powersync/service-module-mongodb-storage 0.20.0 → 0.22.0

package/src/storage/implementation/v3/source-table-utils.ts

@@ -3,6 +3,8 @@ import { BucketDefinitionMapping, ColumnDescriptor, JsonValue, storage } from '@
 import {
   BucketDataSource,
   BucketDefinitionId,
+  EventDefinitionId,
+  HydratedEventDescriptor,
   HydratedSyncConfig,
   MatchingSources,
   ParameterIndexId,
@@ -22,12 +24,13 @@ export interface SourceTableIdentity {
 export interface SourceTableMembershipIds {
   bucketDataSourceIds: BucketDefinitionId[];
   parameterLookupSourceIds: ParameterIndexId[];
+  eventDefinitionIds: EventDefinitionId[];
 }
 
 export interface SourceTableDesiredResolution {
   bucketSourceById: Map<BucketDefinitionId, BucketDataSource>;
   parameterLookupSourceById: Map<ParameterIndexId, ParameterIndexLookupCreator>;
-  triggersEvent: boolean;
+  eventDefinitionById: Map<EventDefinitionId, HydratedEventDescriptor>;
 }
 
 export interface SourceTableReconciliationContext {
@@ -55,7 +58,7 @@ export interface SourceTableReconciliationPlan {
   narrowingUpdates: SourceTableMembershipUpdate[];
   /**
    * Memberships for a new source-table doc covering desired ids no existing doc covers,
-   * or null if no new doc is needed. Empty memberships indicate an event-only table.
+   * or null if no new doc is needed.
    */
   newTableMemberships: SourceTableMembershipIds | null;
   /** Identity-overlapping docs that conflict with the current identity and must be dropped. */
@@ -75,7 +78,8 @@ export interface NewSourceTable {
 export function sourceTableDesiredResolution(
   syncConfig: HydratedSyncConfig,
   ref: SourceTableRef,
-  mapping: BucketDefinitionMapping
+  mapping: BucketDefinitionMapping,
+  eventById: ReadonlyMap<EventDefinitionId, HydratedEventDescriptor>
 ): SourceTableDesiredResolution {
   const matchingSources = syncConfig.getMatchingSources(ref);
   return {
@@ -85,7 +89,7 @@ export function sourceTableDesiredResolution(
     parameterLookupSourceById: new Map(
       matchingSources.parameterLookupSources.map((source) => [mapping.parameterLookupId(source), source] as const)
     ),
-    triggersEvent: syncConfig.tableTriggersEvent(ref)
+    eventDefinitionById: new Map([...eventById].filter(([, event]) => event.tableTriggersEvent(ref)))
   };
 }
 
@@ -116,28 +120,33 @@ function intersectMembershipIds(
 ): SourceTableMembershipIds {
   return {
     bucketDataSourceIds: doc.bucket_data_source_ids.filter((id) => desired.bucketSourceById.has(id)),
-    parameterLookupSourceIds: doc.parameter_lookup_source_ids.filter((id) => desired.parameterLookupSourceById.has(id))
+    parameterLookupSourceIds: doc.parameter_lookup_source_ids.filter((id) => desired.parameterLookupSourceById.has(id)),
+    eventDefinitionIds: doc.event_definition_ids.filter((id) => desired.eventDefinitionById.has(id))
   };
 }
 
 function hasMembershipIds(memberships: SourceTableMembershipIds) {
-  return memberships.bucketDataSourceIds.length > 0 || memberships.parameterLookupSourceIds.length > 0;
+  return (
+    memberships.bucketDataSourceIds.length > 0 ||
+    memberships.parameterLookupSourceIds.length > 0 ||
+    memberships.eventDefinitionIds.length > 0
+  );
 }
 
 function sameMembershipIds(doc: SourceTableDocumentV3, memberships: SourceTableMembershipIds) {
   return (
     sameStringArray(doc.bucket_data_source_ids, memberships.bucketDataSourceIds) &&
-    sameStringArray(doc.parameter_lookup_source_ids, memberships.parameterLookupSourceIds)
+    sameStringArray(doc.parameter_lookup_source_ids, memberships.parameterLookupSourceIds) &&
+    sameStringArray(doc.event_definition_ids, memberships.eventDefinitionIds)
   );
 }
 
 class SourceTableReconciliationPlanner {
   private readonly coveredBucketDataSourceIds = new Set<string>();
   private readonly coveredParameterLookupSourceIds = new Set<string>();
-  private readonly retainedDocIds = new Set<string>();
+  private readonly coveredEventDefinitionIds = new Set<string>();
   private readonly tables: storage.SourceTable[] = [];
   private readonly narrowingUpdates: SourceTableMembershipUpdate[] = [];
-  private retainedEventOnlyTable = false;
 
   constructor(private readonly context: SourceTableReconciliationContext) {}
 
@@ -168,24 +177,15 @@ class SourceTableReconciliationPlanner {
   private retainDoc(doc: SourceTableDocumentV3) {
     const memberships = intersectMembershipIds(doc, this.context.desired);
     const coversDesiredMembership = hasMembershipIds(memberships);
-    const coversEventOnlyTable =
-      !this.desiredHasMembership() && this.context.desired.triggersEvent && !this.retainedEventOnlyTable;
 
     this.recordCoverage(doc, memberships);
-    this.planNarrowingUpdate(doc, memberships, coversDesiredMembership, coversEventOnlyTable);
+    this.planNarrowingUpdate(doc, memberships, coversDesiredMembership);
 
-    if (coversDesiredMembership || coversEventOnlyTable) {
-      this.retainedEventOnlyTable ||= coversEventOnlyTable;
-      this.retainedDocIds.add(doc._id.toHexString());
+    if (coversDesiredMembership) {
       this.tables.push(this.sourceTableFor(doc, memberships));
     }
   }
 
-  private desiredHasMembership() {
-    const { desired } = this.context;
-    return desired.bucketSourceById.size > 0 || desired.parameterLookupSourceById.size > 0;
-  }
-
   private recordCoverage(doc: SourceTableDocumentV3, memberships: SourceTableMembershipIds) {
     this.addCoverage(doc, 'bucket data source', this.coveredBucketDataSourceIds, memberships.bucketDataSourceIds);
     this.addCoverage(
@@ -194,12 +194,14 @@ class SourceTableReconciliationPlanner {
       this.coveredParameterLookupSourceIds,
       memberships.parameterLookupSourceIds
     );
+    this.addCoverage(doc, 'event definition', this.coveredEventDefinitionIds, memberships.eventDefinitionIds);
   }
 
   // Membership sets must be pairwise disjoint across the docs of one physical table:
-  // each desired id is covered by exactly one doc, so each definition receives each
-  // row exactly once. The algorithm maintains this (new docs only get uncovered ids,
-  // narrowing only removes ids) - overlap means the persisted state is corrupt.
+  // each desired id is covered by exactly one doc, so each definition is evaluated through
+  // only one SourceTable for a physical row. Different docs may own different event ids.
+  // The algorithm maintains this (new docs only get uncovered ids, narrowing only removes ids) -
+  // overlap means the persisted state is corrupt.
   private addCoverage(doc: SourceTableDocumentV3, kind: string, covered: Set<string>, ids: string[]) {
     const { schema, name } = this.context.identity;
     for (const id of ids) {
@@ -215,11 +217,9 @@ class SourceTableReconciliationPlanner {
   private planNarrowingUpdate(
     doc: SourceTableDocumentV3,
     memberships: SourceTableMembershipIds,
-    coversDesiredMembership: boolean,
-    coversEventOnlyTable: boolean
+    coversDesiredMembership: boolean
   ) {
-    const shouldNarrow =
-      (coversDesiredMembership || coversEventOnlyTable) && !doc.snapshot_done && !sameMembershipIds(doc, memberships);
+    const shouldNarrow = coversDesiredMembership && !doc.snapshot_done && !sameMembershipIds(doc, memberships);
 
     if (!shouldNarrow) {
       return;
@@ -239,9 +239,12 @@ class SourceTableReconciliationPlanner {
       ),
       parameterLookupSourceIds: [...desired.parameterLookupSourceById.keys()].filter(
         (id) => !this.coveredParameterLookupSourceIds.has(id)
+      ),
+      eventDefinitionIds: [...desired.eventDefinitionById.keys()].filter(
+        (id) => !this.coveredEventDefinitionIds.has(id)
       )
     };
-    if (hasMembershipIds(uncovered) || (desired.triggersEvent && this.tables.length == 0)) {
+    if (hasMembershipIds(uncovered)) {
       return uncovered;
     }
     return null;
@@ -254,6 +257,7 @@ class SourceTableReconciliationPlanner {
       connectionTag,
       syncConfig,
       mapping,
+      desired.eventDefinitionById,
       matchingSourcesFor(desired, memberships),
       memberships
     );
@@ -306,6 +310,7 @@ export function createNewSourceTable(
     snapshot_status: undefined,
     bucket_data_source_ids: memberships.bucketDataSourceIds,
     parameter_lookup_source_ids: memberships.parameterLookupSourceIds,
+    event_definition_ids: memberships.eventDefinitionIds,
     // Records created together share the same source metadata.
     source_metadata: newTableSourceMetadata
   };
@@ -314,6 +319,7 @@ export function createNewSourceTable(
     connectionTag,
     syncConfig,
     mapping,
+    desired.eventDefinitionById,
     matchingSourcesFor(desired, memberships),
     memberships
   );
@@ -322,17 +328,6 @@ export function createNewSourceTable(
   return { doc, table };
 }
 
-export function designateEventCarrier(tables: storage.SourceTable[], triggersEvent: boolean) {
-  if (!triggersEvent) {
-    return;
-  }
-
-  const eventCarrier = tables.find((table) => table.snapshotComplete) ?? tables[0];
-  for (const table of tables) {
-    table.syncEvent = table === eventCarrier;
-  }
-}
-
 function sourceTableMembershipsFromDocument(
   doc: SourceTableDocumentV3,
   syncConfig: HydratedSyncConfig,
@@ -356,6 +351,7 @@ export function sourceTableFromDocument(
   connectionTag: string,
   syncConfig: HydratedSyncConfig,
   mapping: BucketDefinitionMapping,
+  eventById: ReadonlyMap<EventDefinitionId, HydratedEventDescriptor>,
   memberships?: MatchingSources,
   membershipIds?: SourceTableMembershipIds
 ): storage.SourceTable {
@@ -364,7 +360,8 @@ export function sourceTableFromDocument(
     bucketDataSourceIds: resolvedMemberships.bucketDataSources.map((source) => mapping.bucketSourceId(source)),
     parameterLookupSourceIds: resolvedMemberships.parameterLookupSources.map((source) =>
       mapping.parameterLookupId(source)
-    )
+    ),
+    eventDefinitionIds: doc.event_definition_ids.filter((id) => eventById.has(id))
   };
   const table = new storage.SourceTable({
     id: doc._id,
@@ -383,11 +380,12 @@ export function sourceTableFromDocument(
     parameterLookupSources: resolvedMemberships.parameterLookupSources,
     bucketDataSourceIds: new Set(resolvedMembershipIds.bucketDataSourceIds),
     parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds),
+    eventDefinitionIds: new Set(resolvedMembershipIds.eventDefinitionIds),
     sourceMetadata: doc.source_metadata ?? null
   });
   table.syncData = table.bucketDataSources.length > 0;
   table.syncParameters = table.parameterLookupSources.length > 0;
-  table.syncEvent = syncConfig.tableTriggersEvent(table.ref);
+  table.syncEvent = table.eventDefinitionIds!.size > 0;
   table.snapshotStatus =
     doc.snapshot_status == null
       ? undefined

package/src/storage/storage-index.ts

@@ -15,11 +15,13 @@ export { loadBucketDataDocument, serializeBucketData } from './implementation/v3
 export {
   BucketDataDocumentV3,
   BucketOperation,
+  ObjectStorageUsageDocument,
   RecordedLookupV3,
   ReplicationStreamDocumentV3,
   SyncConfigDefinition,
   SyncRuleConfigStateV3,
   taggedBucketParameterDocumentToTagged
 } from './implementation/v3/models.js';
+export * from './implementation/v3/object-storage/ObjectStorageUsage.js';
 export * from './MongoBucketStorage.js';
 export * from './MongoReportStorage.js';

package/src/types/types.ts

@@ -3,6 +3,20 @@ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
 
+/**
+ * AWS defaults mode, as used by the AWS SDK.
+ *
+ * This is the baseline for the object storage request timeouts, so that they do not each need to be
+ * configured individually.
+ *
+ * The SDK's `legacy` and `auto` modes are not selectable here: `legacy` defines no timeouts at all,
+ * and `auto` makes the timeouts depend on where the process happens to be running. Both are still
+ * handled when they come from the AWS environment, and are then treated as `standard`.
+ */
+export const S3DefaultsMode = service_types.enumLiteral('standard', 'in-region', 'cross-region', 'mobile');
+
+export type S3DefaultsMode = t.Encoded<typeof S3DefaultsMode>;
+
 const S3ObjectStorageConfig = t.object({
   type: t.literal('s3'),
   bucket: t.string,
@@ -13,21 +27,24 @@ const S3ObjectStorageConfig = t.object({
   access_key_id: t.string.optional(),
   secret_access_key: t.string.optional(),
   concurrency_limit: t.number.optional(),
+  // Defaults to the AWS_DEFAULTS_MODE environment variable.
+  defaults_mode: S3DefaultsMode.optional(),
   // Chunks whose BSON-serialized size falls below this byte threshold
   // stay inline in MongoDB instead of being offloaded to S3. Default 1024.
   inline_threshold_bytes: t.number.optional()
 });
 
-export const MongoStorageReadPreference = t
-  .literal('primary')
-  .or(t.literal('primaryPreferred'))
-  .or(t.literal('secondary'))
-  .or(t.literal('secondaryPreferred'))
-  .or(t.literal('nearest'));
+export const MongoStorageReadPreference = service_types.enumLiteral(
+  'primary',
+  'primaryPreferred',
+  'secondary',
+  'secondaryPreferred',
+  'nearest'
+);
 
 export type MongoStorageReadPreference = t.Encoded<typeof MongoStorageReadPreference>;
 
-export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(
+export const MongoStorageConfig = service_types.configFile.BaseStorageConfig.and(lib_mongo.BaseMongoConfig).and(
   t.object({
     /**
      * Read preference for bulk checksum and bucket data reads.
@@ -48,6 +65,9 @@ export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(
      */
     clear_batch_throttle_rate: t.number.optional(),
 
+    /** Maximum concurrent buckets across V3 chunk-compaction jobs. Range: 1–64. Default: 4 with object storage, otherwise 2. */
+    chunk_compaction_concurrency: t.number.optional(),
+
     object_storage: S3ObjectStorageConfig.optional()
   })
 );
@@ -56,6 +76,21 @@ export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
 export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
 
 export const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
+export const DEFAULT_CHUNK_COMPACTION_CONCURRENCY = 2;
+export const DEFAULT_OBJECT_STORAGE_CHUNK_COMPACTION_CONCURRENCY = 4;
+
+export function normalizeChunkCompactionConcurrency(value: number | undefined, hasObjectStorage = false): number {
+  const concurrency =
+    value ??
+    (hasObjectStorage ? DEFAULT_OBJECT_STORAGE_CHUNK_COMPACTION_CONCURRENCY : DEFAULT_CHUNK_COMPACTION_CONCURRENCY);
+  if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 64) {
+    throw new ServiceError(
+      ErrorCode.PSYNC_S3201,
+      'storage.chunk_compaction_concurrency must be an integer between 1 and 64'
+    );
+  }
+  return concurrency;
+}
 
 export function normalizeClearBatchThrottleRate(value: number | undefined): number {
   const rate = value ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;

package/src/utils/test-utils.ts

@@ -31,6 +31,7 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag
         checksumOptions: factoryOptions.checksumOptions,
         supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs,
         objectStorage: factoryOptions.objectStorage,
+        chunkCompactionConcurrency: factoryOptions.chunkCompactionConcurrency,
         inlineThresholdBytes: factoryOptions.inlineThresholdBytes
       });
     },

package/src/utils/util.ts

@@ -121,7 +121,9 @@ export function mapOpEntry(row: BucketDataDoc): utils.OplogEntry {
       object_type: row.table,
       object_id: row.row_id,
       checksum: Number(row.checksum),
-      subkey: replicaIdToSubkey(row.source_table!, row.source_key!),
+      // Storage V3 always persists this during replication. The fallback serves V1 data
+      // written before subkey persistence was introduced.
+      subkey: row.subkey ?? replicaIdToSubkey(row.source_table!, row.source_key!),
       data: row.data
     };
   } else {

package/test/src/__snapshots__/storage.test.ts.snap

@@ -75,6 +75,31 @@ exports[`Mongo Sync Bucket Storage - Data - v3 > (insert, delete, insert), (dele
 ]
 `;
 
+exports[`Mongo Sync Bucket Storage - Data - v4 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
 exports[`Mongo Sync Bucket Storage - split buckets > (insert, delete, insert), (delete) 1`] = `
 [
   {