@powersync/service-module-mongodb-storage 0.19.0 → 0.21.0

package/src/storage/implementation/v3/source-table-utils.ts

@@ -1,8 +1,10 @@
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
-import { BucketDefinitionMapping, ColumnDescriptor, storage } from '@powersync/service-core';
+import { BucketDefinitionMapping, ColumnDescriptor, JsonValue, storage } from '@powersync/service-core';
 import {
   BucketDataSource,
   BucketDefinitionId,
+  EventDefinitionId,
+  HydratedEventDescriptor,
   HydratedSyncConfig,
   MatchingSources,
   ParameterIndexId,
@@ -22,12 +24,13 @@ export interface SourceTableIdentity {
 export interface SourceTableMembershipIds {
   bucketDataSourceIds: BucketDefinitionId[];
   parameterLookupSourceIds: ParameterIndexId[];
+  eventDefinitionIds: EventDefinitionId[];
 }
 
 export interface SourceTableDesiredResolution {
   bucketSourceById: Map<BucketDefinitionId, BucketDataSource>;
   parameterLookupSourceById: Map<ParameterIndexId, ParameterIndexLookupCreator>;
-  triggersEvent: boolean;
+  eventDefinitionById: Map<EventDefinitionId, HydratedEventDescriptor>;
 }
 
 export interface SourceTableReconciliationContext {
@@ -38,6 +41,14 @@ export interface SourceTableReconciliationContext {
   syncConfig: HydratedSyncConfig;
   mapping: BucketDefinitionMapping;
   desired: SourceTableDesiredResolution;
+  /**
+   * Candidates the source connector considers safe to reuse.
+   */
+  sourceCompatibleTables: readonly storage.SourceTableCandidate[];
+  /**
+   * Source metadata for records created by this resolution.
+   */
+  newTableSourceMetadata: JsonValue;
 }
 
 export interface SourceTableReconciliationPlan {
@@ -47,7 +58,7 @@ export interface SourceTableReconciliationPlan {
   narrowingUpdates: SourceTableMembershipUpdate[];
   /**
    * Memberships for a new source-table doc covering desired ids no existing doc covers,
-   * or null if no new doc is needed. Empty memberships indicate an event-only table.
+   * or null if no new doc is needed.
    */
   newTableMemberships: SourceTableMembershipIds | null;
   /** Identity-overlapping docs that conflict with the current identity and must be dropped. */
@@ -67,7 +78,8 @@ export interface NewSourceTable {
 export function sourceTableDesiredResolution(
   syncConfig: HydratedSyncConfig,
   ref: SourceTableRef,
-  mapping: BucketDefinitionMapping
+  mapping: BucketDefinitionMapping,
+  eventById: ReadonlyMap<EventDefinitionId, HydratedEventDescriptor>
 ): SourceTableDesiredResolution {
   const matchingSources = syncConfig.getMatchingSources(ref);
   return {
@@ -77,7 +89,7 @@ export function sourceTableDesiredResolution(
     parameterLookupSourceById: new Map(
       matchingSources.parameterLookupSources.map((source) => [mapping.parameterLookupId(source), source] as const)
     ),
-    triggersEvent: syncConfig.tableTriggersEvent(ref)
+    eventDefinitionById: new Map([...eventById].filter(([, event]) => event.tableTriggersEvent(ref)))
   };
 }
 
@@ -108,35 +120,39 @@ function intersectMembershipIds(
 ): SourceTableMembershipIds {
   return {
     bucketDataSourceIds: doc.bucket_data_source_ids.filter((id) => desired.bucketSourceById.has(id)),
-    parameterLookupSourceIds: doc.parameter_lookup_source_ids.filter((id) => desired.parameterLookupSourceById.has(id))
+    parameterLookupSourceIds: doc.parameter_lookup_source_ids.filter((id) => desired.parameterLookupSourceById.has(id)),
+    eventDefinitionIds: doc.event_definition_ids.filter((id) => desired.eventDefinitionById.has(id))
   };
 }
 
 function hasMembershipIds(memberships: SourceTableMembershipIds) {
-  return memberships.bucketDataSourceIds.length > 0 || memberships.parameterLookupSourceIds.length > 0;
+  return (
+    memberships.bucketDataSourceIds.length > 0 ||
+    memberships.parameterLookupSourceIds.length > 0 ||
+    memberships.eventDefinitionIds.length > 0
+  );
 }
 
 function sameMembershipIds(doc: SourceTableDocumentV3, memberships: SourceTableMembershipIds) {
   return (
     sameStringArray(doc.bucket_data_source_ids, memberships.bucketDataSourceIds) &&
-    sameStringArray(doc.parameter_lookup_source_ids, memberships.parameterLookupSourceIds)
+    sameStringArray(doc.parameter_lookup_source_ids, memberships.parameterLookupSourceIds) &&
+    sameStringArray(doc.event_definition_ids, memberships.eventDefinitionIds)
   );
 }
 
 class SourceTableReconciliationPlanner {
   private readonly coveredBucketDataSourceIds = new Set<string>();
   private readonly coveredParameterLookupSourceIds = new Set<string>();
-  private readonly retainedDocIds = new Set<string>();
+  private readonly coveredEventDefinitionIds = new Set<string>();
   private readonly tables: storage.SourceTable[] = [];
   private readonly narrowingUpdates: SourceTableMembershipUpdate[] = [];
-  private retainedEventOnlyTable = false;
 
   constructor(private readonly context: SourceTableReconciliationContext) {}
 
   plan(candidateDocs: SourceTableDocumentV3[]): SourceTableReconciliationPlan {
-    const { identity } = this.context;
     for (const doc of candidateDocs) {
-      if (matchingSourceTableIdentity(doc, identity)) {
+      if (this.isCompatible(doc)) {
         this.retainDoc(doc);
       }
     }
@@ -145,35 +161,31 @@ class SourceTableReconciliationPlanner {
       tables: this.tables,
       narrowingUpdates: this.narrowingUpdates,
       newTableMemberships: this.newTableMemberships(),
-      // Non-retained overlapping docs represent renames / relation-id changes /
-      // replica-identity changes. Same-identity stale docs are retained in production.
-      dropDocs: candidateDocs.filter(
-        (doc) => !this.retainedDocIds.has(doc._id.toHexString()) && !matchingSourceTableIdentity(doc, identity)
-      )
+      // Compatible records remain available for reuse even when not needed by this config.
+      dropDocs: candidateDocs.filter((doc) => !this.isCompatible(doc))
     };
   }
 
+  private isCompatible(doc: SourceTableDocumentV3) {
+    return this.compatibleTableFor(doc) != null;
+  }
+
+  private compatibleTableFor(doc: SourceTableDocumentV3): storage.SourceTableCandidate | undefined {
+    return this.context.sourceCompatibleTables.find((table) => storage.sourceTableIdEquals(table.id, doc._id));
+  }
+
   private retainDoc(doc: SourceTableDocumentV3) {
     const memberships = intersectMembershipIds(doc, this.context.desired);
     const coversDesiredMembership = hasMembershipIds(memberships);
-    const coversEventOnlyTable =
-      !this.desiredHasMembership() && this.context.desired.triggersEvent && !this.retainedEventOnlyTable;
 
     this.recordCoverage(doc, memberships);
-    this.planNarrowingUpdate(doc, memberships, coversDesiredMembership, coversEventOnlyTable);
+    this.planNarrowingUpdate(doc, memberships, coversDesiredMembership);
 
-    if (coversDesiredMembership || coversEventOnlyTable) {
-      this.retainedEventOnlyTable ||= coversEventOnlyTable;
-      this.retainedDocIds.add(doc._id.toHexString());
+    if (coversDesiredMembership) {
       this.tables.push(this.sourceTableFor(doc, memberships));
     }
   }
 
-  private desiredHasMembership() {
-    const { desired } = this.context;
-    return desired.bucketSourceById.size > 0 || desired.parameterLookupSourceById.size > 0;
-  }
-
   private recordCoverage(doc: SourceTableDocumentV3, memberships: SourceTableMembershipIds) {
     this.addCoverage(doc, 'bucket data source', this.coveredBucketDataSourceIds, memberships.bucketDataSourceIds);
     this.addCoverage(
@@ -182,12 +194,14 @@ class SourceTableReconciliationPlanner {
       this.coveredParameterLookupSourceIds,
       memberships.parameterLookupSourceIds
     );
+    this.addCoverage(doc, 'event definition', this.coveredEventDefinitionIds, memberships.eventDefinitionIds);
   }
 
   // Membership sets must be pairwise disjoint across the docs of one physical table:
-  // each desired id is covered by exactly one doc, so each definition receives each
-  // row exactly once. The algorithm maintains this (new docs only get uncovered ids,
-  // narrowing only removes ids) - overlap means the persisted state is corrupt.
+  // each desired id is covered by exactly one doc, so each definition is evaluated through
+  // only one SourceTable for a physical row. Different docs may own different event ids.
+  // The algorithm maintains this (new docs only get uncovered ids, narrowing only removes ids) -
+  // overlap means the persisted state is corrupt.
   private addCoverage(doc: SourceTableDocumentV3, kind: string, covered: Set<string>, ids: string[]) {
     const { schema, name } = this.context.identity;
     for (const id of ids) {
@@ -203,11 +217,9 @@ class SourceTableReconciliationPlanner {
   private planNarrowingUpdate(
     doc: SourceTableDocumentV3,
     memberships: SourceTableMembershipIds,
-    coversDesiredMembership: boolean,
-    coversEventOnlyTable: boolean
+    coversDesiredMembership: boolean
   ) {
-    const shouldNarrow =
-      (coversDesiredMembership || coversEventOnlyTable) && !doc.snapshot_done && !sameMembershipIds(doc, memberships);
+    const shouldNarrow = coversDesiredMembership && !doc.snapshot_done && !sameMembershipIds(doc, memberships);
 
     if (!shouldNarrow) {
       return;
@@ -227,9 +239,12 @@ class SourceTableReconciliationPlanner {
       ),
       parameterLookupSourceIds: [...desired.parameterLookupSourceById.keys()].filter(
         (id) => !this.coveredParameterLookupSourceIds.has(id)
+      ),
+      eventDefinitionIds: [...desired.eventDefinitionById.keys()].filter(
+        (id) => !this.coveredEventDefinitionIds.has(id)
       )
     };
-    if (hasMembershipIds(uncovered) || (desired.triggersEvent && this.tables.length == 0)) {
+    if (hasMembershipIds(uncovered)) {
       return uncovered;
     }
     return null;
@@ -237,39 +252,23 @@ class SourceTableReconciliationPlanner {
 
   private sourceTableFor(doc: SourceTableDocumentV3, memberships: SourceTableMembershipIds): storage.SourceTable {
     const { connectionTag, syncConfig, mapping, desired, storeCurrentData } = this.context;
-    const table = sourceTableFromDocument(
+    const built = sourceTableFromDocument(
       doc,
       connectionTag,
       syncConfig,
       mapping,
+      desired.eventDefinitionById,
       matchingSourcesFor(desired, memberships),
       memberships
     );
+    // Use any metadata update returned by the reconciler.
+    const resolved = this.compatibleTableFor(doc);
+    const table = resolved == null ? built : built.withSourceMetadata(resolved.sourceMetadata);
     table.storeCurrentData = storeCurrentData;
     return table;
   }
 }
 
-function sameReplicaIdColumns(left: ReplicaIdColumn[] | undefined, right: ReplicaIdColumn[]) {
-  return (
-    left != null &&
-    left.length == right.length &&
-    left.every(
-      (column, index) =>
-        column.name == right[index].name && column.type == right[index].type && column.type_oid == right[index].type_oid
-    )
-  );
-}
-
-function matchingSourceTableIdentity(doc: SourceTableDocumentV3, identity: SourceTableIdentity) {
-  return (
-    doc.schema_name == identity.schema &&
-    doc.table_name == identity.name &&
-    (identity.objectId == null || doc.relation_id == identity.objectId) &&
-    sameReplicaIdColumns(doc.replica_id_columns, identity.replicaIdColumns)
-  );
-}
-
 export function overlappingSourceTableFilter(
   connectionId: number,
   identity: Pick<SourceTableIdentity, 'schema' | 'name' | 'objectId'>
@@ -290,7 +289,16 @@ export function createNewSourceTable(
   memberships: SourceTableMembershipIds,
   context: SourceTableReconciliationContext
 ): NewSourceTable {
-  const { connectionId, connectionTag, identity, syncConfig, mapping, desired, storeCurrentData } = context;
+  const {
+    connectionId,
+    connectionTag,
+    identity,
+    syncConfig,
+    mapping,
+    desired,
+    storeCurrentData,
+    newTableSourceMetadata
+  } = context;
   const doc: SourceTableDocumentV3 = {
     _id: id,
     connection_id: connectionId,
@@ -301,13 +309,17 @@ export function createNewSourceTable(
     snapshot_done: false,
     snapshot_status: undefined,
     bucket_data_source_ids: memberships.bucketDataSourceIds,
-    parameter_lookup_source_ids: memberships.parameterLookupSourceIds
+    parameter_lookup_source_ids: memberships.parameterLookupSourceIds,
+    event_definition_ids: memberships.eventDefinitionIds,
+    // Records created together share the same source metadata.
+    source_metadata: newTableSourceMetadata
   };
   const table = sourceTableFromDocument(
     doc,
     connectionTag,
     syncConfig,
     mapping,
+    desired.eventDefinitionById,
     matchingSourcesFor(desired, memberships),
     memberships
   );
@@ -316,17 +328,6 @@ export function createNewSourceTable(
   return { doc, table };
 }
 
-export function designateEventCarrier(tables: storage.SourceTable[], triggersEvent: boolean) {
-  if (!triggersEvent) {
-    return;
-  }
-
-  const eventCarrier = tables.find((table) => table.snapshotComplete) ?? tables[0];
-  for (const table of tables) {
-    table.syncEvent = table === eventCarrier;
-  }
-}
-
 function sourceTableMembershipsFromDocument(
   doc: SourceTableDocumentV3,
   syncConfig: HydratedSyncConfig,
@@ -350,6 +351,7 @@ export function sourceTableFromDocument(
   connectionTag: string,
   syncConfig: HydratedSyncConfig,
   mapping: BucketDefinitionMapping,
+  eventById: ReadonlyMap<EventDefinitionId, HydratedEventDescriptor>,
   memberships?: MatchingSources,
   membershipIds?: SourceTableMembershipIds
 ): storage.SourceTable {
@@ -358,7 +360,8 @@ export function sourceTableFromDocument(
     bucketDataSourceIds: resolvedMemberships.bucketDataSources.map((source) => mapping.bucketSourceId(source)),
     parameterLookupSourceIds: resolvedMemberships.parameterLookupSources.map((source) =>
       mapping.parameterLookupId(source)
-    )
+    ),
+    eventDefinitionIds: doc.event_definition_ids.filter((id) => eventById.has(id))
   };
   const table = new storage.SourceTable({
     id: doc._id,
@@ -368,18 +371,21 @@ export function sourceTableFromDocument(
       name: doc.table_name
     },
     objectId: doc.relation_id,
-    replicaIdColumns: doc.replica_id_columns!.map(
-      (c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
-    ),
+    replicaIdColumns:
+      doc.replica_id_columns?.map(
+        (column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
+      ) ?? [],
     snapshotComplete: doc.snapshot_done,
     bucketDataSources: resolvedMemberships.bucketDataSources,
     parameterLookupSources: resolvedMemberships.parameterLookupSources,
     bucketDataSourceIds: new Set(resolvedMembershipIds.bucketDataSourceIds),
-    parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds)
+    parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds),
+    eventDefinitionIds: new Set(resolvedMembershipIds.eventDefinitionIds),
+    sourceMetadata: doc.source_metadata ?? null
   });
   table.syncData = table.bucketDataSources.length > 0;
   table.syncParameters = table.parameterLookupSources.length > 0;
-  table.syncEvent = syncConfig.tableTriggersEvent(table.ref);
+  table.syncEvent = table.eventDefinitionIds!.size > 0;
   table.snapshotStatus =
     doc.snapshot_status == null
       ? undefined

package/src/storage/storage-index.ts

@@ -15,11 +15,13 @@ export { loadBucketDataDocument, serializeBucketData } from './implementation/v3
 export {
   BucketDataDocumentV3,
   BucketOperation,
+  ObjectStorageUsageDocument,
   RecordedLookupV3,
   ReplicationStreamDocumentV3,
   SyncConfigDefinition,
   SyncRuleConfigStateV3,
   taggedBucketParameterDocumentToTagged
 } from './implementation/v3/models.js';
+export * from './implementation/v3/object-storage/ObjectStorageUsage.js';
 export * from './MongoBucketStorage.js';
 export * from './MongoReportStorage.js';

package/src/types/types.ts

@@ -3,6 +3,20 @@ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
 
+/**
+ * AWS defaults mode, as used by the AWS SDK.
+ *
+ * This is the baseline for the object storage request timeouts, so that they do not each need to be
+ * configured individually.
+ *
+ * The SDK's `legacy` and `auto` modes are not selectable here: `legacy` defines no timeouts at all,
+ * and `auto` makes the timeouts depend on where the process happens to be running. Both are still
+ * handled when they come from the AWS environment, and are then treated as `standard`.
+ */
+export const S3DefaultsMode = service_types.enumLiteral('standard', 'in-region', 'cross-region', 'mobile');
+
+export type S3DefaultsMode = t.Encoded<typeof S3DefaultsMode>;
+
 const S3ObjectStorageConfig = t.object({
   type: t.literal('s3'),
   bucket: t.string,
@@ -13,21 +27,24 @@ const S3ObjectStorageConfig = t.object({
   access_key_id: t.string.optional(),
   secret_access_key: t.string.optional(),
   concurrency_limit: t.number.optional(),
+  // Defaults to the AWS_DEFAULTS_MODE environment variable.
+  defaults_mode: S3DefaultsMode.optional(),
   // Chunks whose BSON-serialized size falls below this byte threshold
   // stay inline in MongoDB instead of being offloaded to S3. Default 1024.
   inline_threshold_bytes: t.number.optional()
 });
 
-export const MongoStorageReadPreference = t
-  .literal('primary')
-  .or(t.literal('primaryPreferred'))
-  .or(t.literal('secondary'))
-  .or(t.literal('secondaryPreferred'))
-  .or(t.literal('nearest'));
+export const MongoStorageReadPreference = service_types.enumLiteral(
+  'primary',
+  'primaryPreferred',
+  'secondary',
+  'secondaryPreferred',
+  'nearest'
+);
 
 export type MongoStorageReadPreference = t.Encoded<typeof MongoStorageReadPreference>;
 
-export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(
+export const MongoStorageConfig = service_types.configFile.BaseStorageConfig.and(lib_mongo.BaseMongoConfig).and(
   t.object({
     /**
      * Read preference for bulk checksum and bucket data reads.

package/src/utils/util.ts

@@ -121,7 +121,9 @@ export function mapOpEntry(row: BucketDataDoc): utils.OplogEntry {
       object_type: row.table,
       object_id: row.row_id,
       checksum: Number(row.checksum),
-      subkey: replicaIdToSubkey(row.source_table!, row.source_key!),
+      // Storage V3 always persists this during replication. The fallback serves V1 data
+      // written before subkey persistence was introduced.
+      subkey: row.subkey ?? replicaIdToSubkey(row.source_table!, row.source_key!),
       data: row.data
     };
   } else {

package/test/src/__snapshots__/storage.test.ts.snap

@@ -75,6 +75,31 @@ exports[`Mongo Sync Bucket Storage - Data - v3 > (insert, delete, insert), (dele
 ]
 `;
 
+exports[`Mongo Sync Bucket Storage - Data - v4 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
 exports[`Mongo Sync Bucket Storage - split buckets > (insert, delete, insert), (delete) 1`] = `
 [
   {