@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260909133214

package/src/storage/implementation/common/PersistedBatch.ts

@@ -65,6 +65,11 @@ export interface PersistedBatchOptions {
   logger?: Logger;
   objectStorage?: ObjectStorage;
   inlineThresholdBytes?: number;
+  /**
+   * Aborts in-flight object storage uploads when replication stops.
+   */
+  signal?: AbortSignal;
+  objectStorageUsageWriterId?: string;
 }
 
 /**
@@ -81,6 +86,8 @@ export abstract class PersistedBatch {
 
   protected readonly objectStorage?: ObjectStorage;
   protected readonly inlineThresholdBytes: number = DEFAULT_INLINE_THRESHOLD_BYTES;
+  protected readonly signal?: AbortSignal;
+  protected readonly objectStorageUsageWriterId?: string;
 
   /**
    * For debug logging only.
@@ -102,6 +109,8 @@ export abstract class PersistedBatch {
     this.currentSize = writtenSize;
     this.logger = options?.logger ?? defaultLogger;
     this.objectStorage = options?.objectStorage;
+    this.signal = options?.signal;
+    this.objectStorageUsageWriterId = options?.objectStorageUsageWriterId;
     if (options?.inlineThresholdBytes != null) {
       this.inlineThresholdBytes = options.inlineThresholdBytes;
     }
@@ -124,7 +133,8 @@ export abstract class PersistedBatch {
       remaining_buckets.set(currentBucketKey(mapped), mapped);
     }
 
-    const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)));
+    const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
+    const dchecksum = BigInt(utils.hashDelete(subkey));
 
     for (const evaluated of options.evaluated) {
       const definitionId = this.getBucketDefinitionId(evaluated.source);
@@ -159,6 +169,7 @@ export abstract class PersistedBatch {
         bucket: evaluated.bucket,
         sourceTableId: options.table.id,
         sourceKey: options.sourceKey,
+        subkey,
         table: evaluated.table,
         rowId: evaluated.id,
         checksum: BigInt(checksum),
@@ -181,6 +192,7 @@ export abstract class PersistedBatch {
         op_id,
         sourceTableId: options.table.id,
         sourceKey: options.sourceKey,
+        subkey,
         table: bucket.table,
         rowId: bucket.id,
         checksum: dchecksum
@@ -217,6 +229,14 @@ export abstract class PersistedBatch {
   protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
   protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
 
+  /**
+   * New bucket operations persist protocol subkeys during replication. Older
+   * V1 documents may not have one, so sync reads retain a fallback.
+   */
+  protected persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string {
+    return replicaIdToSubkey(table, key);
+  }
+
   protected get bucketDataCount(): number {
     return this.bucketData.length;
   }
@@ -234,17 +254,45 @@ export abstract class PersistedBatch {
         bucket,
         lastOp: op_id,
         incrementCount: 1,
-        incrementBytes: bytes
+        incrementBytes: bytes,
+        incrementChunks: 0
       });
     }
   }
 
+  /**
+   * V3 persists operations in chunks. Keep this separate from incrementBucket:
+   * operation counts are known while evaluating rows, while chunk counts and
+   * exact persisted bytes are only known after the writer has chunked a flush.
+   */
+  protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number) {
+    const key = `${definitionId ?? ''}:${bucket}`;
+    const existingState = this.bucketStates.get(key);
+    if (existingState != null) {
+      existingState.incrementChunks += 1;
+      existingState.incrementBytes += bytes;
+    }
+  }
+
+  /**
+   * V3's compactor calculates byte deltas from persisted chunk metadata.
+   * Replace the writer's per-operation estimate with those exact sizes before
+   * flushing the corresponding bucket state.
+   */
+  protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string) {
+    const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
+    if (state != null) {
+      state.incrementBytes = 0;
+    }
+  }
+
   protected addBucketDataPut(options: {
     op_id: InternalOpId;
     bucketKey: BucketKey;
     bucket: string;
     sourceTableId: storage.SourceTable['id'];
     sourceKey: storage.ReplicaId;
+    subkey: string;
     table: string;
     rowId: string;
     checksum: bigint;
@@ -256,6 +304,7 @@ export abstract class PersistedBatch {
       op: 'PUT',
       source_table: mongoTableId(options.sourceTableId),
       source_key: options.sourceKey,
+      subkey: options.subkey,
       table: options.table,
       row_id: options.rowId,
       checksum: options.checksum,
@@ -268,6 +317,7 @@ export abstract class PersistedBatch {
     bucketKey: BucketKey;
     sourceTableId: storage.SourceTable['id'];
     sourceKey: storage.ReplicaId;
+    subkey: string;
     table: string;
     rowId: string;
     checksum: bigint;
@@ -278,6 +328,7 @@ export abstract class PersistedBatch {
       op: 'REMOVE',
       source_table: mongoTableId(options.sourceTableId),
       source_key: options.sourceKey,
+      subkey: options.subkey,
       table: options.table,
       row_id: options.rowId,
       checksum: options.checksum,
@@ -321,32 +372,32 @@ export abstract class PersistedBatch {
         const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000);
 
         this.logger.info(
-          `Flushed ${this.bucketDataCount} + ${this.bucketParameters.length} + ${
+          `Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${
             this.currentDataCount
-          } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`,
+          } records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`,
           {
             flushed: {
               duration: duration,
               size: this.currentSize,
-              bucket_data_count: this.bucketDataCount,
-              parameter_data_count: this.bucketParameters.length,
-              current_data_count: this.currentDataCount,
+              bucket_ops_count: this.bucketDataCount,
+              parameter_indexes_count: this.bucketParameters.length,
+              source_records_count: this.currentDataCount,
               replication_lag_seconds: replicationLag
             }
           }
         );
       } else {
         this.logger.info(
-          `Flushed ${this.bucketDataCount} + ${this.bucketParameters.length} + ${
+          `Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${
             this.currentDataCount
-          } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`,
+          } records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`,
           {
             flushed: {
               duration: duration,
               size: this.currentSize,
-              bucket_data_count: this.bucketDataCount,
-              parameter_data_count: this.bucketParameters.length,
-              current_data_count: this.currentDataCount
+              bucket_ops_count: this.bucketDataCount,
+              parameter_indexes_count: this.bucketParameters.length,
+              source_records_count: this.currentDataCount
             }
           }
         );
@@ -377,4 +428,5 @@ export interface BucketStateUpdate {
   lastOp: InternalOpId;
   incrementCount: number;
   incrementBytes: number;
+  incrementChunks: number;
 }

package/src/storage/implementation/db.ts

@@ -24,8 +24,9 @@ import {
   CurrentDataDocument
 } from './v1/models.js';
 import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
-import { BucketDataDocumentV3 } from './v3/models.js';
-import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
+import { BucketDataDocumentV3, CustomCheckpointRequestDocumentV3 } from './v3/models.js';
+import type { CustomCheckpointRequestCollectionSpecifier } from './v3/VersionedPowerSyncMongoV3.js';
+import { OBJECT_STORAGE_USAGE_COLLECTION, VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
 
 export interface PowerSyncMongoOptions {
   /**
@@ -137,6 +138,20 @@ export class PowerSyncMongo {
     return `source_table_${replicationStreamId}`;
   }
 
+  customCheckpointRequestCollectionName(options: CustomCheckpointRequestCollectionSpecifier) {
+    return `custom_checkpoint_requests_${options.replicationStreamId}_${options.eventId}`;
+  }
+
+  async listCustomCheckpointRequestCollections(
+    replicationStreamId?: number
+  ): Promise<mongo.Collection<CustomCheckpointRequestDocumentV3>[]> {
+    const prefix =
+      replicationStreamId == null
+        ? 'custom_checkpoint_requests_'
+        : `custom_checkpoint_requests_${replicationStreamId}_`;
+    return this.collectionsByPrefix(prefix);
+  }
+
   async listSourceTableCollections(
     replicationStreamId?: number
   ): Promise<mongo.Collection<CommonSourceTableDocument>[]> {
@@ -178,6 +193,10 @@ export class PowerSyncMongo {
     await this.locks.deleteMany({});
     await this.bucket_state.deleteMany({});
     await this.custom_write_checkpoints.deleteMany({});
+    await this.db.collection(OBJECT_STORAGE_USAGE_COLLECTION).deleteMany({});
+    for (const collection of await this.listCustomCheckpointRequestCollections()) {
+      await collection.drop();
+    }
   }
 
   /**

package/src/storage/implementation/models.ts

@@ -74,6 +74,12 @@ export interface BucketDataProperties {
   op: OpType;
   source_table?: bson.ObjectId;
   source_key?: ReplicaId;
+  /**
+   * Stable identifier for the source record in the client protocol.
+   *
+   * This is derived from (source_table, source_key), so persistence is optional but preferred.
+   */
+  subkey?: string;
   table?: string;
   row_id?: string;
   checksum: bigint;
@@ -110,6 +116,10 @@ export interface SourceTableDocument {
   replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined;
   snapshot_done: boolean | undefined;
   snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
+  /**
+   * Source-specific metadata. Absent for legacy records.
+   */
+  source_metadata?: storage.JsonValue;
 }
 
 export interface SourceTableDocumentSnapshotStatus {
@@ -211,6 +221,34 @@ export interface SyncRuleDocumentBase {
   } | null;
 
   storage_version?: number;
+
+  /** Optional operator-supplied label identifying the sync config version. */
+  version_label?: string | null;
+
+  /**
+   * Incremental parameter compaction state for the replication stream.
+   *
+   * Operation ids are allocated across all parameter indexes of a stream, so this is
+   * stream-level state, shared by all sync configs of the stream.
+   */
+  parameter_compaction?: {
+    /**
+     * The exclusive operation-id boundary through which every parameter index in this stream has
+     * been compacted.
+     */
+    compacted_before: InternalOpId;
+
+    /**
+     * Parameter entries below this boundary may no longer be available for checkpoint change
+     * detection, since compaction may have deleted them.
+     *
+     * This is advanced before the first delete of a compaction pass, while
+     * {@link compacted_before} is only advanced after every delete of the pass completed.
+     * A checkpoint transition starting below this boundary must conservatively invalidate all
+     * parameter buckets, since the individual changes may no longer be available.
+     */
+    checkpoint_changes_invalid_before?: InternalOpId;
+  };
 }
 
 export interface SyncRuleCheckpointFields {

package/src/storage/implementation/v1/MongoBucketBatchV1.ts

@@ -1,5 +1,7 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { ColumnDescriptor, InternalOpId, SourceTable, storage } from '@powersync/service-core';
+import { HydratedSyncConfig } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { mongoTableId } from '../../../utils/util.js';
 import { calculateCheckpointState } from '../CheckpointState.js';
@@ -9,7 +11,7 @@ import { SourceRecordStore } from '../common/SourceRecordStore.js';
 import { PersistedBatchV1 } from './PersistedBatchV1.js';
 import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
 import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
-import { SyncRuleDocumentV1 } from './models.js';
+import { SourceTableDocumentV1, SyncRuleDocumentV1 } from './models.js';
 
 export class MongoBucketBatchV1 extends MongoBucketBatch {
   declare readonly db: VersionedPowerSyncMongoV1;
@@ -49,8 +51,39 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
     // No-op for V1: source records live in a shared collection.
   }
 
+  protected override async batchCreateCustomWriteCheckpoints(
+    session: mongo.ClientSession,
+    opId: InternalOpId
+  ): Promise<void> {
+    await this.db.custom_write_checkpoints.bulkWrite(
+      this.write_checkpoint_batch.map((checkpointOptions) => {
+        const set: Record<string, unknown> = {
+          checkpoint: checkpointOptions.checkpoint,
+          sync_rules_id: checkpointOptions.sync_rules_id,
+          op_id: opId
+        };
+        if (checkpointOptions.checkpoint_requested_at != null) {
+          set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
+        }
+
+        return {
+          updateOne: {
+            filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
+            update: {
+              $set: set,
+              ...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
+            },
+            upsert: true
+          }
+        };
+      }),
+      { session }
+    );
+  }
+
   async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
     const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
+    const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
     const { connection_id, source } = options;
     const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
 
@@ -63,20 +96,38 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
     let result: storage.ResolveTablesResult | null = null;
     await this.db.client.withSession(async (session) => {
       const col = this.db.sourceTablesV1(this.replicationStreamId);
-      const filter: any = {
-        group_id: this.replicationStreamId,
-        connection_id,
-        schema_name: schema,
-        table_name: name,
-        replica_id_columns2: normalizedReplicaIdColumns
-      };
+
+      // Find records that overlap by name or relation id.
+      const orClauses: Record<string, unknown>[] = [{ schema_name: schema, table_name: name }];
       if (objectId != null) {
-        filter.relation_id = objectId;
+        orClauses.push({ relation_id: objectId });
       }
+      const candidateDocs = await col
+        .find({ group_id: this.replicationStreamId, connection_id, $or: orClauses }, { session })
+        .toArray();
 
-      let doc = await col.findOne(filter, { session });
-      if (doc == null) {
-        doc = {
+      const candidateTables = candidateDocs.map((doc) =>
+        this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows)
+      );
+      const candidates = candidateTables.map((table) => table.clone());
+      const resolution = await reconcile({ source, candidates });
+      storage.validateSourceTableCandidateResolution(candidates, resolution);
+
+      for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
+        await col.updateOne({ _id: mongoTableId(id) }, { $set: { source_metadata: sourceMetadata } }, { session });
+      }
+
+      const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
+      const compatibleTables = materializedResolution.compatibleTables;
+
+      // V1 keeps one record, preferring one that has already been snapshotted.
+      const reuse = compatibleTables.find((table) => table.snapshotComplete) ?? compatibleTables[0] ?? null;
+
+      let sourceTable: storage.SourceTable;
+      if (reuse != null) {
+        sourceTable = reuse;
+      } else {
+        const doc: SourceTableDocumentV1 = {
           _id: options.idGenerator ? (options.idGenerator() as bson.ObjectId) : new bson.ObjectId(),
           group_id: this.replicationStreamId,
           connection_id,
@@ -86,69 +137,18 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
           replica_id_columns: null,
           replica_id_columns2: normalizedReplicaIdColumns,
           snapshot_done: false,
-          snapshot_status: undefined
+          snapshot_status: undefined,
+          source_metadata: resolution.newTableValues.sourceMetadata
         };
         await col.insertOne(doc, { session });
+        sourceTable = this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows);
       }
 
-      const sourceTable = new storage.SourceTable({
-        id: doc._id,
-        ref: source,
-        objectId,
-        replicaIdColumns,
-        snapshotComplete: doc.snapshot_done ?? true,
-        ...syncRules.getMatchingSources(source)
-      });
-      sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
-      sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
-      sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
-      sourceTable.storeCurrentData = sendsCompleteRows !== true;
-      sourceTable.snapshotStatus =
-        doc.snapshot_status == null
-          ? undefined
-          : {
-              lastKey: doc.snapshot_status.last_key?.buffer ?? null,
-              totalEstimatedCount: doc.snapshot_status.total_estimated_count,
-              replicatedCount: doc.snapshot_status.replicated_count
-            };
-
-      const truncateFilter = [{ schema_name: schema, table_name: name }] as any[];
-      if (objectId != null) {
-        truncateFilter.push({ relation_id: objectId });
-      }
-      const truncate = await col
-        .find(
-          {
-            group_id: this.replicationStreamId,
-            connection_id,
-            _id: { $ne: doc._id },
-            $or: truncateFilter
-          },
-          { session }
-        )
-        .toArray();
-      const dropTables = truncate.map((dropDoc) => {
-        const ref = {
-          connectionTag,
-          schema: dropDoc.schema_name,
-          name: dropDoc.table_name
-        };
-        const table = new storage.SourceTable({
-          id: dropDoc._id,
-          ref,
-          objectId: dropDoc.relation_id,
-          replicaIdColumns:
-            dropDoc.replica_id_columns2?.map(
-              (c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
-            ) ?? [],
-          snapshotComplete: dropDoc.snapshot_done ?? true,
-          ...syncRules.getMatchingSources(ref)
-        });
-        table.syncEvent = syncRules.tableTriggersEvent(ref);
-        table.syncData = table.bucketDataSources.length > 0;
-        table.syncParameters = table.parameterLookupSources.length > 0;
-        return table;
-      });
+      const dropTables = [
+        ...materializedResolution.incompatibleTables,
+        // V1 only keeps one SourceTable per physical table.
+        ...compatibleTables.filter((table) => !storage.sourceTableIdEquals(table.id, sourceTable.id))
+      ];
 
       result = { tables: [sourceTable], dropTables };
     });
@@ -156,38 +156,33 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
     return result!;
   }
 
-  async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
-    const doc = await this.db.sourceTablesV1(this.replicationStreamId).findOne(
-      {
-        group_id: this.replicationStreamId,
-        _id: mongoTableId(table.id)
-      },
-      { session: this.session }
-    );
-    if (doc == null) {
-      return null;
-    }
-
-    const ref = {
-      connectionTag: table.ref.connectionTag,
-      schema: doc.schema_name,
-      name: doc.table_name
-    };
-    const sourceTable = new storage.SourceTable({
+  /**
+   * Build a SourceTable from its persisted v1 record.
+   */
+  private hydrateSourceTable(
+    doc: SourceTableDocumentV1,
+    connectionTag: string,
+    syncRules: HydratedSyncConfig,
+    sendsCompleteRows: boolean | undefined
+  ): storage.SourceTable {
+    const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
+    const table = new storage.SourceTable({
       id: doc._id,
       ref,
       objectId: doc.relation_id,
       replicaIdColumns:
         doc.replica_id_columns2?.map(
-          (c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
+          (column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
         ) ?? [],
       snapshotComplete: doc.snapshot_done ?? true,
-      ...this.sync_rules.getMatchingSources(ref)
+      sourceMetadata: doc.source_metadata ?? null,
+      ...syncRules.getMatchingSources(ref)
     });
-    sourceTable.syncEvent = this.sync_rules.tableTriggersEvent(ref);
-    sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
-    sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
-    sourceTable.snapshotStatus =
+    table.syncEvent = syncRules.tableTriggersEvent(ref);
+    table.syncData = table.bucketDataSources.length > 0;
+    table.syncParameters = table.parameterLookupSources.length > 0;
+    table.storeCurrentData = sendsCompleteRows !== true;
+    table.snapshotStatus =
       doc.snapshot_status == null
         ? undefined
         : {
@@ -195,7 +190,23 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
             totalEstimatedCount: doc.snapshot_status.total_estimated_count,
             replicatedCount: doc.snapshot_status.replicated_count
           };
-    return sourceTable;
+    return table;
+  }
+
+  async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
+    const doc = await this.db.sourceTablesV1(this.replicationStreamId).findOne(
+      {
+        group_id: this.replicationStreamId,
+        _id: mongoTableId(table.id)
+      },
+      { session: this.session }
+    );
+    if (doc == null) {
+      return null;
+    }
+
+    // Use the default storeCurrentData value when reading outside resolution.
+    return this.hydrateSourceTable(doc, table.ref.connectionTag, this.sync_rules, undefined);
   }
 
   async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult> {