@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

package/CHANGELOG.md

@@ -1,17 +1,112 @@
 # @powersync/service-module-mongodb-storage
 
-## 0.0.0-dev-20260602073133
+## 0.0.0-dev-20260803144812
+
+### Minor Changes
+
+- 2189250: Add `/sync/checkpoint-request` for client-supplied checkpoint request ids, previously called write checkpoint ids. The route returns the stored `checkpoint_request_id`, storage now treats managed request ids as monotonic per user/client, custom checkpoint request ids continue to use the existing `checkpoint` field for backwards compatibility, and `checkpoint_requested_at` metadata lets compact jobs remove expired request-derived checkpoint records.
+
+  This release includes storage migrations for the checkpoint request metadata. Self-hosters should run migrations as part of the upgrade.
+
+- 922f974: Add experimental support for storing bucket data chunks on S3.
+
+### Patch Changes
+
+- 37591e9: [MongoDB Storage] Use batched deletes for clearing data.
+- 37d1856: Stability and performance fixes for MongoDB storage V3.
+- be14bfe: [MongoDB Storage V3] Tweak size estimates and chunking to use actual BSON sizes.
+- aab068b: [MongoDB Storage V3] Add has_clear_op field and simplify checksum queries.
+- Updated dependencies [2189250]
+- Updated dependencies [922f974]
+- Updated dependencies [c4860c9]
+- Updated dependencies [37591e9]
+- Updated dependencies [483415d]
+- Updated dependencies [8daa300]
+- Updated dependencies [aab068b]
+- Updated dependencies [37591e9]
+- Updated dependencies [be42e25]
+- Updated dependencies [cb4c627]
+  - @powersync/lib-services-framework@0.0.0-dev-20260803144812
+  - @powersync/service-core@0.0.0-dev-20260803144812
+  - @powersync/service-types@0.0.0-dev-20260803144812
+  - @powersync/lib-service-mongodb@0.0.0-dev-20260803144812
+  - @powersync/service-sync-rules@0.0.0-dev-20260803144812
+
+## 0.18.3
+
+### Patch Changes
+
+- ea31f64: Potential fix and improved stack trace for 'evicted' errors.
+- 1f854d3: Fix for occasional readConcern errors when using bulk_read_preference.
+- Updated dependencies [ea71bf3]
+- Updated dependencies [ea31f64]
+- Updated dependencies [edc6ed4]
+  - @powersync/service-sync-rules@0.39.0
+  - @powersync/service-core@1.23.3
+  - @powersync/lib-services-framework@0.9.8
+  - @powersync/lib-service-mongodb@0.6.29
+
+## 0.18.2
+
+### Patch Changes
+
+- e4f683d: [MongoDB Storage] Add experimental option to allow reading data from secondaries.
+- c2edf86: Add `api.parameters.bucket_count_cache_ttl_minutes` configuration option.
+- df9ab1e: Batch concurrent write checkpoint requests and increase request limits to improve throughput.
+- b9f6b6f: [MongoDB Storage] Optimize writes under high replication lag.
+- Updated dependencies [71d4a0a]
+- Updated dependencies [e4f683d]
+- Updated dependencies [71d4a0a]
+- Updated dependencies [a6ae678]
+- Updated dependencies [c2edf86]
+- Updated dependencies [df9ab1e]
+  - @powersync/service-core@1.23.2
+  - @powersync/lib-service-mongodb@0.6.28
+  - @powersync/service-sync-rules@0.38.1
+  - @powersync/service-types@0.16.1
+  - @powersync/lib-services-framework@0.9.7
+
+## 0.18.1
+
+### Patch Changes
+
+- 7e65360: Restructure replication locks for updated sync config.
+- Updated dependencies [7e65360]
+  - @powersync/service-core@1.23.1
+
+## 0.18.0
+
+### Minor Changes
+
+- a91a08f: [Experimental] Enable incremental reprocessing for MongoDB source + MongoDB storage. This includes significant changes to the v3 storage format.
+
+### Patch Changes
+
+- 101ea67: Include table names in "Cannot mark snapshot done while source tables still require snapshotting" error.
+- Updated dependencies [a91a08f]
+- Updated dependencies [184c39f]
+- Updated dependencies [c3f75df]
+- Updated dependencies [4bd35ea]
+  - @powersync/service-core@1.23.0
+  - @powersync/service-types@0.16.0
+  - @powersync/service-sync-rules@0.38.0
+  - @powersync/lib-services-framework@0.9.6
+  - @powersync/lib-service-mongodb@0.6.27
+
+## 0.17.0
 
 ### Minor Changes
 
 - e2bf1ad: [Internal] rework resolveTables to handle multiple SourceTables.
 - cd5a222: [MongoDB V3 Storage] Split storage for "sync config definition" versus "replication stream status".
 - 15e2466: [MongoDB] Support snapshotting concurrently with streaming in storage v3+.
+- ebeaa3b: [Postgres] Skip storing a redundant copy of replicated rows in `current_data` for tables with `REPLICA IDENTITY FULL`.
 
 ### Patch Changes
 
 - 6e2a57e: Refactor HydratedSyncConfig to support multiple SyncConfigs.
 - 92cc83b: Add the experimental `unstable_sqlite_expression_engine` sync config option to evaluate Sync Streams with SQLite.
+- Updated dependencies [17fd96b]
 - Updated dependencies [6e2a57e]
 - Updated dependencies [ec6df9f]
 - Updated dependencies [99d33d5]
@@ -24,12 +119,13 @@
 - Updated dependencies [92cc83b]
 - Updated dependencies [0aab0f9]
 - Updated dependencies [15e2466]
+- Updated dependencies [ebeaa3b]
 - Updated dependencies [b116857]
 - Updated dependencies [a94b6c3]
-  - @powersync/service-sync-rules@0.0.0-dev-20260602073133
-  - @powersync/service-core@0.0.0-dev-20260602073133
-  - @powersync/lib-services-framework@0.0.0-dev-20260602073133
-  - @powersync/lib-service-mongodb@0.0.0-dev-20260602073133
+  - @powersync/service-core@1.22.0
+  - @powersync/service-sync-rules@0.37.0
+  - @powersync/lib-services-framework@0.9.5
+  - @powersync/lib-service-mongodb@0.6.26
 
 ## 0.16.0
 

package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts

@@ -0,0 +1,3 @@
+import { migrations } from '@powersync/service-core';
+export declare const up: migrations.PowerSyncMigrationFunction;
+export declare const down: migrations.PowerSyncMigrationFunction;

package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js

@@ -0,0 +1,41 @@
+import * as storage from '../../../storage/storage-index.js';
+export const up = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = storage.createPowerSyncMongo(configuration.storage);
+    try {
+        await db.write_checkpoints.createIndex({
+            checkpoint_requested_at: 1
+        }, {
+            name: 'checkpoint_requested_at',
+            // Only client-requested checkpoints have this field; generated
+            // checkpoints leave it unset. This keeps the index limited to the
+            // documents the compact job's retention delete scans.
+            partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
+        });
+        await db.custom_write_checkpoints.createIndex({
+            checkpoint_requested_at: 1
+        }, {
+            name: 'checkpoint_requested_at',
+            partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
+        });
+    }
+    finally {
+        await db.client.close();
+    }
+};
+export const down = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = storage.createPowerSyncMongo(configuration.storage);
+    try {
+        if (await db.write_checkpoints.indexExists('checkpoint_requested_at')) {
+            await db.write_checkpoints.dropIndex('checkpoint_requested_at');
+        }
+        if (await db.custom_write_checkpoints.indexExists('checkpoint_requested_at')) {
+            await db.custom_write_checkpoints.dropIndex('checkpoint_requested_at');
+        }
+    }
+    finally {
+        await db.client.close();
+    }
+};
+//# sourceMappingURL=1782950400000-checkpoint-requested-at-index.js.map

package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"1782950400000-checkpoint-requested-at-index.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,mCAAmC,CAAC;AAG7D,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAErF,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,iBAAiB,CAAC,WAAW,CACpC;YACE,uBAAuB,EAAE,CAAC;SAC3B,EACD;YACE,IAAI,EAAE,yBAAyB;YAC/B,+DAA+D;YAC/D,kEAAkE;YAClE,sDAAsD;YACtD,uBAAuB,EAAE,EAAE,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;SACxE,CACF,CAAC;QACF,MAAM,EAAE,CAAC,wBAAwB,CAAC,WAAW,CAC3C;YACE,uBAAuB,EAAE,CAAC;SAC3B,EACD;YACE,IAAI,EAAE,yBAAyB;YAC/B,uBAAuB,EAAE,EAAE,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;SACxE,CACF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IAEZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAErF,IAAI,CAAC;QACH,IAAI,MAAM,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC,yBAAyB,CAAC,EAAE,CAAC;YACtE,MAAM,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QAClE,CAAC;QACD,IAAI,MAAM,EAAE,CAAC,wBAAwB,CAAC,WAAW,CAAC,yBAAyB,CAAC,EAAE,CAAC;YAC7E,MAAM,EAAE,CAAC,wBAAwB,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC"}

package/dist/storage/MongoBucketStorage.d.ts

@@ -1,35 +1,60 @@
 import { GetIntanceOptions, storage } from '@powersync/service-core';
 import { DO_NOT_LOG } from '@powersync/lib-services-framework';
+import { mongo } from '@powersync/lib-service-mongodb';
 import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { PowerSyncMongo } from './implementation/db.js';
 import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
-import { MongoPersistedSyncRulesContentV1, MongoPersistedSyncRulesContentV3 } from './implementation/MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from './implementation/MongoPersistedReplicationStream.js';
+import { ObjectStorage } from './implementation/v3/object-storage/ObjectStorage.js';
 export interface MongoBucketStorageOptions {
-    checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
+    checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+    objectStorage?: ObjectStorage;
+    inlineThresholdBytes?: number;
+    /**
+     * Prefix for replication stream name and Postgres logical replication slot name.
+     */
+    replicationStreamNamePrefix: string;
+    readPreference?: mongo.ReadPreference;
+    checksumCacheTtlMs?: number;
+    clearBatchThrottleRate?: number;
+    /**
+     * Reuse a compatible active replication stream by appending a new sync config.
+     *
+     * This currently requires source replication support. MongoDB sources can process multiple
+     * sync configs in one replication stream, but other source connectors still expect a single
+     * sync config per stream.
+     */
+    supportsMultipleSyncConfigs?: boolean;
 }
 export declare class MongoBucketStorage extends storage.BucketStorageFactory {
-    private internalOptions?;
+    private options;
     [DO_NOT_LOG]: boolean;
     private readonly client;
     private readonly session;
-    readonly slot_name_prefix: string;
+    readonly replicationStreamNamePrefix: string;
     private activeStorageCache;
     readonly db: PowerSyncMongo;
-    constructor(db: PowerSyncMongo, options: {
-        slot_name_prefix: string;
-    }, internalOptions?: MongoBucketStorageOptions | undefined);
+    constructor(db: PowerSyncMongo, options: MongoBucketStorageOptions);
     [Symbol.asyncDispose](): Promise<void>;
-    getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage;
+    getInstance(replicationStream: storage.PersistedReplicationStream, options?: GetIntanceOptions): MongoSyncBucketStorage;
     getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier>;
-    restartReplication(sync_rules_group_id: number): Promise<void>;
+    restartReplication(replicationStreamId: number): Promise<void>;
     private updateSyncRulesV3;
-    updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3>;
-    getActiveSyncRulesContent(): Promise<MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3 | null>;
-    private getSyncRulesContent;
-    getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3 | null>;
-    getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]>;
-    getStoppedSyncRules(): Promise<storage.PersistedSyncRulesContent[]>;
-    getActiveStorage(): Promise<MongoSyncBucketStorage | null>;
+    private loadSyncConfigDefinitions;
+    /**
+     * Load _all_ definition mappings for a replication stream - used as a base to generate new ids.
+     */
+    private loadHistoricalSyncConfigRuleMappings;
+    private stopEmbeddedDeployingConfigs;
+    private logIncrementalDefinitionChanges;
+    private appendSyncConfigToStream;
+    updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedReplicationStream>;
+    getActiveSyncConfig(): Promise<storage.ResolvedSyncConfig | null>;
+    private replicationStreamFromDoc;
+    getDeployingSyncConfig(): Promise<storage.ResolvedSyncConfig | null>;
+    getReplicatingReplicationStreams(): Promise<storage.PersistedReplicationStream[]>;
+    getStoppedReplicationStreams(): Promise<storage.PersistedReplicationStream[]>;
+    private resolvedSyncConfigFromDoc;
     getStorageMetrics(): Promise<storage.StorageMetrics>;
     getPowerSyncInstanceId(): Promise<string>;
 }