@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/CHANGELOG.md

@@ -1,17 +1,188 @@
 # @powersync/service-module-mongodb-storage
 
-## 0.0.0-dev-20260602073133
+## 0.0.0-dev-20260909133214
+
+### Patch Changes
+
+- Updated dependencies [70b00a4]
+  - @powersync/service-sync-rules@0.0.0-dev-20260909133214
+  - @powersync/lib-services-framework@0.0.0-dev-20260909133214
+  - @powersync/service-core@0.0.0-dev-20260909133214
+  - @powersync/lib-service-mongodb@0.0.0-dev-20260909133214
+
+## 0.21.0
+
+### Minor Changes
+
+- 189bd9e: Add storage_version: 4 as a stable storage version.
+- 332d649: [MongoDB Storage] Support incremental parameter compacting jobs.
+- 33a3c23: Add optional version_label for sync configs.
+- fccdff2: Add object storage metrics, and storage metrics split by sync config.
+- 989d9e1: Scope MongoDB v3 custom write checkpoint records and reads to the stream-assigned event definition that created them. Custom checkpoint mode names the event whose id is resolved through the active sync config's persisted mapping, while event handlers copy the assigned id from replication event payloads when writing checkpoints. This keeps active and incrementally processing checkpoint events separate when their definitions change.
+- 646c0de: Track replication events as MongoDB storage v3 source-table memberships so unchanged event definitions retain their assigned ids while new or changed definitions are resnapshotted during incremental reprocessing.
+- 2407f71: Apply timeouts and abortSignal handling to all S3 operations. The broad timeouts can be configured using the new `storage.object_storage.defaults_mode` option, or the `AWS_DEFAULTS_MODE` environment variable. Upgrade S3 SDK to fix further timeout issues.
+- a997c88: Restructure MongoDB V3 bucket compacting.
+
+### Patch Changes
+
+- e11c54b: Add S3 timing info for sync checkpoints.
+- aecf844: Persist subkey instead of re-computing every time.
+- cc121b3: [MongoDB Storage V3] Force create new replication stream when the old one is invalidated, and fix other consistency issues with V3 replication stream updates.
+- 4e4063b: Optimize runtime type checks, closes https://github.com/powersync-ja/powersync-service/issues/771.
+- 4037e8f: Add storage.default_storage_version config option.
+- dd7a22f: Add the sync config version label to attributed storage metrics while excluding it from shared OTLP telemetry.
+- Updated dependencies [e11c54b]
+- Updated dependencies [4b6a23e]
+- Updated dependencies [189bd9e]
+- Updated dependencies [bb08068]
+- Updated dependencies [0077a8d]
+- Updated dependencies [332d649]
+- Updated dependencies [33a3c23]
+- Updated dependencies [fccdff2]
+- Updated dependencies [989d9e1]
+- Updated dependencies [646c0de]
+- Updated dependencies [cc121b3]
+- Updated dependencies [4e4063b]
+- Updated dependencies [2407f71]
+- Updated dependencies [10131ca]
+- Updated dependencies [4037e8f]
+- Updated dependencies [3972bac]
+- Updated dependencies [dd7a22f]
+- Updated dependencies [a997c88]
+  - @powersync/service-core@1.26.0
+  - @powersync/service-sync-rules@0.41.0
+  - @powersync/service-types@0.18.0
+  - @powersync/lib-services-framework@0.10.1
+  - @powersync/lib-service-mongodb@0.6.31
+
+## 0.20.0
+
+### Minor Changes
+
+- 798d739: Add source-owned `SourceTable` reconciliation.
+
+  `resolveTables()` now queries all overlapping persisted candidates and passes them to a
+  source-provided reconciler that returns compatible and incompatible tables, can return
+  modified compatible copies, and supplies values used for potential new records. Storage persists allowlisted
+  source metadata differences but never interprets them. MongoDB v1/v3 and PostgreSQL storage were
+  refactored to this candidate-first model (PostgreSQL gains a nullable `source_metadata` JSONB column
+  via migration).
+
+### Patch Changes
+
+- Updated dependencies [27b56cb]
+- Updated dependencies [798d739]
+  - @powersync/service-core@1.25.0
+
+## 0.19.0
+
+### Minor Changes
+
+- 2189250: Add `/sync/checkpoint-request` for client-supplied checkpoint request ids, previously called write checkpoint ids. The route returns the stored `checkpoint_request_id`, storage now treats managed request ids as monotonic per user/client, custom checkpoint request ids continue to use the existing `checkpoint` field for backwards compatibility, and `checkpoint_requested_at` metadata lets compact jobs remove expired request-derived checkpoint records.
+
+  This release includes storage migrations for the checkpoint request metadata. Self-hosters should run migrations as part of the upgrade.
+
+- 922f974: Add experimental support for storing bucket data chunks on S3.
+
+### Patch Changes
+
+- 37591e9: [MongoDB Storage] Use batched deletes for clearing data.
+- 63707af: Stability and performance fixes for MongoDB storage V3.
+- be14bfe: [MongoDB Storage V3] Tweak size estimates and chunking to use actual BSON sizes.
+- aab068b: [MongoDB Storage V3] Add has_clear_op field and simplify checksum queries.
+- b3cbc25: Update "Flushed X + Y + Z updates" log to include more detail
+- Updated dependencies [087b61e]
+- Updated dependencies [2189250]
+- Updated dependencies [922f974]
+- Updated dependencies [c4860c9]
+- Updated dependencies [37591e9]
+- Updated dependencies [483415d]
+- Updated dependencies [8daa300]
+- Updated dependencies [aab068b]
+- Updated dependencies [37591e9]
+- Updated dependencies [be42e25]
+- Updated dependencies [cb4c627]
+  - @powersync/service-core@1.24.0
+  - @powersync/lib-services-framework@0.10.0
+  - @powersync/service-types@0.17.0
+  - @powersync/lib-service-mongodb@0.6.30
+  - @powersync/service-sync-rules@0.40.0
+
+## 0.18.3
+
+### Patch Changes
+
+- ea31f64: Potential fix and improved stack trace for 'evicted' errors.
+- 1f854d3: Fix for occasional readConcern errors when using bulk_read_preference.
+- Updated dependencies [ea71bf3]
+- Updated dependencies [ea31f64]
+- Updated dependencies [edc6ed4]
+  - @powersync/service-sync-rules@0.39.0
+  - @powersync/service-core@1.23.3
+  - @powersync/lib-services-framework@0.9.8
+  - @powersync/lib-service-mongodb@0.6.29
+
+## 0.18.2
+
+### Patch Changes
+
+- e4f683d: [MongoDB Storage] Add experimental option to allow reading data from secondaries.
+- c2edf86: Add `api.parameters.bucket_count_cache_ttl_minutes` configuration option.
+- df9ab1e: Batch concurrent write checkpoint requests and increase request limits to improve throughput.
+- b9f6b6f: [MongoDB Storage] Optimize writes under high replication lag.
+- Updated dependencies [71d4a0a]
+- Updated dependencies [e4f683d]
+- Updated dependencies [71d4a0a]
+- Updated dependencies [a6ae678]
+- Updated dependencies [c2edf86]
+- Updated dependencies [df9ab1e]
+  - @powersync/service-core@1.23.2
+  - @powersync/lib-service-mongodb@0.6.28
+  - @powersync/service-sync-rules@0.38.1
+  - @powersync/service-types@0.16.1
+  - @powersync/lib-services-framework@0.9.7
+
+## 0.18.1
+
+### Patch Changes
+
+- 7e65360: Restructure replication locks for updated sync config.
+- Updated dependencies [7e65360]
+  - @powersync/service-core@1.23.1
+
+## 0.18.0
+
+### Minor Changes
+
+- a91a08f: [Experimental] Enable incremental reprocessing for MongoDB source + MongoDB storage. This includes significant changes to the v3 storage format.
+
+### Patch Changes
+
+- 101ea67: Include table names in "Cannot mark snapshot done while source tables still require snapshotting" error.
+- Updated dependencies [a91a08f]
+- Updated dependencies [184c39f]
+- Updated dependencies [c3f75df]
+- Updated dependencies [4bd35ea]
+  - @powersync/service-core@1.23.0
+  - @powersync/service-types@0.16.0
+  - @powersync/service-sync-rules@0.38.0
+  - @powersync/lib-services-framework@0.9.6
+  - @powersync/lib-service-mongodb@0.6.27
+
+## 0.17.0
 
 ### Minor Changes
 
 - e2bf1ad: [Internal] rework resolveTables to handle multiple SourceTables.
 - cd5a222: [MongoDB V3 Storage] Split storage for "sync config definition" versus "replication stream status".
 - 15e2466: [MongoDB] Support snapshotting concurrently with streaming in storage v3+.
+- ebeaa3b: [Postgres] Skip storing a redundant copy of replicated rows in `current_data` for tables with `REPLICA IDENTITY FULL`.
 
 ### Patch Changes
 
 - 6e2a57e: Refactor HydratedSyncConfig to support multiple SyncConfigs.
 - 92cc83b: Add the experimental `unstable_sqlite_expression_engine` sync config option to evaluate Sync Streams with SQLite.
+- Updated dependencies [17fd96b]
 - Updated dependencies [6e2a57e]
 - Updated dependencies [ec6df9f]
 - Updated dependencies [99d33d5]
@@ -24,12 +195,13 @@
 - Updated dependencies [92cc83b]
 - Updated dependencies [0aab0f9]
 - Updated dependencies [15e2466]
+- Updated dependencies [ebeaa3b]
 - Updated dependencies [b116857]
 - Updated dependencies [a94b6c3]
-  - @powersync/service-sync-rules@0.0.0-dev-20260602073133
-  - @powersync/service-core@0.0.0-dev-20260602073133
-  - @powersync/lib-services-framework@0.0.0-dev-20260602073133
-  - @powersync/lib-service-mongodb@0.0.0-dev-20260602073133
+  - @powersync/service-core@1.22.0
+  - @powersync/service-sync-rules@0.37.0
+  - @powersync/lib-services-framework@0.9.5
+  - @powersync/lib-service-mongodb@0.6.26
 
 ## 0.16.0
 

package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts

@@ -0,0 +1,3 @@
+import { migrations } from '@powersync/service-core';
+export declare const up: migrations.PowerSyncMigrationFunction;
+export declare const down: migrations.PowerSyncMigrationFunction;

package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js

@@ -0,0 +1,41 @@
+import * as storage from '../../../storage/storage-index.js';
+export const up = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = storage.createPowerSyncMongo(configuration.storage);
+    try {
+        await db.write_checkpoints.createIndex({
+            checkpoint_requested_at: 1
+        }, {
+            name: 'checkpoint_requested_at',
+            // Only client-requested checkpoints have this field; generated
+            // checkpoints leave it unset. This keeps the index limited to the
+            // documents the compact job's retention delete scans.
+            partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
+        });
+        await db.custom_write_checkpoints.createIndex({
+            checkpoint_requested_at: 1
+        }, {
+            name: 'checkpoint_requested_at',
+            partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
+        });
+    }
+    finally {
+        await db.client.close();
+    }
+};
+export const down = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = storage.createPowerSyncMongo(configuration.storage);
+    try {
+        if (await db.write_checkpoints.indexExists('checkpoint_requested_at')) {
+            await db.write_checkpoints.dropIndex('checkpoint_requested_at');
+        }
+        if (await db.custom_write_checkpoints.indexExists('checkpoint_requested_at')) {
+            await db.custom_write_checkpoints.dropIndex('checkpoint_requested_at');
+        }
+    }
+    finally {
+        await db.client.close();
+    }
+};
+//# sourceMappingURL=1782950400000-checkpoint-requested-at-index.js.map

package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"1782950400000-checkpoint-requested-at-index.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,mCAAmC,CAAC;AAG7D,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAErF,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,iBAAiB,CAAC,WAAW,CACpC;YACE,uBAAuB,EAAE,CAAC;SAC3B,EACD;YACE,IAAI,EAAE,yBAAyB;YAC/B,+DAA+D;YAC/D,kEAAkE;YAClE,sDAAsD;YACtD,uBAAuB,EAAE,EAAE,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;SACxE,CACF,CAAC;QACF,MAAM,EAAE,CAAC,wBAAwB,CAAC,WAAW,CAC3C;YACE,uBAAuB,EAAE,CAAC;SAC3B,EACD;YACE,IAAI,EAAE,yBAAyB;YAC/B,uBAAuB,EAAE,EAAE,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;SACxE,CACF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IAEZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAErF,IAAI,CAAC;QACH,IAAI,MAAM,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC,yBAAyB,CAAC,EAAE,CAAC;YACtE,MAAM,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QAClE,CAAC;QACD,IAAI,MAAM,EAAE,CAAC,wBAAwB,CAAC,WAAW,CAAC,yBAAyB,CAAC,EAAE,CAAC;YAC7E,MAAM,EAAE,CAAC,wBAAwB,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC"}

package/dist/storage/MongoBucketStorage.d.ts

@@ -1,35 +1,88 @@
 import { GetIntanceOptions, storage } from '@powersync/service-core';
 import { DO_NOT_LOG } from '@powersync/lib-services-framework';
+import { mongo } from '@powersync/lib-service-mongodb';
 import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { PowerSyncMongo } from './implementation/db.js';
 import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
-import { MongoPersistedSyncRulesContentV1, MongoPersistedSyncRulesContentV3 } from './implementation/MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from './implementation/MongoPersistedReplicationStream.js';
+import { ObjectStorage } from './implementation/v3/object-storage/ObjectStorage.js';
 export interface MongoBucketStorageOptions {
-    checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
+    checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+    objectStorage?: ObjectStorage;
+    inlineThresholdBytes?: number;
+    /**
+     * Prefix for replication stream name and Postgres logical replication slot name.
+     */
+    replicationStreamNamePrefix: string;
+    readPreference?: mongo.ReadPreference;
+    checksumCacheTtlMs?: number;
+    clearBatchThrottleRate?: number;
+    defaultStorageVersion?: number;
+    /**
+     * Reuse a compatible active replication stream by appending a new sync config.
+     *
+     * This currently requires source replication support. MongoDB sources can process multiple
+     * sync configs in one replication stream, but other source connectors still expect a single
+     * sync config per stream.
+     */
+    supportsMultipleSyncConfigs?: boolean;
 }
 export declare class MongoBucketStorage extends storage.BucketStorageFactory {
-    private internalOptions?;
+    private options;
     [DO_NOT_LOG]: boolean;
     private readonly client;
-    private readonly session;
-    readonly slot_name_prefix: string;
+    readonly replicationStreamNamePrefix: string;
     private activeStorageCache;
     readonly db: PowerSyncMongo;
-    constructor(db: PowerSyncMongo, options: {
-        slot_name_prefix: string;
-    }, internalOptions?: MongoBucketStorageOptions | undefined);
+    constructor(db: PowerSyncMongo, options: MongoBucketStorageOptions);
     [Symbol.asyncDispose](): Promise<void>;
-    getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage;
+    getInstance(replicationStream: storage.PersistedReplicationStream, options?: GetIntanceOptions): MongoSyncBucketStorage;
     getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier>;
-    restartReplication(sync_rules_group_id: number): Promise<void>;
-    private updateSyncRulesV3;
-    updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3>;
-    getActiveSyncRulesContent(): Promise<MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3 | null>;
-    private getSyncRulesContent;
-    getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3 | null>;
-    getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]>;
-    getStoppedSyncRules(): Promise<storage.PersistedSyncRulesContent[]>;
-    getActiveStorage(): Promise<MongoSyncBucketStorage | null>;
+    restartReplication(replicationStreamId: number): Promise<void>;
+    /**
+     * Keep an active config readable after its replication stream is invalidated.
+     *
+     * Incremental V3 streams can retain stopped historical configs and embed a deploying config.
+     * Read-side storage requires exactly one ACTIVE or ERRORED config, so only the active config
+     * transitions to ERRORED; every other embedded config remains or becomes STOP.
+     */
+    private errorActiveStreamForReplacement;
+    private updateSyncRulesInTransaction;
+    /**
+     * Update to a V3 sync config.
+     *
+     * Does support cases where existing replication streams are on V1.
+     */
+    private updateSyncConfigV3InTransaction;
+    private loadSyncConfigDefinitions;
+    /**
+     * Load _all_ definition mappings for a replication stream - used as a base to generate new ids.
+     */
+    private loadHistoricalSyncConfigRuleMappings;
+    private logIncrementalDefinitionChanges;
+    private appendSyncConfigToStream;
+    updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedReplicationStream>;
+    private withTransaction;
+    /**
+     * Update to a V1 sync config.
+     *
+     * Does support cases where existing replication streams are on V3.
+     */
+    private updateSyncConfigV1InTransaction;
+    /**
+     * Stop top-level and embedded processing work before creating a replacement stream.
+     *
+     * Use in the same transaction that the replacement stream is created.
+     */
+    private stopExistingProcessingWork;
+    getActiveSyncConfig(): Promise<storage.ResolvedSyncConfig | null>;
+    private getActiveSyncConfigInternal;
+    private replicationStreamFromDoc;
+    getDeployingSyncConfig(): Promise<storage.ResolvedSyncConfig | null>;
+    private getDeployingSyncConfigInternal;
+    getReplicatingReplicationStreams(): Promise<storage.PersistedReplicationStream[]>;
+    getStoppedReplicationStreams(): Promise<storage.PersistedReplicationStream[]>;
+    private resolvedSyncConfigFromDoc;
     getStorageMetrics(): Promise<storage.StorageMetrics>;
     getPowerSyncInstanceId(): Promise<string>;
 }