@powersync/service-module-mongodb 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -7
- package/ci/test-connection.yaml +12 -0
- package/dist/api/MongoRouteAPIAdapter.d.ts +2 -2
- package/dist/api/MongoRouteAPIAdapter.js +73 -143
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/api/infer-collection-schema.d.ts +12 -0
- package/dist/api/infer-collection-schema.js +142 -0
- package/dist/api/infer-collection-schema.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/module/MongoModule.d.ts +1 -2
- package/dist/module/MongoModule.js +2 -2
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +18 -2
- package/dist/replication/ChangeStream.js +275 -307
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.d.ts +4 -1
- package/dist/replication/ChangeStreamReplicationJob.js +6 -2
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/ChangeStreamReplicator.d.ts +3 -0
- package/dist/replication/ChangeStreamReplicator.js +3 -0
- package/dist/replication/ChangeStreamReplicator.js.map +1 -1
- package/dist/replication/MongoRelation.js +5 -2
- package/dist/replication/MongoRelation.js.map +1 -1
- package/dist/replication/MongoReplicationQueryProvider.d.ts +87 -0
- package/dist/replication/MongoReplicationQueryProvider.js +23 -0
- package/dist/replication/MongoReplicationQueryProvider.js.map +1 -0
- package/dist/replication/MongoReplicationStream.d.ts +85 -0
- package/dist/replication/MongoReplicationStream.js +101 -0
- package/dist/replication/MongoReplicationStream.js.map +1 -0
- package/dist/replication/MongoSnapshotQuery.d.ts +2 -0
- package/dist/replication/MongoSnapshotQuery.js +12 -2
- package/dist/replication/MongoSnapshotQuery.js.map +1 -1
- package/dist/replication/MongoSnapshotter.d.ts +7 -2
- package/dist/replication/MongoSnapshotter.js +64 -91
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/RawChangeStream.d.ts +4 -12
- package/dist/replication/RawChangeStream.js +1 -18
- package/dist/replication/RawChangeStream.js.map +1 -1
- package/dist/replication/SourceRowConverter.d.ts +1 -0
- package/dist/replication/SourceRowConverter.js +4 -2
- package/dist/replication/SourceRowConverter.js.map +1 -1
- package/dist/replication/bufferToSqlite.d.ts +3 -1
- package/dist/replication/bufferToSqlite.js +35 -23
- package/dist/replication/bufferToSqlite.js.map +1 -1
- package/dist/replication/checkpoints/CheckpointImplementation.d.ts +4 -1
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +4 -1
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +1 -1
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -1
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +4 -1
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +2 -1
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -1
- package/dist/replication/replication-index.d.ts +6 -0
- package/dist/replication/replication-index.js +6 -0
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/test-utils/ChangeStreamTestContext.d.ts +131 -0
- package/dist/test-utils/ChangeStreamTestContext.js +391 -0
- package/dist/test-utils/ChangeStreamTestContext.js.map +1 -0
- package/dist/test-utils/test-utils-index.d.ts +1 -0
- package/dist/test-utils/test-utils-index.js +2 -0
- package/dist/test-utils/test-utils-index.js.map +1 -0
- package/dist/types/types.d.ts +5 -0
- package/dist/types/types.js +20 -2
- package/dist/types/types.js.map +1 -1
- package/package.json +9 -9
- package/src/api/MongoRouteAPIAdapter.ts +79 -127
- package/src/api/infer-collection-schema.ts +153 -0
- package/src/index.ts +1 -0
- package/src/module/MongoModule.ts +3 -4
- package/src/replication/ChangeStream.ts +303 -350
- package/src/replication/ChangeStreamReplicationJob.ts +9 -3
- package/src/replication/ChangeStreamReplicator.ts +5 -0
- package/src/replication/MongoRelation.ts +8 -1
- package/src/replication/MongoReplicationQueryProvider.ts +108 -0
- package/src/replication/MongoReplicationStream.ts +201 -0
- package/src/replication/MongoSnapshotQuery.ts +19 -3
- package/src/replication/MongoSnapshotter.ts +76 -98
- package/src/replication/RawChangeStream.ts +5 -32
- package/src/replication/SourceRowConverter.ts +4 -2
- package/src/replication/bufferToSqlite.ts +43 -23
- package/src/replication/checkpoints/CheckpointImplementation.ts +1 -1
- package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +2 -2
- package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +3 -2
- package/src/replication/replication-index.ts +6 -0
- package/src/test-utils/ChangeStreamTestContext.ts +520 -0
- package/src/test-utils/test-utils-index.ts +1 -0
- package/src/types/types.ts +27 -2
- package/test/src/buffer_to_sqlite.test.ts +9 -0
- package/test/src/change_stream.test.ts +76 -3
- package/test/src/change_stream_test_setup.ts +38 -0
- package/test/src/chunked_snapshot.test.ts +3 -3
- package/test/src/config.test.ts +34 -0
- package/test/src/documentdb_mode.test.ts +4 -5
- package/test/src/raw_change_stream.test.ts +0 -63
- package/test/src/replication_stream.test.ts +311 -0
- package/test/src/resume.test.ts +10 -4
- package/test/src/resuming_snapshots.test.ts +3 -3
- package/test/src/schema.test.ts +223 -0
- package/test/src/slow_tests.test.ts +3 -3
- package/test/src/snapshot_query_provider.test.ts +95 -0
- package/test/src/stream_progress.test.ts +312 -0
- package/test/src/util.ts +11 -29
- package/tsconfig.tsbuildinfo +1 -1
- package/test/src/change_stream_utils.ts +0 -375
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export * from './api/MongoRouteAPIAdapter.js';
|
|
2
2
|
export * from './module/MongoModule.js';
|
|
3
3
|
export * from './replication/replication-index.js';
|
|
4
|
+
export * as test_utils from './test-utils/test-utils-index.js';
|
|
4
5
|
export * from './types/types.js';
|
|
5
6
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,oCAAoC,CAAC;AACnD,cAAc,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,oCAAoC,CAAC;AACnD,OAAO,KAAK,UAAU,MAAM,kCAAkC,CAAC;AAC/D,cAAc,kBAAkB,CAAC"}
|
|
@@ -4,11 +4,10 @@ export declare class MongoModule extends replication.ReplicationModule<types.Mon
|
|
|
4
4
|
constructor();
|
|
5
5
|
protected createRouteAPIAdapter(): api.RouteAPI;
|
|
6
6
|
protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator;
|
|
7
|
-
onInitialized(context: system.ServiceContextContainer): Promise<void>;
|
|
8
7
|
/**
|
|
9
8
|
* Combines base config with normalized connection settings
|
|
10
9
|
*/
|
|
11
|
-
|
|
10
|
+
protected resolveConfig(config: types.MongoConnectionConfig): types.ResolvedConnectionConfig;
|
|
12
11
|
teardown(options: TearDownOptions): Promise<void>;
|
|
13
12
|
testConnection(config: types.MongoConnectionConfig): Promise<ConnectionTestResult>;
|
|
14
13
|
static testConnection(normalizedConfig: types.NormalizedMongoConnectionConfig): Promise<ConnectionTestResult>;
|
|
@@ -28,10 +28,10 @@ export class MongoModule extends replication.ReplicationModule {
|
|
|
28
28
|
storageEngine: context.storageEngine,
|
|
29
29
|
metricsEngine: context.metricsEngine,
|
|
30
30
|
connectionFactory: connectionFactory,
|
|
31
|
-
rateLimiter: new MongoErrorRateLimiter()
|
|
31
|
+
rateLimiter: new MongoErrorRateLimiter(),
|
|
32
|
+
heartbeatIntervalSeconds: normalisedConfig.heartbeat_interval_seconds
|
|
32
33
|
});
|
|
33
34
|
}
|
|
34
|
-
async onInitialized(context) { }
|
|
35
35
|
/**
|
|
36
36
|
* Combines base config with normalized connection settings
|
|
37
37
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoModule.js","sourceRoot":"","sources":["../../src/module/MongoModule.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAEL,kCAAkC,EAElC,WAAW,EAGZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,qBAAqB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,MAAM,qCAAqC,CAAC;AAC/E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAY,SAAQ,WAAW,CAAC,iBAA8C;IACzF;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS,CAAC,qBAAqB;YACrC,YAAY,EAAE,KAAK,CAAC,qBAAqB;SAC1C,CAAC,CAAC;IACL,CAAC;IAES,qBAAqB;QAC7B,OAAO,IAAI,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC,CAAC;IAC3E,CAAC;IAES,gBAAgB,CAAC,OAA8B;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;QAEzE,OAAO,IAAI,sBAAsB,CAAC;YAChC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,IAAI,EAAE,CAAC;YACtD,gBAAgB,EAAE,gBAAgB;YAClC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,iBAAiB,EAAE,iBAAiB;YACpC,WAAW,EAAE,IAAI,qBAAqB,EAAE;
|
|
1
|
+
{"version":3,"file":"MongoModule.js","sourceRoot":"","sources":["../../src/module/MongoModule.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAEL,kCAAkC,EAElC,WAAW,EAGZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,qBAAqB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,MAAM,qCAAqC,CAAC;AAC/E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAY,SAAQ,WAAW,CAAC,iBAA8C;IACzF;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS,CAAC,qBAAqB;YACrC,YAAY,EAAE,KAAK,CAAC,qBAAqB;SAC1C,CAAC,CAAC;IACL,CAAC;IAES,qBAAqB;QAC7B,OAAO,IAAI,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC,CAAC;IAC3E,CAAC;IAES,gBAAgB,CAAC,OAA8B;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;QAEzE,OAAO,IAAI,sBAAsB,CAAC;YAChC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,IAAI,EAAE,CAAC;YACtD,gBAAgB,EAAE,gBAAgB;YAClC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,iBAAiB,EAAE,iBAAiB;YACpC,WAAW,EAAE,IAAI,qBAAqB,EAAE;YACxC,wBAAwB,EAAE,gBAAgB,CAAC,0BAA0B;SACtE,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,MAAmC;QACzD,OAAO;YACL,GAAG,MAAM;YACT,GAAG,KAAK,CAAC,yBAAyB,CAAC,MAAM,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,QAAQ;IACV,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAmC;QACtD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,OAAO,MAAM,WAAW,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,gBAAuD;QACjF,MAAM,iBAAiB,GAAG,IAAI,YAAY,CAAC,gBAAgB,EAAE;YAC3D,8CAA8C;YAC9C,0DAA0D;YAC1D,0FAA0F;YAC1F,eAAe,EAAE,KAAK;YACtB,wBAAwB,EAAE,KAAK;SAChC,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,wBAAwB,CAAC,iBAAiB,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;QACD,OAAO;YACL,qBAAqB,EAAE,gBAAgB,CAAC,GAAG;SAC5C,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -3,6 +3,7 @@ import { DatabaseConnectionError, Logger } from '@powersync/lib-services-framewo
|
|
|
3
3
|
import { MetricsEngine, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core';
|
|
4
4
|
import { HydratedSyncConfig } from '@powersync/service-sync-rules';
|
|
5
5
|
import { MongoManager } from './MongoManager.js';
|
|
6
|
+
import { MongoReplicationQueryProviderFactory } from './MongoReplicationQueryProvider.js';
|
|
6
7
|
import { MongoSnapshotterHooks } from './MongoSnapshotter.js';
|
|
7
8
|
import { ProjectedChangeStreamDocument } from './RawChangeStream.js';
|
|
8
9
|
export interface ChangeStreamOptions {
|
|
@@ -27,6 +28,10 @@ export interface ChangeStreamOptions {
|
|
|
27
28
|
keepaliveIntervalMs?: number;
|
|
28
29
|
storageHooks?: storage.StorageHooks;
|
|
29
30
|
snapshotHooks?: MongoSnapshotterHooks;
|
|
31
|
+
/**
|
|
32
|
+
* Create one adapter per replication attempt, shared with its snapshotter and source probes.
|
|
33
|
+
*/
|
|
34
|
+
createReplicationQueryProvider?: MongoReplicationQueryProviderFactory;
|
|
30
35
|
logger?: Logger;
|
|
31
36
|
}
|
|
32
37
|
/**
|
|
@@ -65,14 +70,23 @@ export declare class ChangeStream {
|
|
|
65
70
|
private changeStreamTimeout;
|
|
66
71
|
private storageHooks;
|
|
67
72
|
private readonly sourceRowConverter;
|
|
73
|
+
private readonly queryProvider;
|
|
68
74
|
private keepaliveIntervalMs;
|
|
69
75
|
private isDocumentDb;
|
|
70
76
|
private _checkpointImplementation;
|
|
77
|
+
/**
|
|
78
|
+
* Last persisted resume timestamp as Date.now(). This represents the equivalent timestamp from the
|
|
79
|
+
* source database, rather than the time when we persisted it.
|
|
80
|
+
*/
|
|
81
|
+
private lastPersistedResumeTimestamp;
|
|
82
|
+
private lastBatchCheckpoint;
|
|
71
83
|
constructor(options: ChangeStreamOptions);
|
|
72
84
|
get stopped(): boolean;
|
|
73
85
|
private get usePostImages();
|
|
74
86
|
private get configurePostImages();
|
|
75
|
-
/**
|
|
87
|
+
/**
|
|
88
|
+
* The active checkpoint strategy. Only valid after ensureDetected().
|
|
89
|
+
*/
|
|
76
90
|
private get checkpointImplementation();
|
|
77
91
|
/**
|
|
78
92
|
* Detect DocumentDB and select the checkpoint implementation for the streaming
|
|
@@ -94,10 +108,12 @@ export declare class ChangeStream {
|
|
|
94
108
|
private initReplication;
|
|
95
109
|
private cleanupStoppedSyncConfigs;
|
|
96
110
|
private streamChanges;
|
|
97
|
-
private
|
|
111
|
+
private openChangeStream;
|
|
98
112
|
private rawToSqliteRow;
|
|
113
|
+
private createBatchCheckpoint;
|
|
99
114
|
streamChangesInternal(): Promise<void>;
|
|
100
115
|
getReplicationLagMillis(): number | undefined;
|
|
116
|
+
keepAlive(): Promise<void>;
|
|
101
117
|
private lastTouchedAt;
|
|
102
118
|
private touch;
|
|
103
119
|
}
|