@powersync/service-module-mongodb 0.0.0-dev-20260929095716 → 0.0.0-dev-20261005120023
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -7
- package/dist/replication/ChangeStreamReplicator.d.ts +1 -0
- package/dist/replication/ChangeStreamReplicator.js +3 -0
- package/dist/replication/ChangeStreamReplicator.js.map +1 -1
- package/package.json +9 -9
- package/src/replication/ChangeStreamReplicator.ts +4 -0
- package/test/src/snapshot_query_provider.test.ts +18 -10
- package/test/src/stream_progress.test.ts +11 -7
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,10 +1,22 @@
|
|
|
1
1
|
# @powersync/service-module-mongodb
|
|
2
2
|
|
|
3
|
-
## 0.0.0-dev-
|
|
3
|
+
## 0.0.0-dev-20261005120023
|
|
4
4
|
|
|
5
5
|
### Minor Changes
|
|
6
6
|
|
|
7
|
-
-
|
|
7
|
+
- f706f43: Add a replication adapter interface for snapshot predicates and ordered change/progress streams. Persist safe resume positions after preceding writes have flushed, including when an adapter excludes all data events. Preserve exact-token resumes within transactions and export a pluggable, disposable ChangeStreamTestContext for module integration tests.
|
|
8
|
+
|
|
9
|
+
### Patch Changes
|
|
10
|
+
|
|
11
|
+
- Updated dependencies [ef5e87c]
|
|
12
|
+
- Updated dependencies [e703ca5]
|
|
13
|
+
- Updated dependencies [324093e]
|
|
14
|
+
- @powersync/service-sync-rules@0.0.0-dev-20261005120023
|
|
15
|
+
- @powersync/service-core@0.0.0-dev-20261005120023
|
|
16
|
+
- @powersync/lib-services-framework@0.0.0-dev-20261005120023
|
|
17
|
+
- @powersync/lib-service-mongodb@0.0.0-dev-20261005120023
|
|
18
|
+
|
|
19
|
+
## 0.21.2
|
|
8
20
|
|
|
9
21
|
### Patch Changes
|
|
10
22
|
|
|
@@ -12,18 +24,20 @@
|
|
|
12
24
|
- 54d6f4b: Report a MongoDB view matched by a wildcard table pattern as a warning, instead of failing sync config validation. Previously, a view matching a pattern such as `"orders%"` made the whole validation request fail with `Cannot get name for wildcard table`.
|
|
13
25
|
- 18f0c95: Add `fixed_booleans_in_json` to sync booleans in JSON columns as booleans instead of as 0/1 integer.
|
|
14
26
|
- Updated dependencies [0483d4e]
|
|
15
|
-
- Updated dependencies [c8e8b75]
|
|
16
27
|
- Updated dependencies [28f024b]
|
|
28
|
+
- Updated dependencies [5078032]
|
|
17
29
|
- Updated dependencies [424bbc0]
|
|
30
|
+
- Updated dependencies [e87703a]
|
|
18
31
|
- Updated dependencies [b70f467]
|
|
19
32
|
- Updated dependencies [18f0c95]
|
|
20
33
|
- Updated dependencies [c60db6e]
|
|
21
34
|
- Updated dependencies [0483d4e]
|
|
22
35
|
- Updated dependencies [a8c3f15]
|
|
23
|
-
- @powersync/service-core@
|
|
24
|
-
- @powersync/service-
|
|
25
|
-
- @powersync/
|
|
26
|
-
- @powersync/lib-
|
|
36
|
+
- @powersync/service-core@1.27.0
|
|
37
|
+
- @powersync/service-types@0.19.0
|
|
38
|
+
- @powersync/service-sync-rules@0.43.0
|
|
39
|
+
- @powersync/lib-services-framework@0.10.3
|
|
40
|
+
- @powersync/lib-service-mongodb@0.6.33
|
|
27
41
|
|
|
28
42
|
## 0.21.1
|
|
29
43
|
|
|
@@ -10,6 +10,7 @@ export declare class ChangeStreamReplicator extends replication.AbstractReplicat
|
|
|
10
10
|
private readonly connectionFactory;
|
|
11
11
|
private readonly createReplicationQueryProvider;
|
|
12
12
|
constructor(options: ChangeStreamReplicatorOptions);
|
|
13
|
+
protected get defaultSchema(): string;
|
|
13
14
|
createJob(options: replication.CreateJobOptions): ChangeStreamReplicationJob;
|
|
14
15
|
cleanUp(syncRulesStorage: storage.SyncRulesBucketStorage): Promise<void>;
|
|
15
16
|
stop(): Promise<void>;
|
|
@@ -9,6 +9,9 @@ export class ChangeStreamReplicator extends replication.AbstractReplicator {
|
|
|
9
9
|
this.connectionFactory = options.connectionFactory;
|
|
10
10
|
this.createReplicationQueryProvider = options.createReplicationQueryProvider;
|
|
11
11
|
}
|
|
12
|
+
get defaultSchema() {
|
|
13
|
+
return this.connectionFactory.dbConnectionConfig.database;
|
|
14
|
+
}
|
|
12
15
|
createJob(options) {
|
|
13
16
|
return new ChangeStreamReplicationJob({
|
|
14
17
|
id: this.createJobId(options.storage.replicationStreamId),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChangeStreamReplicator.js","sourceRoot":"","sources":["../../src/replication/ChangeStreamReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAW,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAS7E,MAAM,OAAO,sBAAuB,SAAQ,WAAW,CAAC,kBAA8C;IACnF,iBAAiB,CAA2B;IAC5C,8BAA8B,CAAmD;IAElG,YAAY,OAAsC;QAChD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,OAAO,CAAC,8BAA8B,CAAC;IAC/E,CAAC;IAED,SAAS,CAAC,OAAqC;QAC7C,OAAO,IAAI,0BAA0B,CAAC;YACpC,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC;YACzD,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,8BAA8B,EAAE,IAAI,CAAC,8BAA8B;YACnE,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,IAAI,CAAC,WAAW;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,gBAAgD;QAC5D,4BAA4B;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,OAAO,MAAM,WAAW,CAAC,cAAc,CAAC,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC;IACrF,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"ChangeStreamReplicator.js","sourceRoot":"","sources":["../../src/replication/ChangeStreamReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAW,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAS7E,MAAM,OAAO,sBAAuB,SAAQ,WAAW,CAAC,kBAA8C;IACnF,iBAAiB,CAA2B;IAC5C,8BAA8B,CAAmD;IAElG,YAAY,OAAsC;QAChD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,OAAO,CAAC,8BAA8B,CAAC;IAC/E,CAAC;IAED,IAAc,aAAa;QACzB,OAAO,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,QAAQ,CAAC;IAC5D,CAAC;IAED,SAAS,CAAC,OAAqC;QAC7C,OAAO,IAAI,0BAA0B,CAAC;YACpC,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC;YACzD,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,8BAA8B,EAAE,IAAI,CAAC,8BAA8B;YACnE,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,IAAI,CAAC,WAAW;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,gBAAgD;QAC5D,4BAA4B;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,OAAO,MAAM,WAAW,CAAC,cAAc,CAAC,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC;IACrF,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powersync/service-module-mongodb",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
|
-
"version": "0.0.0-dev-
|
|
4
|
+
"version": "0.0.0-dev-20261005120023",
|
|
5
5
|
"license": "FSL-1.1-ALv2",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"publishConfig": {
|
|
@@ -15,17 +15,17 @@
|
|
|
15
15
|
"bson": "^6.10.4",
|
|
16
16
|
"ts-codec": "^1.3.0",
|
|
17
17
|
"uuid": "^14.0.0",
|
|
18
|
-
"@powersync/lib-service-mongodb": "0.0.0-dev-
|
|
19
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
20
|
-
"@powersync/service-core": "0.0.0-dev-
|
|
18
|
+
"@powersync/lib-service-mongodb": "0.0.0-dev-20261005120023",
|
|
19
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20261005120023",
|
|
20
|
+
"@powersync/service-core": "0.0.0-dev-20261005120023",
|
|
21
21
|
"@powersync/service-jsonbig": "0.17.13",
|
|
22
|
-
"@powersync/service-sync-rules": "0.0.0-dev-
|
|
23
|
-
"@powersync/service-types": "0.
|
|
22
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20261005120023",
|
|
23
|
+
"@powersync/service-types": "0.19.0"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
27
|
-
"@powersync/service-module-mongodb-storage": "0.0.0-dev-
|
|
28
|
-
"@powersync/service-module-postgres-storage": "0.0.0-dev-
|
|
26
|
+
"@powersync/service-core-tests": "0.0.0-dev-20261005120023",
|
|
27
|
+
"@powersync/service-module-mongodb-storage": "0.0.0-dev-20261005120023",
|
|
28
|
+
"@powersync/service-module-postgres-storage": "0.0.0-dev-20261005120023"
|
|
29
29
|
},
|
|
30
30
|
"scripts": {
|
|
31
31
|
"build": "tsc -b",
|
|
@@ -19,6 +19,10 @@ export class ChangeStreamReplicator extends replication.AbstractReplicator<Chang
|
|
|
19
19
|
this.createReplicationQueryProvider = options.createReplicationQueryProvider;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
protected get defaultSchema(): string {
|
|
23
|
+
return this.connectionFactory.dbConnectionConfig.database;
|
|
24
|
+
}
|
|
25
|
+
|
|
22
26
|
createJob(options: replication.CreateJobOptions): ChangeStreamReplicationJob {
|
|
23
27
|
return new ChangeStreamReplicationJob({
|
|
24
28
|
id: this.createJobId(options.storage.replicationStreamId),
|
|
@@ -28,19 +28,23 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB snapshot quer
|
|
|
28
28
|
streamOptions: {
|
|
29
29
|
snapshotChunkLength: 2,
|
|
30
30
|
createReplicationQueryProvider: ({ connectionTag, defaultSchema, syncConfig }) => {
|
|
31
|
-
// Providers need the actual source namespace and parsed
|
|
32
|
-
// predicates. This sync config has no
|
|
31
|
+
// Providers need the actual source namespace and parsed source-table options to build their
|
|
32
|
+
// predicates. This sync config has no source-table options, so those are empty.
|
|
33
33
|
expect(connectionTag).toBe(context.connectionTag);
|
|
34
34
|
expect(defaultSchema).toBe(context.db.databaseName);
|
|
35
|
-
expect(syncConfig.
|
|
35
|
+
expect(syncConfig.sourceTableConfig).toEqual({});
|
|
36
36
|
return { ...DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER, getSnapshotFilter };
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
39
|
});
|
|
40
|
-
await context.updateSyncRules(/* yaml */ `
|
|
40
|
+
await context.updateSyncRules(/* yaml */ `
|
|
41
|
+
# Sync config fixture.
|
|
42
|
+
config:
|
|
43
|
+
edition: 3
|
|
44
|
+
streams:
|
|
41
45
|
global:
|
|
42
|
-
|
|
43
|
-
|
|
46
|
+
query: SELECT _id AS id, label FROM documents
|
|
47
|
+
`);
|
|
44
48
|
// The collection treats 'LIVE' and 'live' as equal by default. Provider filters use simple
|
|
45
49
|
// collation, so id 6 must be excluded for its 'LIVE' label even though it passes the id conditions.
|
|
46
50
|
const collection = await context.db.createCollection<{ _id: number; label: string }>('documents', {
|
|
@@ -50,7 +54,7 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB snapshot quer
|
|
|
50
54
|
Array.from({ length: 14 }, (_, i) => ({ _id: i + 1, label: (i + 1) % 3 == 0 ? 'LIVE' : 'live' }))
|
|
51
55
|
);
|
|
52
56
|
await context.replicateSnapshot();
|
|
53
|
-
const rows = (await context.getBucketData('global[]')).filter((op) => op.op == 'PUT');
|
|
57
|
+
const rows = (await context.getBucketData('global|0[]')).filter((op) => op.op == 'PUT');
|
|
54
58
|
// Matching ids 4, 8 and 10 require two pages at a page size of two. After the first page ends
|
|
55
59
|
// at id 8, pagination adds its own $expr requiring _id > 8. It must combine that condition
|
|
56
60
|
// with the provider's $expr: id 10 is included, while ids 9 and 11 through 14 remain excluded.
|
|
@@ -78,10 +82,14 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB snapshot quer
|
|
|
78
82
|
createReplicationQueryProvider: () => ({ validateSource, openChangeStream, getSnapshotFilter })
|
|
79
83
|
}
|
|
80
84
|
});
|
|
81
|
-
await context.updateSyncRules(/* yaml */ `
|
|
85
|
+
await context.updateSyncRules(/* yaml */ `
|
|
86
|
+
# Sync config fixture.
|
|
87
|
+
config:
|
|
88
|
+
edition: 3
|
|
89
|
+
streams:
|
|
82
90
|
global:
|
|
83
|
-
|
|
84
|
-
|
|
91
|
+
query: SELECT _id AS id FROM documents
|
|
92
|
+
`);
|
|
85
93
|
await expect(context.replicateSnapshot()).rejects.toThrow('unsupported test source');
|
|
86
94
|
expect(validateSource).toHaveBeenCalledWith({
|
|
87
95
|
connectionManager: context.connectionManager,
|
|
@@ -17,10 +17,14 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB durable adapt
|
|
|
17
17
|
let originalLsn: string;
|
|
18
18
|
{
|
|
19
19
|
await using context = await openChangeStreamTestContext(factory, { storageVersion });
|
|
20
|
-
await context.updateSyncRules(/* yaml */ `
|
|
20
|
+
await context.updateSyncRules(/* yaml */ `
|
|
21
|
+
# Sync config fixture.
|
|
22
|
+
config:
|
|
23
|
+
edition: 3
|
|
24
|
+
streams:
|
|
21
25
|
global:
|
|
22
|
-
|
|
23
|
-
|
|
26
|
+
query: SELECT _id AS id FROM documents WHERE _id = 1
|
|
27
|
+
`);
|
|
24
28
|
await context.db.createCollection('documents');
|
|
25
29
|
await context.replicateSnapshot();
|
|
26
30
|
await context.getCheckpoint();
|
|
@@ -115,7 +119,7 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB durable adapt
|
|
|
115
119
|
await context.loadActiveSyncRules();
|
|
116
120
|
context.startStreaming();
|
|
117
121
|
expect(
|
|
118
|
-
(await context.getBucketData('global[]')).filter((op) => op.op == 'PUT').map((op) => op.object_id)
|
|
122
|
+
(await context.getBucketData('global|0[]')).filter((op) => op.op == 'PUT').map((op) => op.object_id)
|
|
119
123
|
).toEqual(['1']);
|
|
120
124
|
});
|
|
121
125
|
|
|
@@ -125,7 +129,7 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB durable adapt
|
|
|
125
129
|
{
|
|
126
130
|
await using context = await openChangeStreamTestContext(factory, { storageVersion });
|
|
127
131
|
await context.updateSyncRules(
|
|
128
|
-
'
|
|
132
|
+
'config:\n edition: 3\nstreams:\n global:\n query: SELECT _id AS id FROM documents'
|
|
129
133
|
);
|
|
130
134
|
await context.db.createCollection('documents');
|
|
131
135
|
await context.replicateSnapshot();
|
|
@@ -168,7 +172,7 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB durable adapt
|
|
|
168
172
|
await context.loadActiveSyncRules();
|
|
169
173
|
context.startStreaming();
|
|
170
174
|
await context.getCheckpoint();
|
|
171
|
-
const operations = await context.getBucketData('global[]');
|
|
175
|
+
const operations = await context.getBucketData('global|0[]');
|
|
172
176
|
// Every insert in this transaction shares a clusterTime, but each has its own resume token.
|
|
173
177
|
// A timestamp dedupe guard here would lose the unprocessed suffix after the third batch.
|
|
174
178
|
expect(
|
|
@@ -189,7 +193,7 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB durable adapt
|
|
|
189
193
|
{
|
|
190
194
|
await using context = await openChangeStreamTestContext(factory, { storageVersion });
|
|
191
195
|
await context.updateSyncRules(
|
|
192
|
-
'
|
|
196
|
+
'config:\n edition: 3\nstreams:\n global:\n query: SELECT _id AS id FROM documents'
|
|
193
197
|
);
|
|
194
198
|
await context.db.createCollection('documents');
|
|
195
199
|
await context.replicateSnapshot();
|