@powersync/service-module-mongodb 0.13.1 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. package/CHANGELOG.md +39 -0
  2. package/dist/replication/MongoManager.js +9 -8
  3. package/dist/replication/MongoManager.js.map +1 -1
  4. package/dist/types/types.d.ts +9 -7
  5. package/dist/types/types.js +1 -1
  6. package/dist/types/types.js.map +1 -1
  7. package/package.json +9 -9
  8. package/src/replication/MongoManager.ts +10 -8
  9. package/src/types/types.ts +4 -1
  10. package/test/src/change_stream.test.ts +20 -18
  11. package/test/src/change_stream_utils.ts +47 -12
  12. package/test/src/chunked_snapshot.test.ts +8 -4
  13. package/test/src/resume.test.ts +8 -5
  14. package/test/src/resuming_snapshots.test.ts +9 -5
  15. package/test/src/slow_tests.test.ts +8 -6
  16. package/test/src/util.ts +34 -9
  17. package/test/tsconfig.json +3 -7
  18. package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md CHANGED
@@ -1,5 +1,44 @@
1
1
  # @powersync/service-module-mongodb
2
2
 
3
+ ## 0.14.0
4
+
5
+ ### Minor Changes
6
+
7
+ - 8a4c34e: Refactor `BucketStorageFactory` and `PersistedSyncRulesContent` to be abstract classes instead of interfaces.
8
+ - 8bd83e8: Introduce storage versions.
9
+
10
+ ### Patch Changes
11
+
12
+ - 15aea77: Support connection parameters via database URL query string. PostgreSQL supports `connect_timeout`. MongoDB supports `connectTimeoutMS`, `socketTimeoutMS`, `serverSelectionTimeoutMS`, `maxPoolSize`, `maxIdleTimeMS`. MySQL supports `connectTimeout`, `connectionLimit`, `queueLimit`.
13
+ - Updated dependencies [15aea77]
14
+ - Updated dependencies [0998251]
15
+ - Updated dependencies [65f3c89]
16
+ - Updated dependencies [1c45667]
17
+ - Updated dependencies [8785a3f]
18
+ - Updated dependencies [8a4c34e]
19
+ - Updated dependencies [b440093]
20
+ - Updated dependencies [d7ff4ad]
21
+ - Updated dependencies [c683322]
22
+ - Updated dependencies [8bd83e8]
23
+ - Updated dependencies [83989b2]
24
+ - Updated dependencies [79a9729]
25
+ - Updated dependencies [5edd95f]
26
+ - @powersync/lib-service-mongodb@0.6.20
27
+ - @powersync/service-core@1.20.0
28
+ - @powersync/service-types@0.15.0
29
+ - @powersync/service-sync-rules@0.32.0
30
+ - @powersync/lib-services-framework@0.8.3
31
+
32
+ ## 0.13.2
33
+
34
+ ### Patch Changes
35
+
36
+ - Updated dependencies [a04252d]
37
+ - @powersync/service-sync-rules@0.31.1
38
+ - @powersync/lib-services-framework@0.8.2
39
+ - @powersync/service-core@1.19.2
40
+ - @powersync/lib-service-mongodb@0.6.19
41
+
3
42
  ## 0.13.1
4
43
 
5
44
  ### Patch Changes
package/dist/replication/MongoManager.js CHANGED
@@ -11,6 +11,7 @@ export class MongoManager extends BaseObserver {
11
11
  constructor(options, overrides) {
12
12
  super();
13
13
  this.options = options;
14
+ const params = options.connectionParams;
14
15
  // The pool is lazy - no connections are opened until a query is performed.
15
16
  this.client = new mongo.MongoClient(options.uri, {
16
17
  auth: {
@@ -18,12 +19,12 @@ export class MongoManager extends BaseObserver {
18
19
  password: options.password
19
20
  },
20
21
  lookup: options.lookup,
21
- // Time for connection to timeout
22
- connectTimeoutMS: 5_000,
23
- // Time for individual requests to timeout
24
- socketTimeoutMS: 60_000,
25
- // How long to wait for new primary selection
26
- serverSelectionTimeoutMS: 30_000,
22
+ // Time for connection to timeout (URL param overrides default)
23
+ connectTimeoutMS: params.connectTimeoutMS ?? 5_000,
24
+ // Time for individual requests to timeout (URL param overrides default)
25
+ socketTimeoutMS: params.socketTimeoutMS ?? 60_000,
26
+ // How long to wait for new primary selection (URL param overrides default)
27
+ serverSelectionTimeoutMS: params.serverSelectionTimeoutMS ?? 30_000,
27
28
  // Identify the client
28
29
  appName: `powersync ${POWERSYNC_VERSION}`,
29
30
  // Deprecated in the driver - in a future release we may have to rely on appName only.
@@ -35,9 +36,9 @@ export class MongoManager extends BaseObserver {
35
36
  // Avoid too many connections:
36
37
  // 1. It can overwhelm the source database.
37
38
  // 2. Processing too many queries in parallel can cause the process to run out of memory.
38
- maxPoolSize: 8,
39
+ maxPoolSize: params.maxPoolSize ?? 8,
39
40
  maxConnecting: 3,
40
- maxIdleTimeMS: 60_000,
41
+ maxIdleTimeMS: params.maxIdleTimeMS ?? 60_000,
41
42
  ...BSON_DESERIALIZE_DATA_OPTIONS,
42
43
  ...overrides
43
44
  });
package/dist/replication/MongoManager.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"MongoManager.js","sourceRoot":"","sources":["../../src/replication/MongoManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAGvD,OAAO,EAAE,6BAA6B,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAMjE;;GAEG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAkC;IAKzD;IAJO,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAE7B,YACS,OAAwC,EAC/C,SAAoC;QAEpC,KAAK,EAAE,CAAC;QAHD,YAAO,GAAP,OAAO,CAAiC;QAI/C,2EAA2E;QAC3E,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE;YAC/C,IAAI,EAAE;gBACJ,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B;YAED,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,iCAAiC;YACjC,gBAAgB,EAAE,KAAK;YACvB,0CAA0C;YAC1C,eAAe,EAAE,MAAM;YACvB,6CAA6C;YAC7C,wBAAwB,EAAE,MAAM;YAEhC,sBAAsB;YACtB,OAAO,EAAE,aAAa,iBAAiB,EAAE;YACzC,sFAAsF;YACtF,UAAU,EAAE;gBACV,4CAA4C;gBAC5C,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE,iBAAiB;aAC3B;YAED,8BAA8B;YAC9B,2CAA2C;YAC3C,yFAAyF;YACzF,WAAW,EAAE,CAAC;YAEd,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,MAAM;YAErB,GAAG,6BAA6B;YAEhC,GAAG,SAAS;SACb,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,gBAAgB,CAAC,CAAC,QAAQ,EAAE,EAAE;YACjC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
1
+ {"version":3,"file":"MongoManager.js","sourceRoot":"","sources":["../../src/replication/MongoManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAGvD,OAAO,EAAE,6BAA6B,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAMjE;;GAEG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAkC;IAKzD;IAJO,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAE7B,YACS,OAAwC,EAC/C,SAAoC;QAEpC,KAAK,EAAE,CAAC;QAHD,YAAO,GAAP,OAAO,CAAiC;QAI/C,MAAM,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;QAExC,2EAA2E;QAC3E,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE;YAC/C,IAAI,EAAE;gBACJ,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B;YAED,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,+DAA+D;YAC/D,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,KAAK;YAClD,wEAAwE;YACxE,eAAe,EAAE,MAAM,CAAC,eAAe,IAAI,MAAM;YACjD,2EAA2E;YAC3E,wBAAwB,EAAE,MAAM,CAAC,wBAAwB,IAAI,MAAM;YAEnE,sBAAsB;YACtB,OAAO,EAAE,aAAa,iBAAiB,EAAE;YACzC,sFAAsF;YACtF,UAAU,EAAE;gBACV,4CAA4C;gBAC5C,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE,iBAAiB;aAC3B;YAED,8BAA8B;YAC9B,2CAA2C;YAC3C,yFAAyF;YACzF,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC;YAEpC,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,MAAM;YAE7C,GAAG,6BAA6B;YAEhC,GAAG,SAAS;SACb,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,gBAAgB,CAAC,CAAC,QAAQ,EAAE,EAAE;YACjC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
package/dist/types/types.d.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import type { MongoConnectionParams } from '@powersync/lib-service-mongodb/types';
1
2
  import { LookupFunction } from 'node:net';
2
3
  import * as t from 'ts-codec';
3
4
  export declare enum PostImagesOption {
@@ -42,31 +43,32 @@ export interface NormalizedMongoConnectionConfig {
42
43
  password?: string;
43
44
  lookup?: LookupFunction;
44
45
  postImages: PostImagesOption;
46
+ connectionParams: MongoConnectionParams;
45
47
  }
46
48
  export declare const MongoConnectionConfig: t.Intersection<t.Codec<{
49
+ type: string;
50
+ id?: string | undefined;
51
+ tag?: string | undefined;
52
+ debug_api?: boolean | undefined;
53
+ } & {
47
54
  uri: string;
48
55
  type: "mongodb";
49
56
  database?: string | undefined;
50
57
  username?: string | undefined;
51
58
  password?: string | undefined;
52
59
  reject_ip_ranges?: string[] | undefined;
53
- } & {
60
+ }, {
54
61
  type: string;
55
62
  id?: string | undefined;
56
63
  tag?: string | undefined;
57
64
  debug_api?: boolean | undefined;
58
- }, {
65
+ } & {
59
66
  uri: string;
60
67
  type: "mongodb";
61
68
  database?: string | undefined;
62
69
  username?: string | undefined;
63
70
  password?: string | undefined;
64
71
  reject_ip_ranges?: string[] | undefined;
65
- } & {
66
- type: string;
67
- id?: string | undefined;
68
- tag?: string | undefined;
69
- debug_api?: boolean | undefined;
70
72
  }, string, t.CodecProps>, t.ObjectCodec<{
71
73
  post_images: t.OptionalCodec<t.Codec<"off" | "auto_configure" | "read_only", "off" | "auto_configure" | "read_only", string, t.CodecProps>>;
72
74
  }>>;
package/dist/types/types.js CHANGED
@@ -35,7 +35,7 @@ export var PostImagesOption;
35
35
  */
36
36
  PostImagesOption["READ_ONLY"] = "read_only";
37
37
  })(PostImagesOption || (PostImagesOption = {}));
38
- export const MongoConnectionConfig = lib_mongo.BaseMongoConfig.and(service_types.configFile.DataSourceConfig).and(t.object({
38
+ export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_mongo.BaseMongoConfig).and(t.object({
39
39
  // Replication specific settings
40
40
  post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional()
41
41
  }));
package/dist/types/types.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,sCAAsC,CAAC;AAClE,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAN,IAAY,gBAkCX;AAlCD,WAAY,gBAAgB;IAC1B;;;;;;;OAOG;IACH,+BAAW,CAAA;IAEX;;;;;;;OAOG;IACH,qDAAiC,CAAA;IAEjC;;;;;;;;;;;OAWG;IACH,2CAAuB,CAAA;AACzB,CAAC,EAlCW,gBAAgB,KAAhB,gBAAgB,QAkC3B;AAiBD,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAC/G,CAAC,CAAC,MAAM,CAAC;IACP,gCAAgC;IAChC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;CACpG,CAAC,CACH,CAAC;AAaF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAqC;IAC7E,MAAM,IAAI,GAAG,SAAS,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAErD,OAAO;QACL,GAAG,IAAI;QACP,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAC7B,UAAU,EAAG,OAAO,CAAC,WAA4C,IAAI,gBAAgB,CAAC,GAAG;KAC1F,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,sCAAsC,CAAC;AAElE,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAN,IAAY,gBAkCX;AAlCD,WAAY,gBAAgB;IAC1B;;;;;;;OAOG;IACH,+BAAW,CAAA;IAEX;;;;;;;OAOG;IACH,qDAAiC,CAAA;IAEjC;;;;;;;;;;;OAWG;IACH,2CAAuB,CAAA;AACzB,CAAC,EAlCW,gBAAgB,KAAhB,gBAAgB,QAkC3B;AAmBD,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,GAAG,CAC/G,CAAC,CAAC,MAAM,CAAC;IACP,gCAAgC;IAChC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;CACpG,CAAC,CACH,CAAC;AAaF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAqC;IAC7E,MAAM,IAAI,GAAG,SAAS,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAErD,OAAO;QACL,GAAG,IAAI;QACP,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAC7B,UAAU,EAAG,OAAO,CAAC,WAA4C,IAAI,gBAAgB,CAAC,GAAG;KAC1F,CAAC;AACJ,CAAC"}
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@powersync/service-module-mongodb",
3
3
  "repository": "https://github.com/powersync-ja/powersync-service",
4
4
  "types": "dist/index.d.ts",
5
- "version": "0.13.1",
5
+ "version": "0.14.0",
6
6
  "main": "dist/index.js",
7
7
  "license": "FSL-1.1-ALv2",
8
8
  "type": "module",
@@ -25,17 +25,17 @@
25
25
  "bson": "^6.10.4",
26
26
  "ts-codec": "^1.3.0",
27
27
  "uuid": "^11.1.0",
28
- "@powersync/lib-service-mongodb": "0.6.18",
29
- "@powersync/lib-services-framework": "0.8.1",
30
- "@powersync/service-core": "1.19.1",
28
+ "@powersync/lib-service-mongodb": "0.6.20",
29
+ "@powersync/lib-services-framework": "0.8.3",
30
+ "@powersync/service-core": "1.20.0",
31
31
  "@powersync/service-jsonbig": "0.17.12",
32
- "@powersync/service-sync-rules": "0.31.0",
33
- "@powersync/service-types": "0.14.0"
32
+ "@powersync/service-sync-rules": "0.32.0",
33
+ "@powersync/service-types": "0.15.0"
34
34
  },
35
35
  "devDependencies": {
36
- "@powersync/service-core-tests": "0.13.1",
37
- "@powersync/service-module-mongodb-storage": "0.13.1",
38
- "@powersync/service-module-postgres-storage": "0.11.1"
36
+ "@powersync/service-core-tests": "0.14.0",
37
+ "@powersync/service-module-mongodb-storage": "0.14.0",
38
+ "@powersync/service-module-postgres-storage": "0.12.0"
39
39
  },
40
40
  "scripts": {
41
41
  "build": "tsc -b",
package/src/replication/MongoManager.ts CHANGED
@@ -20,6 +20,8 @@ export class MongoManager extends BaseObserver<MongoManagerListener> {
20
20
  overrides?: mongo.MongoClientOptions
21
21
  ) {
22
22
  super();
23
+ const params = options.connectionParams;
24
+
23
25
  // The pool is lazy - no connections are opened until a query is performed.
24
26
  this.client = new mongo.MongoClient(options.uri, {
25
27
  auth: {
@@ -28,12 +30,12 @@ export class MongoManager extends BaseObserver<MongoManagerListener> {
28
30
  },
29
31
 
30
32
  lookup: options.lookup,
31
- // Time for connection to timeout
32
- connectTimeoutMS: 5_000,
33
- // Time for individual requests to timeout
34
- socketTimeoutMS: 60_000,
35
- // How long to wait for new primary selection
36
- serverSelectionTimeoutMS: 30_000,
33
+ // Time for connection to timeout (URL param overrides default)
34
+ connectTimeoutMS: params.connectTimeoutMS ?? 5_000,
35
+ // Time for individual requests to timeout (URL param overrides default)
36
+ socketTimeoutMS: params.socketTimeoutMS ?? 60_000,
37
+ // How long to wait for new primary selection (URL param overrides default)
38
+ serverSelectionTimeoutMS: params.serverSelectionTimeoutMS ?? 30_000,
37
39
 
38
40
  // Identify the client
39
41
  appName: `powersync ${POWERSYNC_VERSION}`,
@@ -47,10 +49,10 @@ export class MongoManager extends BaseObserver<MongoManagerListener> {
47
49
  // Avoid too many connections:
48
50
  // 1. It can overwhelm the source database.
49
51
  // 2. Processing too many queries in parallel can cause the process to run out of memory.
50
- maxPoolSize: 8,
52
+ maxPoolSize: params.maxPoolSize ?? 8,
51
53
 
52
54
  maxConnecting: 3,
53
- maxIdleTimeMS: 60_000,
55
+ maxIdleTimeMS: params.maxIdleTimeMS ?? 60_000,
54
56
 
55
57
  ...BSON_DESERIALIZE_DATA_OPTIONS,
56
58
 
package/src/types/types.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb/types';
2
+ import type { MongoConnectionParams } from '@powersync/lib-service-mongodb/types';
2
3
  import * as service_types from '@powersync/service-types';
3
4
  import { LookupFunction } from 'node:net';
4
5
  import * as t from 'ts-codec';
@@ -52,9 +53,11 @@ export interface NormalizedMongoConnectionConfig {
52
53
  lookup?: LookupFunction;
53
54
 
54
55
  postImages: PostImagesOption;
56
+
57
+ connectionParams: MongoConnectionParams;
55
58
  }
56
59
 
57
- export const MongoConnectionConfig = lib_mongo.BaseMongoConfig.and(service_types.configFile.DataSourceConfig).and(
60
+ export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_mongo.BaseMongoConfig).and(
58
61
  t.object({
59
62
  // Replication specific settings
60
63
  post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional()
package/test/src/change_stream.test.ts CHANGED
@@ -3,12 +3,11 @@ import { setTimeout } from 'node:timers/promises';
3
3
  import { describe, expect, test, vi } from 'vitest';
4
4
 
5
5
  import { mongo } from '@powersync/lib-service-mongodb';
6
- import { storage } from '@powersync/service-core';
7
6
  import { test_utils } from '@powersync/service-core-tests';
8
7
 
9
8
  import { PostImagesOption } from '@module/types/types.js';
10
9
  import { ChangeStreamTestContext } from './change_stream_utils.js';
11
- import { describeWithStorage } from './util.js';
10
+ import { describeWithStorage, StorageVersionTestContext } from './util.js';
12
11
 
13
12
  const BASIC_SYNC_RULES = `
14
13
  bucket_definitions:
@@ -21,9 +20,12 @@ describe('change stream', () => {
21
20
  describeWithStorage({ timeout: 20_000 }, defineChangeStreamTests);
22
21
  });
23
22
 
24
- function defineChangeStreamTests(factory: storage.TestStorageFactory) {
23
+ function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
24
+ const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
25
+ return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
26
+ };
25
27
  test('replicating basic values', async () => {
26
- await using context = await ChangeStreamTestContext.open(factory, {
28
+ await using context = await openContext({
27
29
  mongoOptions: { postImages: PostImagesOption.READ_ONLY }
28
30
  });
29
31
  const { db } = context;
@@ -59,7 +61,7 @@ bucket_definitions:
59
61
  });
60
62
 
61
63
  test('replicating wildcard', async () => {
62
- await using context = await ChangeStreamTestContext.open(factory);
64
+ await using context = await openContext();
63
65
  const { db } = context;
64
66
  await context.updateSyncRules(`
65
67
  bucket_definitions:
@@ -91,7 +93,7 @@ bucket_definitions:
91
93
  });
92
94
 
93
95
  test('updateLookup - no fullDocument available', async () => {
94
- await using context = await ChangeStreamTestContext.open(factory, {
96
+ await using context = await openContext({
95
97
  mongoOptions: { postImages: PostImagesOption.OFF }
96
98
  });
97
99
  const { db, client } = context;
@@ -137,7 +139,7 @@ bucket_definitions:
137
139
  test('postImages - autoConfigure', async () => {
138
140
  // Similar to the above test, but with postImages enabled.
139
141
  // This resolves the consistency issue.
140
- await using context = await ChangeStreamTestContext.open(factory, {
142
+ await using context = await openContext({
141
143
  mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
142
144
  });
143
145
  const { db, client } = context;
@@ -185,7 +187,7 @@ bucket_definitions:
185
187
  test('postImages - on', async () => {
186
188
  // Similar to postImages - autoConfigure, but does not auto-configure.
187
189
  // changeStreamPreAndPostImages must be manually configured.
188
- await using context = await ChangeStreamTestContext.open(factory, {
190
+ await using context = await openContext({
189
191
  mongoOptions: { postImages: PostImagesOption.READ_ONLY }
190
192
  });
191
193
  const { db, client } = context;
@@ -230,7 +232,7 @@ bucket_definitions:
230
232
  });
231
233
 
232
234
  test('replicating case sensitive table', async () => {
233
- await using context = await ChangeStreamTestContext.open(factory);
235
+ await using context = await openContext();
234
236
  const { db } = context;
235
237
  await context.updateSyncRules(`
236
238
  bucket_definitions:
@@ -254,7 +256,7 @@ bucket_definitions:
254
256
  });
255
257
 
256
258
  test('replicating large values', async () => {
257
- await using context = await ChangeStreamTestContext.open(factory);
259
+ await using context = await openContext();
258
260
  const { db } = context;
259
261
  await context.updateSyncRules(`
260
262
  bucket_definitions:
@@ -285,7 +287,7 @@ bucket_definitions:
285
287
  });
286
288
 
287
289
  test('replicating dropCollection', async () => {
288
- await using context = await ChangeStreamTestContext.open(factory);
290
+ await using context = await openContext();
289
291
  const { db } = context;
290
292
  const syncRuleContent = `
291
293
  bucket_definitions:
@@ -317,7 +319,7 @@ bucket_definitions:
317
319
  });
318
320
 
319
321
  test('replicating renameCollection', async () => {
320
- await using context = await ChangeStreamTestContext.open(factory);
322
+ await using context = await openContext();
321
323
  const { db } = context;
322
324
  const syncRuleContent = `
323
325
  bucket_definitions:
@@ -348,7 +350,7 @@ bucket_definitions:
348
350
  });
349
351
 
350
352
  test('initial sync', async () => {
351
- await using context = await ChangeStreamTestContext.open(factory);
353
+ await using context = await openContext();
352
354
  const { db } = context;
353
355
  await context.updateSyncRules(BASIC_SYNC_RULES);
354
356
 
@@ -373,7 +375,7 @@ bucket_definitions:
373
375
  // MongoServerError: PlanExecutor error during aggregation :: caused by :: BSONObj size: 33554925 (0x20001ED) is invalid.
374
376
  // Size must be between 0 and 16793600(16MB)
375
377
 
376
- await using context = await ChangeStreamTestContext.open(factory);
378
+ await using context = await openContext();
377
379
  await context.updateSyncRules(`bucket_definitions:
378
380
  global:
379
381
  data:
@@ -422,7 +424,7 @@ bucket_definitions:
422
424
  });
423
425
 
424
426
  test('collection not in sync rules', async () => {
425
- await using context = await ChangeStreamTestContext.open(factory);
427
+ await using context = await openContext();
426
428
  const { db } = context;
427
429
  await context.updateSyncRules(BASIC_SYNC_RULES);
428
430
 
@@ -439,7 +441,7 @@ bucket_definitions:
439
441
  });
440
442
 
441
443
  test('postImages - new collection with postImages enabled', async () => {
442
- await using context = await ChangeStreamTestContext.open(factory, {
444
+ await using context = await openContext({
443
445
  mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
444
446
  });
445
447
  const { db } = context;
@@ -472,7 +474,7 @@ bucket_definitions:
472
474
  });
473
475
 
474
476
  test('postImages - new collection with postImages disabled', async () => {
475
- await using context = await ChangeStreamTestContext.open(factory, {
477
+ await using context = await openContext({
476
478
  mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
477
479
  });
478
480
  const { db } = context;
@@ -502,7 +504,7 @@ bucket_definitions:
502
504
  });
503
505
 
504
506
  test('recover from error', async () => {
505
- await using context = await ChangeStreamTestContext.open(factory);
507
+ await using context = await openContext();
506
508
  const { db } = context;
507
509
  await context.updateSyncRules(`
508
510
  bucket_definitions:
package/test/src/change_stream_utils.ts CHANGED
@@ -4,11 +4,15 @@ import {
4
4
  createCoreReplicationMetrics,
5
5
  initializeCoreReplicationMetrics,
6
6
  InternalOpId,
7
+ LEGACY_STORAGE_VERSION,
7
8
  OplogEntry,
8
9
  ProtocolOpId,
9
10
  ReplicationCheckpoint,
11
+ STORAGE_VERSION_CONFIG,
10
12
  SyncRulesBucketStorage,
11
- TestStorageOptions
13
+ TestStorageOptions,
14
+ updateSyncRulesFromYaml,
15
+ utils
12
16
  } from '@powersync/service-core';
13
17
  import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
14
18
 
@@ -23,6 +27,7 @@ export class ChangeStreamTestContext {
23
27
  private _walStream?: ChangeStream;
24
28
  private abortController = new AbortController();
25
29
  private streamPromise?: Promise<void>;
30
+ private syncRulesId?: number;
26
31
  public storage?: SyncRulesBucketStorage;
27
32
 
28
33
  /**
@@ -35,6 +40,7 @@ export class ChangeStreamTestContext {
35
40
  factory: (options: TestStorageOptions) => Promise<BucketStorageFactory>,
36
41
  options?: {
37
42
  doNotClear?: boolean;
43
+ storageVersion?: number;
38
44
  mongoOptions?: Partial<NormalizedMongoConnectionConfig>;
39
45
  streamOptions?: Partial<ChangeStreamOptions>;
40
46
  }
@@ -45,13 +51,19 @@ export class ChangeStreamTestContext {
45
51
  if (!options?.doNotClear) {
46
52
  await clearTestDb(connectionManager.db);
47
53
  }
48
- return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions);
54
+
55
+ const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION;
56
+ const versionedBuckets = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false;
57
+
58
+ return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions, storageVersion, versionedBuckets);
49
59
  }
50
60
 
51
61
  constructor(
52
62
  public factory: BucketStorageFactory,
53
63
  public connectionManager: MongoManager,
54
- private streamOptions?: Partial<ChangeStreamOptions>
64
+ private streamOptions: Partial<ChangeStreamOptions> = {},
65
+ private storageVersion: number = LEGACY_STORAGE_VERSION,
66
+ private versionedBuckets: boolean = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false
55
67
  ) {
56
68
  createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
57
69
  initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
@@ -88,7 +100,10 @@ export class ChangeStreamTestContext {
88
100
  }
89
101
 
90
102
  async updateSyncRules(content: string) {
91
- const syncRules = await this.factory.updateSyncRules({ content: content, validate: true });
103
+ const syncRules = await this.factory.updateSyncRules(
104
+ updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion })
105
+ );
106
+ this.syncRulesId = syncRules.id;
92
107
  this.storage = this.factory.getInstance(syncRules);
93
108
  return this.storage!;
94
109
  }
@@ -99,6 +114,7 @@ export class ChangeStreamTestContext {
99
114
  throw new Error(`Next sync rules not available`);
100
115
  }
101
116
 
117
+ this.syncRulesId = syncRules.id;
102
118
  this.storage = this.factory.getInstance(syncRules);
103
119
  return this.storage!;
104
120
  }
@@ -159,9 +175,21 @@ export class ChangeStreamTestContext {
159
175
  return checkpoint;
160
176
  }
161
177
 
178
+ private resolveBucketName(bucket: string) {
179
+ if (!this.versionedBuckets || /^\d+#/.test(bucket)) {
180
+ return bucket;
181
+ }
182
+ if (this.syncRulesId == null) {
183
+ throw new Error('Sync rules not configured - call updateSyncRules() first');
184
+ }
185
+ return `${this.syncRulesId}#${bucket}`;
186
+ }
187
+
162
188
  async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
163
189
  let checkpoint = await this.getCheckpoint(options);
164
- const map = new Map<string, InternalOpId>(Object.entries(buckets));
190
+ const map = new Map<string, InternalOpId>(
191
+ Object.entries(buckets).map(([bucket, opId]) => [this.resolveBucketName(bucket), opId])
192
+ );
165
193
  return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
166
194
  }
167
195
 
@@ -170,8 +198,9 @@ export class ChangeStreamTestContext {
170
198
  if (typeof start == 'string') {
171
199
  start = BigInt(start);
172
200
  }
201
+ const resolvedBucket = this.resolveBucketName(bucket);
173
202
  const checkpoint = await this.getCheckpoint(options);
174
- const map = new Map<string, InternalOpId>([[bucket, start]]);
203
+ const map = new Map<string, InternalOpId>([[resolvedBucket, start]]);
175
204
  let data: OplogEntry[] = [];
176
205
  while (true) {
177
206
  const batch = this.storage!.getBucketDataBatch(checkpoint, map);
@@ -181,20 +210,26 @@ export class ChangeStreamTestContext {
181
210
  if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
182
211
  break;
183
212
  }
184
- map.set(bucket, BigInt(batches[0]!.chunkData.next_after));
213
+ map.set(resolvedBucket, BigInt(batches[0]!.chunkData.next_after));
185
214
  }
186
215
  return data;
187
216
  }
188
217
 
189
- async getChecksums(buckets: string[], options?: { timeout?: number }) {
218
+ async getChecksums(buckets: string[], options?: { timeout?: number }): Promise<utils.ChecksumMap> {
190
219
  let checkpoint = await this.getCheckpoint(options);
191
- return this.storage!.getChecksums(checkpoint, buckets);
220
+ const versionedBuckets = buckets.map((bucket) => this.resolveBucketName(bucket));
221
+ const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets);
222
+
223
+ const unversioned: utils.ChecksumMap = new Map();
224
+ for (let i = 0; i < buckets.length; i++) {
225
+ unversioned.set(buckets[i], checksums.get(versionedBuckets[i])!);
226
+ }
227
+ return unversioned;
192
228
  }
193
229
 
194
230
  async getChecksum(bucket: string, options?: { timeout?: number }) {
195
- let checkpoint = await this.getCheckpoint(options);
196
- const map = await this.storage!.getChecksums(checkpoint, [bucket]);
197
- return map.get(bucket);
231
+ const checksums = await this.getChecksums([bucket], options);
232
+ return checksums.get(bucket);
198
233
  }
199
234
  }
200
235
 
package/test/src/chunked_snapshot.test.ts CHANGED
@@ -1,18 +1,22 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
- import { reduceBucket, TestStorageFactory } from '@powersync/service-core';
2
+ import { reduceBucket } from '@powersync/service-core';
3
3
  import { METRICS_HELPER } from '@powersync/service-core-tests';
4
4
  import { JSONBig } from '@powersync/service-jsonbig';
5
5
  import { SqliteJsonValue } from '@powersync/service-sync-rules';
6
6
  import * as timers from 'timers/promises';
7
7
  import { describe, expect, test } from 'vitest';
8
8
  import { ChangeStreamTestContext } from './change_stream_utils.js';
9
- import { describeWithStorage } from './util.js';
9
+ import { describeWithStorage, StorageVersionTestContext } from './util.js';
10
10
 
11
11
  describe('chunked snapshots', () => {
12
12
  describeWithStorage({ timeout: 120_000 }, defineBatchTests);
13
13
  });
14
14
 
15
- function defineBatchTests(factory: TestStorageFactory) {
15
+ function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext) {
16
+ const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
17
+ return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
18
+ };
19
+
16
20
  // This is not as sensitive to the id type as postgres, but we still test a couple of cases
17
21
  test('chunked snapshot (int32)', async () => {
18
22
  await testChunkedSnapshot({
@@ -93,7 +97,7 @@ function defineBatchTests(factory: TestStorageFactory) {
93
97
  const idToSqlite = options.idToSqlite ?? ((n) => n);
94
98
  const idToString = (id: any) => String(idToSqlite(id));
95
99
 
96
- await using context = await ChangeStreamTestContext.open(factory, {
100
+ await using context = await openContext({
97
101
  // We need to use a smaller chunk size here, so that we can run a query in between chunks
98
102
  streamOptions: { snapshotChunkLength: 100 }
99
103
  });
package/test/src/resume.test.ts CHANGED
@@ -1,19 +1,22 @@
1
1
  import { ChangeStreamInvalidatedError } from '@module/replication/ChangeStream.js';
2
2
  import { MongoManager } from '@module/replication/MongoManager.js';
3
3
  import { normalizeConnectionConfig } from '@module/types/types.js';
4
- import { BucketStorageFactory, TestStorageOptions } from '@powersync/service-core';
5
4
  import { describe, expect, test } from 'vitest';
6
5
  import { ChangeStreamTestContext } from './change_stream_utils.js';
7
6
  import { env } from './env.js';
8
- import { describeWithStorage } from './util.js';
7
+ import { describeWithStorage, StorageVersionTestContext } from './util.js';
9
8
 
10
9
  describe('mongodb resuming replication', () => {
11
10
  describeWithStorage({}, defineResumeTest);
12
11
  });
13
12
 
14
- function defineResumeTest(factoryGenerator: (options?: TestStorageOptions) => Promise<BucketStorageFactory>) {
13
+ function defineResumeTest({ factory: factoryGenerator, storageVersion }: StorageVersionTestContext) {
14
+ const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
15
+ return ChangeStreamTestContext.open(factoryGenerator, { ...options, storageVersion });
16
+ };
17
+
15
18
  test('resuming with a different source database', async () => {
16
- await using context = await ChangeStreamTestContext.open(factoryGenerator);
19
+ await using context = await openContext();
17
20
  const { db } = context;
18
21
 
19
22
  await context.updateSyncRules(/* yaml */
@@ -53,7 +56,7 @@ function defineResumeTest(factoryGenerator: (options?: TestStorageOptions) => Pr
53
56
  const factory = await factoryGenerator({ doNotClear: true });
54
57
 
55
58
  // Create a new context without updating the sync rules
56
- await using context2 = new ChangeStreamTestContext(factory, connectionManager);
59
+ await using context2 = new ChangeStreamTestContext(factory, connectionManager, {}, storageVersion);
57
60
  const activeContent = await factory.getActiveSyncRulesContent();
58
61
  context2.storage = factory.getInstance(activeContent!);
59
62
 
package/test/src/resuming_snapshots.test.ts CHANGED
@@ -8,19 +8,19 @@ import { env } from './env.js';
8
8
  import { describeWithStorage } from './util.js';
9
9
 
10
10
  describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
11
- describeWithStorage({ timeout: 240_000 }, function (factory) {
11
+ describeWithStorage({ timeout: 240_000 }, function ({ factory, storageVersion }) {
12
12
  test('resuming initial replication (1)', async () => {
13
13
  // Stop early - likely to not include deleted row in first replication attempt.
14
- await testResumingReplication(factory, 2000);
14
+ await testResumingReplication(factory, storageVersion, 2000);
15
15
  });
16
16
  test('resuming initial replication (2)', async () => {
17
17
  // Stop late - likely to include deleted row in first replication attempt.
18
- await testResumingReplication(factory, 8000);
18
+ await testResumingReplication(factory, storageVersion, 8000);
19
19
  });
20
20
  });
21
21
  });
22
22
 
23
- async function testResumingReplication(factory: TestStorageFactory, stopAfter: number) {
23
+ async function testResumingReplication(factory: TestStorageFactory, storageVersion: number, stopAfter: number) {
24
24
  // This tests interrupting and then resuming initial replication.
25
25
  // We interrupt replication after test_data1 has fully replicated, and
26
26
  // test_data2 has partially replicated.
@@ -35,7 +35,10 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
35
35
  let startRowCount: number;
36
36
 
37
37
  {
38
- await using context = await ChangeStreamTestContext.open(factory, { streamOptions: { snapshotChunkLength: 1000 } });
38
+ await using context = await ChangeStreamTestContext.open(factory, {
39
+ storageVersion,
40
+ streamOptions: { snapshotChunkLength: 1000 }
41
+ });
39
42
 
40
43
  await context.updateSyncRules(`bucket_definitions:
41
44
  global:
@@ -87,6 +90,7 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
87
90
  // Bypass the usual "clear db on factory open" step.
88
91
  await using context2 = await ChangeStreamTestContext.open(factory, {
89
92
  doNotClear: true,
93
+ storageVersion,
90
94
  streamOptions: { snapshotChunkLength: 1000 }
91
95
  });
92
96