@powersync/service-module-mongodb 0.19.0 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/CHANGELOG.md +39 -0
  2. package/dist/module/MongoModule.js +2 -1
  3. package/dist/module/MongoModule.js.map +1 -1
  4. package/dist/replication/ChangeStream.d.ts +8 -0
  5. package/dist/replication/ChangeStream.js +69 -10
  6. package/dist/replication/ChangeStream.js.map +1 -1
  7. package/dist/replication/ChangeStreamReplicationJob.js +2 -1
  8. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  9. package/dist/replication/MongoSnapshotter.js +5 -0
  10. package/dist/replication/MongoSnapshotter.js.map +1 -1
  11. package/dist/replication/RawChangeStream.js +1 -11
  12. package/dist/replication/RawChangeStream.js.map +1 -1
  13. package/dist/replication/checkpoints/CheckpointImplementation.d.ts +4 -1
  14. package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +4 -1
  15. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +10 -8
  16. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -1
  17. package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +4 -1
  18. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +23 -20
  19. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -1
  20. package/dist/types/types.d.ts +5 -0
  21. package/dist/types/types.js +20 -2
  22. package/dist/types/types.js.map +1 -1
  23. package/package.json +9 -9
  24. package/src/module/MongoModule.ts +2 -1
  25. package/src/replication/ChangeStream.ts +73 -12
  26. package/src/replication/ChangeStreamReplicationJob.ts +2 -1
  27. package/src/replication/MongoSnapshotter.ts +10 -0
  28. package/src/replication/RawChangeStream.ts +1 -15
  29. package/src/replication/checkpoints/CheckpointImplementation.ts +1 -1
  30. package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +11 -9
  31. package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +27 -24
  32. package/src/types/types.ts +27 -2
  33. package/test/src/change_stream_utils.ts +16 -4
  34. package/test/src/config.test.ts +34 -0
  35. package/tsconfig.tsbuildinfo +1 -1
@@ -36,7 +36,10 @@ export declare class SentinelCheckpointImplementation implements CheckpointImple
36
36
  createBatchCheckpoint(): Promise<string>;
37
37
  createFirstBarrier(): Promise<string | null>;
38
38
  keepalive(_batch: storage.BucketStorageBatch, _resumeToken: mongo.ResumeToken): Promise<void>;
39
- lsnFromResumeToken(resumeToken: mongo.ResumeToken): string;
39
+ lsnFromResumeToken(resumeToken: mongo.ResumeToken): {
40
+ lsn: string;
41
+ timestamp: null;
42
+ };
40
43
  createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
41
44
  readonly event: CheckpointEventApi;
42
45
  readonly checkpointClearFilter: mongo.Filter<mongo.Document>;
@@ -93,17 +93,19 @@ export class SentinelCheckpointImplementation {
93
93
  // Pair the bare token with the current coordinate. The coordinate is
94
94
  // frozen between checkpoint events, so a per-batch resume marker only
95
95
  // advances the token; that is all resumption needs (see the design doc).
96
- return new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable;
96
+ return { lsn: new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable, timestamp: null };
97
97
  }
98
98
  async createReplicationHead(callback) {
99
99
  const head = await createSentinelCheckpointLsn(this.context.client, this.context.db);
100
- const result = await callback(head);
101
- // Create another bump to ensure movement after the reported head. This
102
- // covers the race where the head's own event is committed before the
103
- // write checkpoint document is stored.
104
- // Note that this checkpoint should not be associated with a change stream Id.
105
- await createSentinelCheckpointLsn(this.context.client, this.context.db);
106
- return result;
100
+ const { response, shouldAdvance } = await callback(head);
101
+ if (shouldAdvance) {
102
+ // Create another bump to ensure movement after the reported head. This
103
+ // covers the race where the head's own event is committed before the
104
+ // write checkpoint document is stored.
105
+ // Note that this checkpoint should not be associated with a change stream Id.
106
+ await createSentinelCheckpointLsn(this.context.client, this.context.db);
107
+ }
108
+ return response;
107
109
  }
108
110
  event = {
109
111
  observe: (doc) => {
@@ -1 +1 @@
1
- {"version":3,"file":"SentinelCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/SentinelCheckpointImplementation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE1F,OAAO,EAKL,eAAe,EAEhB,MAAM,+BAA+B,CAAC;AAEvC;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,gCAAgC;IAWvB;IAVX,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC;IAE/C;;;;;OAKG;IACK,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC;IAE7C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACvE,CAAC;IAED,YAAY,CAAC,GAAkB;QAC7B,IAAI,CAAC,QAAQ,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACrG,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,QAAQ,MAAM,MAAM,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,sEAAsE;QACtE,+DAA+D;QAC/D,6CAA6C;QAC7C,6EAA6E;QAC7E,4EAA4E;QAC5E,qCAAqC;QACrC,OAAO,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5G,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,uEAAuE;QACvE,2EAA2E;QAC3E,0EAA0E;QAC1E,mBAAmB;QACnB,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAkC,EAAE,YAA+B;QACjF,yEAAyE;QACzE,uEAAuE;QACvE,qEAAqE;QACrE,4EAA4E;QAC5E,kDAAkD;QAClD,EAAE;QACF,0EAA0E;QAC1E,0EAA0E;QAC1E,sEAAsE;QACtE,sEAAsE;QACtE,iFAAiF;QACjF,0EAA0E;QAC1E,yDAAyD;QACzD,EAAE;QACF,yEAAyE;QACzE,yEAAyE;QACzE,qEAAqE;QACrE,sEAAsE;QACtE,kEAAkE;QAClE,2DAA2D;QAC3D,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QACzG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,0FAA0F,CAC3F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,qEAAqE;QACrE,sEAAsE;QACtE,yEAAyE;QACzE,OAAO,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,UAAU,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,IAAI,GAAG,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACrF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpC,uEAAuE;QACvE,qEAAqE;QACrE,uCAAuC;QACvC,8EAA8E;QAC9E,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC;IAChB,CAAC;IAEQ,KAAK,GAAuB;QACnC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,YAAY,IAAI,sBAAsB,EAAE,CAAC;gBAC3C,mEAAmE;gBACnE,qEAAqE;gBACrE,wBAAwB;gBACxB,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,iEAAiE;gBACjE,uEAAuE;gBACvE,wEAAwE;gBACxE,qDAAqD;gBACrD,MAAM,IAAI,4BAA4B,CACpC,0EAA0E,EAC1E,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,aAAa,4CAA4C,CAAC,CACvF,CAAC;YACJ,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;YAEnC,IAAI,IAAyB,CAAC;YAC9B,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,mEAAmE;gBACnE,+DAA+D;gBAC/D,IAAI,GAAG,YAAY,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5D,wEAAwE;gBACxE,IAAI,GAAG,aAAa,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,6CAA6C;gBAC7C,OAAO,SAAS,CAAC;YACnB,CAAC;YAED,4EAA4E;YAC5E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE;YACX,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,IAAI,4BAA4B,CACpC,0EAA0E,EAC1E,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,aAAa,4CAA4C,CAAC,CACvF,CAAC;YACJ,CAAC;YAED,wEAAwE;YACxE,qEAAqE;YACrE,yEAAyE;YACzE,oBAAoB;YACpB,OAAO,IAAI,WAAW,CAAC;gBACrB,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;gBACpC,YAAY,EAAE,GAAG,CAAC,GAAG;aACtB,CAAC,CAAC,UAAU,CAAC;QAChB,CAAC;QAED,eAAe,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;YAC/B,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;gBAClG,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC;QACnG,CAAC;KACF,CAAC;IAEF,wEAAwE;IACxE,4EAA4E;IAC5E,8EAA8E;IAC9E,4EAA4E;IAC5E,0EAA0E;IAC1E,4EAA4E;IAC5E,uEAAuE;IACvE,2EAA2E;IAC3E,sCAAsC;IACtC,EAAE;IACF,uEAAuE;IACvE,yEAAyE;IACzE,yEAAyE;IACzE,2EAA2E;IAC3E,sEAAsE;IACtE,yEAAyE;IACzE,sEAAsE;IACtE,gCAAgC;IACvB,qBAAqB,GAAiC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,sBAAsB,EAAS,EAAE,CAAC;IAE/G,wGAAwG;IAChG,aAAa,CAAC,QAAgB;QACpC,IAAI,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,YAAY,CAAC,OAAuB;QAC1C,IAAI,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,4BAA4B,CACpC,0EAA0E;YAC1E,4EAA4E;YAC5E,IAAI,KAAK,CAAC,iCAAiC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CACzE,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC;IACnB,CAAC;CACF;AAYD,SAAS,uBAAuB,CAAC,GAAkC;IACjE,MAAM,IAAI,GAAG,GAAmC,CAAC;IACjD,IAAI,IAAI,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IACD,MAAM,YAAY,GAAG,cAAc,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,YAAsB,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3G,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC;IACjC,OAAO,MAAM,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"SentinelCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/SentinelCheckpointImplementation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE1F,OAAO,EAKL,eAAe,EAEhB,MAAM,+BAA+B,CAAC;AAEvC;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,gCAAgC;IAWvB;IAVX,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC;IAE/C;;;;;OAKG;IACK,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC;IAE7C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACvE,CAAC;IAED,YAAY,CAAC,GAAkB;QAC7B,IAAI,CAAC,QAAQ,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACrG,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,QAAQ,MAAM,MAAM,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,sEAAsE;QACtE,+DAA+D;QAC/D,6CAA6C;QAC7C,6EAA6E;QAC7E,4EAA4E;QAC5E,qCAAqC;QACrC,OAAO,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5G,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,uEAAuE;QACvE,2EAA2E;QAC3E,0EAA0E;QAC1E,mBAAmB;QACnB,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAkC,EAAE,YAA+B;QACjF,yEAAyE;QACzE,uEAAuE;QACvE,qEAAqE;QACrE,4EAA4E;QAC5E,kDAAkD;QAClD,EAAE;QACF,0EAA0E;QAC1E,0EAA0E;QAC1E,sEAAsE;QACtE,sEAAsE;QACtE,iFAAiF;QACjF,0EAA0E;QAC1E,yDAAyD;QACzD,EAAE;QACF,yEAAyE;QACzE,yEAAyE;QACzE,qEAAqE;QACrE,sEAAsE;QACtE,kEAAkE;QAClE,2DAA2D;QAC3D,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QACzG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,0FAA0F,CAC3F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,qEAAqE;QACrE,sEAAsE;QACtE,yEAAyE;QACzE,OAAO,EAAE,GAAG,EAAE,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACtH,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,IAAI,GAAG,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACrF,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;QACzD,IAAI,aAAa,EAAE,CAAC;YAClB,uEAAuE;YACvE,qEAAqE;YACrE,uCAAuC;YACvC,8EAA8E;YAC9E,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEQ,KAAK,GAAuB;QACnC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,YAAY,IAAI,sBAAsB,EAAE,CAAC;gBAC3C,mEAAmE;gBACnE,qEAAqE;gBACrE,wBAAwB;gBACxB,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,iEAAiE;gBACjE,uEAAuE;gBACvE,wEAAwE;gBACxE,qDAAqD;gBACrD,MAAM,IAAI,4BAA4B,CACpC,0EAA0E,EAC1E,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,aAAa,4CAA4C,CAAC,CACvF,CAAC;YACJ,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;YAEnC,IAAI,IAAyB,CAAC;YAC9B,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,mEAAmE;gBACnE,+DAA+D;gBAC/D,IAAI,GAAG,YAAY,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5D,wEAAwE;gBACxE,IAAI,GAAG,aAAa,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,6CAA6C;gBAC7C,OAAO,SAAS,CAAC;YACnB,CAAC;YAED,4EAA4E;YAC5E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE;YACX,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,IAAI,4BAA4B,CACpC,0EAA0E,EAC1E,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,aAAa,4CAA4C,CAAC,CACvF,CAAC;YACJ,CAAC;YAED,wEAAwE;YACxE,qEAAqE;YACrE,yEAAyE;YACzE,oBAAoB;YACpB,OAAO,IAAI,WAAW,CAAC;gBACrB,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;gBACpC,YAAY,EAAE,GAAG,CAAC,GAAG;aACtB,CAAC,CAAC,UAAU,CAAC;QAChB,CAAC;QAED,eAAe,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;YAC/B,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;gBAClG,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC;QACnG,CAAC;KACF,CAAC;IAEF,wEAAwE;IACxE,4EAA4E;IAC5E,8EAA8E;IAC9E,4EAA4E;IAC5E,0EAA0E;IAC1E,4EAA4E;IAC5E,uEAAuE;IACvE,2EAA2E;IAC3E,sCAAsC;IACtC,EAAE;IACF,uEAAuE;IACvE,yEAAyE;IACzE,yEAAyE;IACzE,2EAA2E;IAC3E,sEAAsE;IACtE,yEAAyE;IACzE,sEAAsE;IACtE,gCAAgC;IACvB,qBAAqB,GAAiC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,sBAAsB,EAAS,EAAE,CAAC;IAE/G,wGAAwG;IAChG,aAAa,CAAC,QAAgB;QACpC,IAAI,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,YAAY,CAAC,OAAuB;QAC1C,IAAI,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,4BAA4B,CACpC,0EAA0E;YAC1E,4EAA4E;YAC5E,IAAI,KAAK,CAAC,iCAAiC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CACzE,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC;IACnB,CAAC;CACF;AAYD,SAAS,uBAAuB,CAAC,GAAkC;IACjE,MAAM,IAAI,GAAG,GAAmC,CAAC;IACjD,IAAI,IAAI,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IACD,MAAM,YAAY,GAAG,cAAc,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,YAAsB,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3G,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC;IACjC,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -17,7 +17,10 @@ export declare class TimestampCheckpointImplementation implements CheckpointImpl
17
17
  createBatchCheckpoint(): Promise<string>;
18
18
  createFirstBarrier(): Promise<string | null>;
19
19
  keepalive(batch: storage.BucketStorageBatch, resumeToken: mongo.ResumeToken): Promise<void>;
20
- lsnFromResumeToken(resumeToken: mongo.ResumeToken): string;
20
+ lsnFromResumeToken(resumeToken: mongo.ResumeToken): {
21
+ lsn: string;
22
+ timestamp: Date;
23
+ };
21
24
  createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
22
25
  readonly event: CheckpointEventApi;
23
26
  readonly checkpointClearFilter: mongo.Filter<mongo.Document>;
@@ -49,7 +49,8 @@ export class TimestampCheckpointImplementation {
49
49
  }
50
50
  lsnFromResumeToken(resumeToken) {
51
51
  // The timestamp is embedded in the resume token.
52
- return MongoLSN.fromResumeToken(resumeToken).comparable;
52
+ const lsn = MongoLSN.fromResumeToken(resumeToken);
53
+ return { lsn: lsn.comparable, timestamp: timestampToDate(lsn.timestamp) };
53
54
  }
54
55
  async createReplicationHead(callback) {
55
56
  const session = this.context.client.startSession();
@@ -59,26 +60,28 @@ export class TimestampCheckpointImplementation {
59
60
  if (head == null) {
60
61
  throw new ServiceAssertionError(`clusterTime not available for write checkpoint`);
61
62
  }
62
- const r = await callback(new MongoLSN({ timestamp: head }).comparable);
63
- // Trigger a change on the changestream, so that the write checkpoint
64
- // is processed without waiting for other writes.
65
- await this.context.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate({
66
- _id: STANDALONE_CHECKPOINT_ID
67
- }, {
68
- $inc: { i: 1 }
69
- }, {
70
- upsert: true,
71
- returnDocument: 'after',
72
- session
73
- });
74
- const time = session.operationTime;
75
- if (time == null) {
76
- throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
63
+ const { response, shouldAdvance } = await callback(new MongoLSN({ timestamp: head }).comparable);
64
+ if (shouldAdvance) {
65
+ // Trigger a change on the changestream, so that the write checkpoint
66
+ // is processed without waiting for other writes.
67
+ await this.context.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate({
68
+ _id: STANDALONE_CHECKPOINT_ID
69
+ }, {
70
+ $inc: { i: 1 }
71
+ }, {
72
+ upsert: true,
73
+ returnDocument: 'after',
74
+ session
75
+ });
76
+ const time = session.operationTime;
77
+ if (time == null) {
78
+ throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
79
+ }
80
+ else if (time.lt(head)) {
81
+ throw new ServiceAssertionError(`operationTime must be > clusterTime`);
82
+ }
77
83
  }
78
- else if (time.lt(head)) {
79
- throw new ServiceAssertionError(`operationTime must be > clusterTime`);
80
- }
81
- return r;
84
+ return response;
82
85
  }
83
86
  finally {
84
87
  await session.endSession();
@@ -1 +1 @@
1
- {"version":3,"file":"TimestampCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/TimestampCheckpointImplementation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACzG,OAAO,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EAIL,eAAe,EACf,iBAAiB,EAElB,MAAM,+BAA+B,CAAC;AAEvC;;;;GAIG;AACH,MAAM,OAAO,iCAAiC;IAGxB;IAFX,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;IAE5C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;IACnF,CAAC;IAED,YAAY,CAAC,IAAmB;QAC9B,wEAAwE;IAC1E,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,6EAA6E;QAC7E,0BAA0B;QAC1B,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACtG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,uBAAuB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,MAAM,iBAAiB,eAAe,GAAG,CACjG,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,wBAAwB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,8EAA8E;QAC9E,OAAO,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAiC,EAAE,WAA8B;QAC/E,oEAAoE;QACpE,kEAAkE;QAClE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAC7E,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,mGAAmG;QACnG,mDAAmD;QACnD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,iDAAiD,eAAe,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAC5F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,iDAAiD;QACjD,OAAO,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACzD,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,CAAC;YAC9C,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;YACpF,CAAC;YAED,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YAEvE,qEAAqE;YACrE,iDAAiD;YACjD,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC,gBAAgB,CACvE;gBACE,GAAG,EAAE,wBAA+B;aACrC,EACD;gBACE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE;aACf,EACD;gBACE,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,OAAO;gBACvB,OAAO;aACR,CACF,CAAC;YACF,MAAM,IAAI,GAAG,OAAO,CAAC,aAAc,CAAC;YACpC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,qBAAqB,CAAC,kDAAkD,CAAC,CAAC;YACtF,CAAC;iBAAM,IAAI,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;YACzE,CAAC;YAED,OAAO,CAAC,CAAC;QACX,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEQ,KAAK,GAAuB;QACnC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,IAAI,sBAAsB,EAAE,CAAC;gBACnE,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,sFAAsF;YACtF,IAAI,YAAY,IAAI,wBAAwB,EAAE,CAAC;gBAC7C,OAAO,YAAY,CAAC;YACtB,CAAC;YACD,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1F,CAAC;QAED,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE;YACX,OAAO,IAAI,QAAQ,CAAC;gBAClB,SAAS,EAAE,iBAAiB,CAAC,GAAG,CAAC;gBACjC,YAAY,EAAE,GAAG,CAAC,GAAG;aACtB,CAAC,CAAC,UAAU,CAAC;QAChB,CAAC;QAED,eAAe,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;YAC/B,8DAA8D;YAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;QACvC,CAAC;KACF,CAAC;IAEF,gFAAgF;IACvE,qBAAqB,GAAiC,EAAE,CAAC;CACnE"}
1
+ {"version":3,"file":"TimestampCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/TimestampCheckpointImplementation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACzG,OAAO,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EAIL,eAAe,EACf,iBAAiB,EAElB,MAAM,+BAA+B,CAAC;AAEvC;;;;GAIG;AACH,MAAM,OAAO,iCAAiC;IAGxB;IAFX,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;IAE5C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;IACnF,CAAC;IAED,YAAY,CAAC,IAAmB;QAC9B,wEAAwE;IAC1E,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,6EAA6E;QAC7E,0BAA0B;QAC1B,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACtG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,uBAAuB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,MAAM,iBAAiB,eAAe,GAAG,CACjG,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,wBAAwB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,8EAA8E;QAC9E,OAAO,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAiC,EAAE,WAA8B;QAC/E,oEAAoE;QACpE,kEAAkE;QAClE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAC7E,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,mGAAmG;QACnG,mDAAmD;QACnD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,iDAAiD,eAAe,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAC5F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,iDAAiD;QACjD,MAAM,GAAG,GAAG,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAClD,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACzD,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,CAAC;YAC9C,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;YACpF,CAAC;YAED,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YAEjG,IAAI,aAAa,EAAE,CAAC;gBAClB,qEAAqE;gBACrE,iDAAiD;gBACjD,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC,gBAAgB,CACvE;oBACE,GAAG,EAAE,wBAA+B;iBACrC,EACD;oBACE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE;iBACf,EACD;oBACE,MAAM,EAAE,IAAI;oBACZ,cAAc,EAAE,OAAO;oBACvB,OAAO;iBACR,CACF,CAAC;gBACF,MAAM,IAAI,GAAG,OAAO,CAAC,aAAc,CAAC;gBACpC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;oBACjB,MAAM,IAAI,qBAAqB,CAAC,kDAAkD,CAAC,CAAC;gBACtF,CAAC;qBAAM,IAAI,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzB,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEQ,KAAK,GAAuB;QACnC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,IAAI,sBAAsB,EAAE,CAAC;gBACnE,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,sFAAsF;YACtF,IAAI,YAAY,IAAI,wBAAwB,EAAE,CAAC;gBAC7C,OAAO,YAAY,CAAC;YACtB,CAAC;YACD,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1F,CAAC;QAED,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE;YACX,OAAO,IAAI,QAAQ,CAAC;gBAClB,SAAS,EAAE,iBAAiB,CAAC,GAAG,CAAC;gBACjC,YAAY,EAAE,GAAG,CAAC,GAAG;aACtB,CAAC,CAAC,UAAU,CAAC;QAChB,CAAC;QAED,eAAe,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;YAC/B,8DAA8D;YAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;QACvC,CAAC;KACF,CAAC;IAEF,gFAAgF;IACvE,qBAAqB,GAAiC,EAAE,CAAC;CACnE"}
@@ -43,6 +43,7 @@ export interface NormalizedMongoConnectionConfig {
43
43
  password?: string;
44
44
  lookup?: LookupFunction;
45
45
  postImages: PostImagesOption;
46
+ heartbeat_interval_seconds: number;
46
47
  connectionParams: MongoConnectionParams;
47
48
  }
48
49
  export declare const MongoConnectionConfig: t.Intersection<t.Codec<{
@@ -71,6 +72,10 @@ export declare const MongoConnectionConfig: t.Intersection<t.Codec<{
71
72
  reject_ip_ranges?: string[] | undefined;
72
73
  }, string, t.CodecProps>, t.ObjectCodec<{
73
74
  post_images: t.OptionalCodec<t.Codec<"off" | "auto_configure" | "read_only", "off" | "auto_configure" | "read_only", string, t.CodecProps>>;
75
+ /**
76
+ * Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
77
+ */
78
+ heartbeat_interval_seconds: t.OptionalCodec<t.Codec<number | null, number | null, string, t.CodecProps>>;
74
79
  }>>;
75
80
  /**
76
81
  * Config input specified when starting services
@@ -1,4 +1,5 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb/types';
2
+ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
2
3
  import * as service_types from '@powersync/service-types';
3
4
  import * as t from 'ts-codec';
4
5
  export var PostImagesOption;
@@ -35,9 +36,16 @@ export var PostImagesOption;
35
36
  */
36
37
  PostImagesOption["READ_ONLY"] = "read_only";
37
38
  })(PostImagesOption || (PostImagesOption = {}));
39
+ const DEFAULT_PING_INTERVAL_SECONDS = 60;
40
+ const MIN_PING_INTERVAL_SECONDS = 5;
41
+ const MAX_PING_INTERVAL_SECONDS = 60;
38
42
  export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_mongo.BaseMongoConfig).and(t.object({
39
43
  // Replication specific settings
40
- post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional()
44
+ post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional(),
45
+ /**
46
+ * Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
47
+ */
48
+ heartbeat_interval_seconds: t.number.or(t.Null).optional()
41
49
  }));
42
50
  /**
43
51
  * Validate and normalize connection options.
@@ -50,7 +58,17 @@ export function normalizeConnectionConfig(options) {
50
58
  ...base,
51
59
  id: options.id ?? 'default',
52
60
  tag: options.tag ?? 'default',
53
- postImages: options.post_images ?? PostImagesOption.OFF
61
+ postImages: options.post_images ?? PostImagesOption.OFF,
62
+ heartbeat_interval_seconds: normalizeHeartbeatInterval(options.heartbeat_interval_seconds)
54
63
  };
55
64
  }
65
+ function normalizeHeartbeatInterval(value) {
66
+ if (value == null) {
67
+ return DEFAULT_PING_INTERVAL_SECONDS;
68
+ }
69
+ if (!Number.isFinite(value) || value < MIN_PING_INTERVAL_SECONDS || value > MAX_PING_INTERVAL_SECONDS) {
70
+ throw new ServiceError(ErrorCode.PSYNC_S1109, `MongoDB connection: heartbeat_interval_seconds must be between ${MIN_PING_INTERVAL_SECONDS} and ${MAX_PING_INTERVAL_SECONDS} seconds; null or omitted defaults to ${DEFAULT_PING_INTERVAL_SECONDS}`);
71
+ }
72
+ return value;
73
+ }
56
74
  //# sourceMappingURL=types.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,sCAAsC,CAAC;AAClE,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAN,IAAY,gBAkCX;AAlCD,WAAY,gBAAgB;IAC1B;;;;;;;OAOG;IACH,+BAAW,CAAA;IAEX;;;;;;;OAOG;IACH,qDAAiC,CAAA;IAEjC;;;;;;;;;;;OAWG;IACH,2CAAuB,CAAA;AACzB,CAAC,EAlCW,gBAAgB,KAAhB,gBAAgB,QAkC3B;AAmBD,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,GAAG,CAC/G,CAAC,CAAC,MAAM,CAAC;IACP,gCAAgC;IAChC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;CACpG,CAAC,CACH,CAAC;AAaF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAqC;IAC7E,MAAM,IAAI,GAAG,SAAS,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAErD,OAAO;QACL,GAAG,IAAI;QACP,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAC7B,UAAU,EAAG,OAAO,CAAC,WAA4C,IAAI,gBAAgB,CAAC,GAAG;KAC1F,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,sCAAsC,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAN,IAAY,gBAkCX;AAlCD,WAAY,gBAAgB;IAC1B;;;;;;;OAOG;IACH,+BAAW,CAAA;IAEX;;;;;;;OAOG;IACH,qDAAiC,CAAA;IAEjC;;;;;;;;;;;OAWG;IACH,2CAAuB,CAAA;AACzB,CAAC,EAlCW,gBAAgB,KAAhB,gBAAgB,QAkC3B;AAED,MAAM,6BAA6B,GAAG,EAAE,CAAC;AACzC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,yBAAyB,GAAG,EAAE,CAAC;AAqBrC,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,GAAG,CAC/G,CAAC,CAAC,MAAM,CAAC;IACP,gCAAgC;IAChC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnG;;OAEG;IACH,0BAA0B,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC3D,CAAC,CACH,CAAC;AAaF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAqC;IAC7E,MAAM,IAAI,GAAG,SAAS,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAErD,OAAO;QACL,GAAG,IAAI;QACP,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAC7B,UAAU,EAAG,OAAO,CAAC,WAA4C,IAAI,gBAAgB,CAAC,GAAG;QACzF,0BAA0B,EAAE,0BAA0B,CAAC,OAAO,CAAC,0BAA0B,CAAC;KAC3F,CAAC;AACJ,CAAC;AAED,SAAS,0BAA0B,CAAC,KAAgC;IAClE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAClB,OAAO,6BAA6B,CAAC;IACvC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,yBAAyB,IAAI,KAAK,GAAG,yBAAyB,EAAE,CAAC;QACtG,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,kEAAkE,yBAAyB,QAAQ,yBAAyB,yCAAyC,6BAA6B,EAAE,CACrM,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@powersync/service-module-mongodb",
3
3
  "repository": "https://github.com/powersync-ja/powersync-service",
4
- "version": "0.19.0",
4
+ "version": "0.20.1",
5
5
  "license": "FSL-1.1-ALv2",
6
6
  "type": "module",
7
7
  "publishConfig": {
@@ -15,17 +15,17 @@
15
15
  "bson": "^6.10.4",
16
16
  "ts-codec": "^1.3.0",
17
17
  "uuid": "^14.0.0",
18
- "@powersync/lib-services-framework": "0.9.8",
19
- "@powersync/lib-service-mongodb": "0.6.29",
20
- "@powersync/service-core": "1.23.3",
18
+ "@powersync/lib-service-mongodb": "0.6.30",
19
+ "@powersync/lib-services-framework": "0.10.0",
20
+ "@powersync/service-core": "1.25.0",
21
21
  "@powersync/service-jsonbig": "0.17.13",
22
- "@powersync/service-sync-rules": "0.39.0",
23
- "@powersync/service-types": "0.16.1"
22
+ "@powersync/service-sync-rules": "0.40.0",
23
+ "@powersync/service-types": "0.17.0"
24
24
  },
25
25
  "devDependencies": {
26
- "@powersync/service-core-tests": "0.17.2",
27
- "@powersync/service-module-mongodb-storage": "0.18.3",
28
- "@powersync/service-module-postgres-storage": "0.16.3"
26
+ "@powersync/service-core-tests": "0.18.0",
27
+ "@powersync/service-module-mongodb-storage": "0.20.0",
28
+ "@powersync/service-module-postgres-storage": "0.18.0"
29
29
  },
30
30
  "scripts": {
31
31
  "build": "tsc -b",
@@ -39,7 +39,8 @@ export class MongoModule extends replication.ReplicationModule<types.MongoConnec
39
39
  storageEngine: context.storageEngine,
40
40
  metricsEngine: context.metricsEngine,
41
41
  connectionFactory: connectionFactory,
42
- rateLimiter: new MongoErrorRateLimiter()
42
+ rateLimiter: new MongoErrorRateLimiter(),
43
+ heartbeatIntervalSeconds: normalisedConfig.heartbeat_interval_seconds
43
44
  });
44
45
  }
45
46
 
@@ -123,15 +123,27 @@ export class ChangeStream {
123
123
 
124
124
  private isDocumentDb = false;
125
125
  private _checkpointImplementation: CheckpointImplementation | null = null;
126
+ /**
127
+ * Last persisted resume timestamp as Date.now(). This represents the equivalent timestamp from the
128
+ * source database, rather than the time when we persisted it.
129
+ */
130
+ private lastPersistedResumeTimestamp = 0;
131
+ private lastBatchCheckpoint = 0;
126
132
 
127
133
  constructor(options: ChangeStreamOptions) {
128
134
  this.storage = options.storage;
129
135
  this.metrics = options.metrics;
130
136
  this.group_id = options.storage.replicationStreamId;
131
137
  this.connections = options.connections;
132
- this.maxAwaitTimeMS = options.maxAwaitTimeMS ?? 10_000;
133
138
  this.snapshotChunkLength = options.snapshotChunkLength ?? 6_000;
134
139
  this.keepaliveIntervalMs = options.keepaliveIntervalMs ?? 60_000;
140
+ if (options.maxAwaitTimeMS) {
141
+ this.maxAwaitTimeMS = options.maxAwaitTimeMS;
142
+ } else {
143
+ // Default to 10s, unless keepaliveIntervalMs is shorter. In that case, use keepaliveIntervalMs,
144
+ // but add a bit of a margin, so that our keepalive logic is triggered on each empty change stream batch.
145
+ this.maxAwaitTimeMS = Math.min(10_000, this.keepaliveIntervalMs + 100);
146
+ }
135
147
  this.storageHooks = options.storageHooks;
136
148
  this.client = this.connections.client;
137
149
  this.defaultDb = this.connections.db;
@@ -598,6 +610,12 @@ export class ChangeStream {
598
610
  return this.sourceRowConverter.rawToSqliteRow(row);
599
611
  }
600
612
 
613
+ private async createBatchCheckpoint() {
614
+ const checkpoint = await this.checkpointImplementation.createBatchCheckpoint();
615
+ this.lastBatchCheckpoint = Date.now();
616
+ return checkpoint;
617
+ }
618
+
601
619
  async streamChangesInternal() {
602
620
  await this.ensureDetected();
603
621
  const transactionsReplicatedMetric = this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED);
@@ -643,7 +661,7 @@ export class ChangeStream {
643
661
  // Always start with a checkpoint.
644
662
  // This helps us to clear errors when restarting, even if there is
645
663
  // no data to replicate.
646
- let waitForCheckpointLsn: string | null = await this.checkpointImplementation.createBatchCheckpoint();
664
+ let waitForCheckpointLsn: string | null = await this.createBatchCheckpoint();
647
665
 
648
666
  let splitDocument: ProjectedChangeStreamDocument | null = null;
649
667
 
@@ -663,11 +681,15 @@ export class ChangeStream {
663
681
  }
664
682
  this.touch();
665
683
  if (events.length == 0) {
666
- // No changes in this batch, but we still want to keep the connection alive.
684
+ // No changes in this batch, but we still want to persist progress.
667
685
  // We do this by persisting a keepalive checkpoint.
668
686
  // If we don't update it on empty events, we do keep consistency, but resuming the stream
669
687
  // with old tokens may cause connection timeouts.
670
- if (waitForCheckpointLsn == null && performance.now() - lastEmptyResume > this.keepaliveIntervalMs) {
688
+ const hadRecentKeepalive = performance.now() - lastEmptyResume < this.keepaliveIntervalMs;
689
+ if (waitForCheckpointLsn == null && !hadRecentKeepalive) {
690
+ // Case 1: We have no changes, and we are not waiting for a checkpoint to be created,
691
+ // and we have not recently persisted a keepalive. Persist one now, and call setResumeLsn() below.
692
+ // This is the normal case for an idle stream.
671
693
  // The implementation persists a keepalive (timestamp) or bumps the
672
694
  // sentinel so a later event commits (sentinel). Logging is handled
673
695
  // inside the implementation.
@@ -675,12 +697,15 @@ export class ChangeStream {
675
697
  this.touch();
676
698
  lastEmptyResume = performance.now();
677
699
  this.replicationLag.markStarted();
678
- }
679
-
680
- // If we have no changes, we can just persist the keepalive.
681
- // This is throttled to once per interval.
682
- if (performance.now() - lastEmptyResume < this.keepaliveIntervalMs) {
700
+ } else if (hadRecentKeepalive) {
701
+ // Case 2: We have no changes, and may or may not be waiting for a checkpoint to be created.
702
+ // We have recently persisted a keepalive.
703
+ // Continue waiting.
683
704
  continue;
705
+ } else {
706
+ // Case 3: Waiting for a checkpoint; have not had a recent keepalive.
707
+ // We cannot call checkpointImplementation.keepalive() here, but we do call
708
+ // setResumeLsn() below.
684
709
  }
685
710
  }
686
711
 
@@ -802,7 +827,7 @@ export class ChangeStream {
802
827
  if (hasBufferedChanges && waitForCheckpointLsn == null) {
803
828
  // Buffered changes - create a new batch checkpoint to rate limit commits
804
829
  using _ = tracer.span('source_checkpoint');
805
- waitForCheckpointLsn = await this.checkpointImplementation.createBatchCheckpoint();
830
+ waitForCheckpointLsn = await this.createBatchCheckpoint();
806
831
  continue;
807
832
  } else if (waitForCheckpointLsn != null) {
808
833
  // Skip this checkpoint - wait for the batch checkpoint.
@@ -836,7 +861,7 @@ export class ChangeStream {
836
861
  ) {
837
862
  if (waitForCheckpointLsn == null) {
838
863
  using _ = tracer.span('source_checkpoint');
839
- waitForCheckpointLsn = await this.checkpointImplementation.createBatchCheckpoint();
864
+ waitForCheckpointLsn = await this.createBatchCheckpoint();
840
865
  }
841
866
 
842
867
  const rel = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
@@ -909,10 +934,18 @@ export class ChangeStream {
909
934
  // Batches are generally large (64MB or 6000 events, whichever comes first),
910
935
  // so this is a good natural point to flush and mark progress.
911
936
  // We avoid this when splitDocument is set, since we cannot resume in the middle of a split event.
912
- const lsn = this.checkpointImplementation.lsnFromResumeToken(resumeToken);
937
+ const { lsn, timestamp } = this.checkpointImplementation.lsnFromResumeToken(resumeToken);
913
938
  await batch.flush({ oldestUncommittedChange: this.replicationLag.oldestUncommittedChange });
914
939
  // TODO: We should consider making this standard behavior of flush().
915
940
  await batch.setResumeLsn(lsn);
941
+
942
+ if (timestamp != null) {
943
+ // Note that this timestamp provided by MongoDB is not exact - it can be around 10s behind.
944
+ this.lastPersistedResumeTimestamp = timestamp.getTime();
945
+ } else {
946
+ // DocumentDB: No timestamp associated with the resumeToken. Just use the current time.
947
+ this.lastPersistedResumeTimestamp = Date.now();
948
+ }
916
949
  }
917
950
 
918
951
  batchSpan.end();
@@ -940,6 +973,34 @@ export class ChangeStream {
940
973
  return this.replicationLag.getLagMillis();
941
974
  }
942
975
 
976
+ async keepAlive() {
977
+ // This is called on an interval of keepaliveIntervalMs.
978
+
979
+ // This writes to _powersync_checkpoints.
980
+ // Main use case: When there are massive bulk writes to another database or collection, the change stream may
981
+ // start timing out due to reading through too much data in one batch. This breaks up those bulk writes,
982
+ // allowing the change stream to make progress.
983
+
984
+ // When there is low replication traffic, regular calls to setResumeLsn() with recent resume tokens make this unnecessary.
985
+ // We track that using lastPersistedResumeTimestamp.
986
+ // We add a bit of a margin here, since setResumeLsn is not called on an exact interval.
987
+ const staleResumeLsn = Date.now() - this.lastPersistedResumeTimestamp > this.keepaliveIntervalMs * 1.1;
988
+
989
+ // When there is replication lag, that resumeToken can get too outdated. In that case, we periodically create a
990
+ // new back checkpoint. This interval is controlled by the frequency of the keepAlive() call,
991
+ // while also skipping if there was another call to createBatchCheckpoint().
992
+ // We use a factor of 0.9 here, to make sure this is called on every keepAlive() interval, unless there
993
+ // was another call to createBatchCheckpoint().
994
+ const staleBatchCheckpoint = Date.now() - this.lastBatchCheckpoint > this.keepaliveIntervalMs * 0.9;
995
+
996
+ // We don't use oldestUncommittedChange here, since that may be unset in some edge cases where we do need
997
+ // to persist new checkpoints.
998
+ if (staleResumeLsn && staleBatchCheckpoint) {
999
+ await this.ensureDetected();
1000
+ await this.createBatchCheckpoint();
1001
+ }
1002
+ }
1003
+
943
1004
  private lastTouchedAt = performance.now();
944
1005
 
945
1006
  private touch() {
@@ -24,7 +24,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
24
24
  }
25
25
 
26
26
  async keepAlive() {
27
- // Nothing needed here
27
+ await this.lastStream?.keepAlive();
28
28
  }
29
29
 
30
30
  async replicate() {
@@ -71,6 +71,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
71
71
  storage: this.options.storage,
72
72
  metrics: this.options.metrics,
73
73
  connections: connectionManager,
74
+ keepaliveIntervalMs: connectionManager.options.heartbeat_interval_seconds * 1_000,
74
75
  logger: this.logger
75
76
  });
76
77
  this.lastStream = stream;
@@ -99,6 +99,9 @@ export class MongoSnapshotter {
99
99
  this.abortSignal.addEventListener('abort', () => {
100
100
  this.nextItemQueued?.resolve();
101
101
  });
102
+
103
+ // Errors here should not result in uncaught rejection - calling waitForInitialSnapshot is optional.
104
+ void this.initialSnapshotDone.promise.catch(() => {});
102
105
  }
103
106
 
104
107
  private get usePostImages() {
@@ -418,6 +421,13 @@ export class MongoSnapshotter {
418
421
  return [];
419
422
  }
420
423
 
424
+ if (tablePattern.isSchemaWildcard) {
425
+ throw new ServiceError(
426
+ ErrorCode.PSYNC_R2201,
427
+ 'Schema wildcards ("%") in table patterns are not supported for MongoDB connections.'
428
+ );
429
+ }
430
+
421
431
  const nameFilter = tablePattern.isWildcard
422
432
  ? new RegExp('^' + escapeRegExp(tablePattern.tablePrefix))
423
433
  : tablePattern.name;
@@ -1,4 +1,4 @@
1
- import { isMongoNetworkTimeoutError, isMongoServerError, mongo } from '@powersync/lib-service-mongodb';
1
+ import { isMaxTimeMSExpiredError, isMongoServerError, isTimeoutError, mongo } from '@powersync/lib-service-mongodb';
2
2
  import {
3
3
  DatabaseConnectionError,
4
4
  ErrorCode,
@@ -426,20 +426,6 @@ export function parseChangeDocument(buffer: Buffer): ProjectedChangeStreamDocume
426
426
  return doc as any;
427
427
  }
428
428
 
429
- function isMaxTimeMSExpiredError(e: unknown) {
430
- return (
431
- isMongoServerError(e) &&
432
- (e.codeName == 'MaxTimeMSExpired' ||
433
- // MongoDB documents code 50 as MaxTimeMSExpired. Azure DocumentDB reports
434
- // this code with codeName ExceededTimeLimit.
435
- e.code == 50)
436
- );
437
- }
438
-
439
- function isTimeoutError(e: unknown) {
440
- return isMongoNetworkTimeoutError(e) || isMaxTimeMSExpiredError(e);
441
- }
442
-
443
429
  function isResumableChangeStreamError(e: unknown) {
444
430
  // See: https://github.com/mongodb/specifications/blob/master/source/change-streams/change-streams.md#resumable-error
445
431
  if (!isMongoServerError(e)) {
@@ -128,7 +128,7 @@ export interface CheckpointImplementation {
128
128
  * The timestamp implementation parses the timestamp embedded in the token.
129
129
  * The sentinel implementation pairs the token with the current coordinate.
130
130
  */
131
- lsnFromResumeToken(resumeToken: mongo.ResumeToken): string;
131
+ lsnFromResumeToken(resumeToken: mongo.ResumeToken): { lsn: string; timestamp: Date | null };
132
132
 
133
133
  /**
134
134
  * Source-side replication head for write checkpoints. The LSN passed to the
@@ -108,22 +108,24 @@ export class SentinelCheckpointImplementation implements CheckpointImplementatio
108
108
  );
109
109
  }
110
110
 
111
- lsnFromResumeToken(resumeToken: mongo.ResumeToken): string {
111
+ lsnFromResumeToken(resumeToken: mongo.ResumeToken) {
112
112
  // Pair the bare token with the current coordinate. The coordinate is
113
113
  // frozen between checkpoint events, so a per-batch resume marker only
114
114
  // advances the token; that is all resumption needs (see the design doc).
115
- return new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable;
115
+ return { lsn: new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable, timestamp: null };
116
116
  }
117
117
 
118
118
  async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
119
119
  const head = await createSentinelCheckpointLsn(this.context.client, this.context.db);
120
- const result = await callback(head);
121
- // Create another bump to ensure movement after the reported head. This
122
- // covers the race where the head's own event is committed before the
123
- // write checkpoint document is stored.
124
- // Note that this checkpoint should not be associated with a change stream Id.
125
- await createSentinelCheckpointLsn(this.context.client, this.context.db);
126
- return result;
120
+ const { response, shouldAdvance } = await callback(head);
121
+ if (shouldAdvance) {
122
+ // Create another bump to ensure movement after the reported head. This
123
+ // covers the race where the head's own event is committed before the
124
+ // write checkpoint document is stored.
125
+ // Note that this checkpoint should not be associated with a change stream Id.
126
+ await createSentinelCheckpointLsn(this.context.client, this.context.db);
127
+ }
128
+ return response;
127
129
  }
128
130
 
129
131
  readonly event: CheckpointEventApi = {
@@ -67,9 +67,10 @@ export class TimestampCheckpointImplementation implements CheckpointImplementati
67
67
  );
68
68
  }
69
69
 
70
- lsnFromResumeToken(resumeToken: mongo.ResumeToken): string {
70
+ lsnFromResumeToken(resumeToken: mongo.ResumeToken) {
71
71
  // The timestamp is embedded in the resume token.
72
- return MongoLSN.fromResumeToken(resumeToken).comparable;
72
+ const lsn = MongoLSN.fromResumeToken(resumeToken);
73
+ return { lsn: lsn.comparable, timestamp: timestampToDate(lsn.timestamp) };
73
74
  }
74
75
 
75
76
  async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
@@ -81,31 +82,33 @@ export class TimestampCheckpointImplementation implements CheckpointImplementati
81
82
  throw new ServiceAssertionError(`clusterTime not available for write checkpoint`);
82
83
  }
83
84
 
84
- const r = await callback(new MongoLSN({ timestamp: head }).comparable);
85
-
86
- // Trigger a change on the changestream, so that the write checkpoint
87
- // is processed without waiting for other writes.
88
- await this.context.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
89
- {
90
- _id: STANDALONE_CHECKPOINT_ID as any
91
- },
92
- {
93
- $inc: { i: 1 }
94
- },
95
- {
96
- upsert: true,
97
- returnDocument: 'after',
98
- session
85
+ const { response, shouldAdvance } = await callback(new MongoLSN({ timestamp: head }).comparable);
86
+
87
+ if (shouldAdvance) {
88
+ // Trigger a change on the changestream, so that the write checkpoint
89
+ // is processed without waiting for other writes.
90
+ await this.context.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
91
+ {
92
+ _id: STANDALONE_CHECKPOINT_ID as any
93
+ },
94
+ {
95
+ $inc: { i: 1 }
96
+ },
97
+ {
98
+ upsert: true,
99
+ returnDocument: 'after',
100
+ session
101
+ }
102
+ );
103
+ const time = session.operationTime!;
104
+ if (time == null) {
105
+ throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
106
+ } else if (time.lt(head)) {
107
+ throw new ServiceAssertionError(`operationTime must be > clusterTime`);
99
108
  }
100
- );
101
- const time = session.operationTime!;
102
- if (time == null) {
103
- throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
104
- } else if (time.lt(head)) {
105
- throw new ServiceAssertionError(`operationTime must be > clusterTime`);
106
109
  }
107
110
 
108
- return r;
111
+ return response;
109
112
  } finally {
110
113
  await session.endSession();
111
114
  }