@powersync/service-module-mongodb 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/CHANGELOG.md +95 -7
  2. package/ci/test-connection.yaml +12 -0
  3. package/dist/api/MongoRouteAPIAdapter.d.ts +2 -2
  4. package/dist/api/MongoRouteAPIAdapter.js +73 -143
  5. package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
  6. package/dist/api/infer-collection-schema.d.ts +12 -0
  7. package/dist/api/infer-collection-schema.js +142 -0
  8. package/dist/api/infer-collection-schema.js.map +1 -0
  9. package/dist/index.d.ts +1 -0
  10. package/dist/index.js +1 -0
  11. package/dist/index.js.map +1 -1
  12. package/dist/module/MongoModule.d.ts +1 -2
  13. package/dist/module/MongoModule.js +2 -2
  14. package/dist/module/MongoModule.js.map +1 -1
  15. package/dist/replication/ChangeStream.d.ts +18 -2
  16. package/dist/replication/ChangeStream.js +275 -307
  17. package/dist/replication/ChangeStream.js.map +1 -1
  18. package/dist/replication/ChangeStreamReplicationJob.d.ts +4 -1
  19. package/dist/replication/ChangeStreamReplicationJob.js +6 -2
  20. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  21. package/dist/replication/ChangeStreamReplicator.d.ts +3 -0
  22. package/dist/replication/ChangeStreamReplicator.js +3 -0
  23. package/dist/replication/ChangeStreamReplicator.js.map +1 -1
  24. package/dist/replication/MongoRelation.js +5 -2
  25. package/dist/replication/MongoRelation.js.map +1 -1
  26. package/dist/replication/MongoReplicationQueryProvider.d.ts +87 -0
  27. package/dist/replication/MongoReplicationQueryProvider.js +23 -0
  28. package/dist/replication/MongoReplicationQueryProvider.js.map +1 -0
  29. package/dist/replication/MongoReplicationStream.d.ts +85 -0
  30. package/dist/replication/MongoReplicationStream.js +101 -0
  31. package/dist/replication/MongoReplicationStream.js.map +1 -0
  32. package/dist/replication/MongoSnapshotQuery.d.ts +2 -0
  33. package/dist/replication/MongoSnapshotQuery.js +12 -2
  34. package/dist/replication/MongoSnapshotQuery.js.map +1 -1
  35. package/dist/replication/MongoSnapshotter.d.ts +7 -2
  36. package/dist/replication/MongoSnapshotter.js +64 -91
  37. package/dist/replication/MongoSnapshotter.js.map +1 -1
  38. package/dist/replication/RawChangeStream.d.ts +4 -12
  39. package/dist/replication/RawChangeStream.js +1 -18
  40. package/dist/replication/RawChangeStream.js.map +1 -1
  41. package/dist/replication/SourceRowConverter.d.ts +1 -0
  42. package/dist/replication/SourceRowConverter.js +4 -2
  43. package/dist/replication/SourceRowConverter.js.map +1 -1
  44. package/dist/replication/bufferToSqlite.d.ts +3 -1
  45. package/dist/replication/bufferToSqlite.js +35 -23
  46. package/dist/replication/bufferToSqlite.js.map +1 -1
  47. package/dist/replication/checkpoints/CheckpointImplementation.d.ts +4 -1
  48. package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +4 -1
  49. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +1 -1
  50. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -1
  51. package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +4 -1
  52. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +2 -1
  53. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -1
  54. package/dist/replication/replication-index.d.ts +6 -0
  55. package/dist/replication/replication-index.js +6 -0
  56. package/dist/replication/replication-index.js.map +1 -1
  57. package/dist/test-utils/ChangeStreamTestContext.d.ts +131 -0
  58. package/dist/test-utils/ChangeStreamTestContext.js +391 -0
  59. package/dist/test-utils/ChangeStreamTestContext.js.map +1 -0
  60. package/dist/test-utils/test-utils-index.d.ts +1 -0
  61. package/dist/test-utils/test-utils-index.js +2 -0
  62. package/dist/test-utils/test-utils-index.js.map +1 -0
  63. package/dist/types/types.d.ts +5 -0
  64. package/dist/types/types.js +20 -2
  65. package/dist/types/types.js.map +1 -1
  66. package/package.json +9 -9
  67. package/src/api/MongoRouteAPIAdapter.ts +79 -127
  68. package/src/api/infer-collection-schema.ts +153 -0
  69. package/src/index.ts +1 -0
  70. package/src/module/MongoModule.ts +3 -4
  71. package/src/replication/ChangeStream.ts +303 -350
  72. package/src/replication/ChangeStreamReplicationJob.ts +9 -3
  73. package/src/replication/ChangeStreamReplicator.ts +5 -0
  74. package/src/replication/MongoRelation.ts +8 -1
  75. package/src/replication/MongoReplicationQueryProvider.ts +108 -0
  76. package/src/replication/MongoReplicationStream.ts +201 -0
  77. package/src/replication/MongoSnapshotQuery.ts +19 -3
  78. package/src/replication/MongoSnapshotter.ts +76 -98
  79. package/src/replication/RawChangeStream.ts +5 -32
  80. package/src/replication/SourceRowConverter.ts +4 -2
  81. package/src/replication/bufferToSqlite.ts +43 -23
  82. package/src/replication/checkpoints/CheckpointImplementation.ts +1 -1
  83. package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +2 -2
  84. package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +3 -2
  85. package/src/replication/replication-index.ts +6 -0
  86. package/src/test-utils/ChangeStreamTestContext.ts +520 -0
  87. package/src/test-utils/test-utils-index.ts +1 -0
  88. package/src/types/types.ts +27 -2
  89. package/test/src/buffer_to_sqlite.test.ts +9 -0
  90. package/test/src/change_stream.test.ts +76 -3
  91. package/test/src/change_stream_test_setup.ts +38 -0
  92. package/test/src/chunked_snapshot.test.ts +3 -3
  93. package/test/src/config.test.ts +34 -0
  94. package/test/src/documentdb_mode.test.ts +4 -5
  95. package/test/src/raw_change_stream.test.ts +0 -63
  96. package/test/src/replication_stream.test.ts +311 -0
  97. package/test/src/resume.test.ts +10 -4
  98. package/test/src/resuming_snapshots.test.ts +3 -3
  99. package/test/src/schema.test.ts +223 -0
  100. package/test/src/slow_tests.test.ts +3 -3
  101. package/test/src/snapshot_query_provider.test.ts +95 -0
  102. package/test/src/stream_progress.test.ts +312 -0
  103. package/test/src/util.ts +11 -29
  104. package/tsconfig.tsbuildinfo +1 -1
  105. package/test/src/change_stream_utils.ts +0 -375
package/dist/index.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  export * from './api/MongoRouteAPIAdapter.js';
2
2
  export * from './module/MongoModule.js';
3
3
  export * from './replication/replication-index.js';
4
+ export * as test_utils from './test-utils/test-utils-index.js';
4
5
  export * from './types/types.js';
package/dist/index.js CHANGED
@@ -1,5 +1,6 @@
1
1
  export * from './api/MongoRouteAPIAdapter.js';
2
2
  export * from './module/MongoModule.js';
3
3
  export * from './replication/replication-index.js';
4
+ export * as test_utils from './test-utils/test-utils-index.js';
4
5
  export * from './types/types.js';
5
6
  //# sourceMappingURL=index.js.map
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,oCAAoC,CAAC;AACnD,cAAc,kBAAkB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,oCAAoC,CAAC;AACnD,OAAO,KAAK,UAAU,MAAM,kCAAkC,CAAC;AAC/D,cAAc,kBAAkB,CAAC"}
@@ -4,11 +4,10 @@ export declare class MongoModule extends replication.ReplicationModule<types.Mon
4
4
  constructor();
5
5
  protected createRouteAPIAdapter(): api.RouteAPI;
6
6
  protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator;
7
- onInitialized(context: system.ServiceContextContainer): Promise<void>;
8
7
  /**
9
8
  * Combines base config with normalized connection settings
10
9
  */
11
- private resolveConfig;
10
+ protected resolveConfig(config: types.MongoConnectionConfig): types.ResolvedConnectionConfig;
12
11
  teardown(options: TearDownOptions): Promise<void>;
13
12
  testConnection(config: types.MongoConnectionConfig): Promise<ConnectionTestResult>;
14
13
  static testConnection(normalizedConfig: types.NormalizedMongoConnectionConfig): Promise<ConnectionTestResult>;
@@ -28,10 +28,10 @@ export class MongoModule extends replication.ReplicationModule {
28
28
  storageEngine: context.storageEngine,
29
29
  metricsEngine: context.metricsEngine,
30
30
  connectionFactory: connectionFactory,
31
- rateLimiter: new MongoErrorRateLimiter()
31
+ rateLimiter: new MongoErrorRateLimiter(),
32
+ heartbeatIntervalSeconds: normalisedConfig.heartbeat_interval_seconds
32
33
  });
33
34
  }
34
- async onInitialized(context) { }
35
35
  /**
36
36
  * Combines base config with normalized connection settings
37
37
  */
@@ -1 +1 @@
1
- {"version":3,"file":"MongoModule.js","sourceRoot":"","sources":["../../src/module/MongoModule.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAEL,kCAAkC,EAElC,WAAW,EAGZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,qBAAqB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,MAAM,qCAAqC,CAAC;AAC/E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAY,SAAQ,WAAW,CAAC,iBAA8C;IACzF;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS,CAAC,qBAAqB;YACrC,YAAY,EAAE,KAAK,CAAC,qBAAqB;SAC1C,CAAC,CAAC;IACL,CAAC;IAES,qBAAqB;QAC7B,OAAO,IAAI,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC,CAAC;IAC3E,CAAC;IAES,gBAAgB,CAAC,OAA8B;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;QAEzE,OAAO,IAAI,sBAAsB,CAAC;YAChC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,IAAI,EAAE,CAAC;YACtD,gBAAgB,EAAE,gBAAgB;YAClC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,iBAAiB,EAAE,iBAAiB;YACpC,WAAW,EAAE,IAAI,qBAAqB,EAAE;SACzC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAuC,IAAkB,CAAC;IAE9E;;OAEG;IACK,aAAa,CAAC,MAAmC;QACvD,OAAO;YACL,GAAG,MAAM;YACT,GAAG,KAAK,CAAC,yBAAyB,CAAC,MAAM,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,QAAQ;IACV,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAmC;QACtD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,OAAO,MAAM,WAAW,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,gBAAuD;QACjF,MAAM,iBAAiB,GAAG,IAAI,YAAY,CAAC,gBAAgB,EAAE;YAC3D,8CAA8C;YAC9C,0DAA0D;YAC1D,0FAA0F;YAC1F,eAAe,EAAE,KAAK;YACtB,wBAAwB,EAAE,KAAK;SAChC,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,wBAAwB,CAAC,iBAAiB,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;QACD,OAAO;YACL,qBAAqB,EAAE,gBAAgB,CAAC,GAAG;SAC5C,CAAC;IACJ,CAAC;CACF"}
1
+ {"version":3,"file":"MongoModule.js","sourceRoot":"","sources":["../../src/module/MongoModule.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAEL,kCAAkC,EAElC,WAAW,EAGZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,qBAAqB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,MAAM,qCAAqC,CAAC;AAC/E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAY,SAAQ,WAAW,CAAC,iBAA8C;IACzF;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS,CAAC,qBAAqB;YACrC,YAAY,EAAE,KAAK,CAAC,qBAAqB;SAC1C,CAAC,CAAC;IACL,CAAC;IAES,qBAAqB;QAC7B,OAAO,IAAI,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC,CAAC;IAC3E,CAAC;IAES,gBAAgB,CAAC,OAA8B;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;QAEzE,OAAO,IAAI,sBAAsB,CAAC;YAChC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,IAAI,EAAE,CAAC;YACtD,gBAAgB,EAAE,gBAAgB;YAClC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,iBAAiB,EAAE,iBAAiB;YACpC,WAAW,EAAE,IAAI,qBAAqB,EAAE;YACxC,wBAAwB,EAAE,gBAAgB,CAAC,0BAA0B;SACtE,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,MAAmC;QACzD,OAAO;YACL,GAAG,MAAM;YACT,GAAG,KAAK,CAAC,yBAAyB,CAAC,MAAM,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,QAAQ;IACV,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAmC;QACtD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,OAAO,MAAM,WAAW,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,gBAAuD;QACjF,MAAM,iBAAiB,GAAG,IAAI,YAAY,CAAC,gBAAgB,EAAE;YAC3D,8CAA8C;YAC9C,0DAA0D;YAC1D,0FAA0F;YAC1F,eAAe,EAAE,KAAK;YACtB,wBAAwB,EAAE,KAAK;SAChC,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,wBAAwB,CAAC,iBAAiB,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;QACD,OAAO;YACL,qBAAqB,EAAE,gBAAgB,CAAC,GAAG;SAC5C,CAAC;IACJ,CAAC;CACF"}
@@ -3,6 +3,7 @@ import { DatabaseConnectionError, Logger } from '@powersync/lib-services-framewo
3
3
  import { MetricsEngine, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core';
4
4
  import { HydratedSyncConfig } from '@powersync/service-sync-rules';
5
5
  import { MongoManager } from './MongoManager.js';
6
+ import { MongoReplicationQueryProviderFactory } from './MongoReplicationQueryProvider.js';
6
7
  import { MongoSnapshotterHooks } from './MongoSnapshotter.js';
7
8
  import { ProjectedChangeStreamDocument } from './RawChangeStream.js';
8
9
  export interface ChangeStreamOptions {
@@ -27,6 +28,10 @@ export interface ChangeStreamOptions {
27
28
  keepaliveIntervalMs?: number;
28
29
  storageHooks?: storage.StorageHooks;
29
30
  snapshotHooks?: MongoSnapshotterHooks;
31
+ /**
32
+ * Create one adapter per replication attempt, shared with its snapshotter and source probes.
33
+ */
34
+ createReplicationQueryProvider?: MongoReplicationQueryProviderFactory;
30
35
  logger?: Logger;
31
36
  }
32
37
  /**
@@ -65,14 +70,23 @@ export declare class ChangeStream {
65
70
  private changeStreamTimeout;
66
71
  private storageHooks;
67
72
  private readonly sourceRowConverter;
73
+ private readonly queryProvider;
68
74
  private keepaliveIntervalMs;
69
75
  private isDocumentDb;
70
76
  private _checkpointImplementation;
77
+ /**
78
+ * Last persisted resume timestamp as Date.now(). This represents the equivalent timestamp from the
79
+ * source database, rather than the time when we persisted it.
80
+ */
81
+ private lastPersistedResumeTimestamp;
82
+ private lastBatchCheckpoint;
71
83
  constructor(options: ChangeStreamOptions);
72
84
  get stopped(): boolean;
73
85
  private get usePostImages();
74
86
  private get configurePostImages();
75
- /** The active checkpoint strategy. Only valid after ensureDetected(). */
87
+ /**
88
+ * The active checkpoint strategy. Only valid after ensureDetected().
89
+ */
76
90
  private get checkpointImplementation();
77
91
  /**
78
92
  * Detect DocumentDB and select the checkpoint implementation for the streaming
@@ -94,10 +108,12 @@ export declare class ChangeStream {
94
108
  private initReplication;
95
109
  private cleanupStoppedSyncConfigs;
96
110
  private streamChanges;
97
- private rawChangeStreamBatches;
111
+ private openChangeStream;
98
112
  private rawToSqliteRow;
113
+ private createBatchCheckpoint;
99
114
  streamChangesInternal(): Promise<void>;
100
115
  getReplicationLagMillis(): number | undefined;
116
+ keepAlive(): Promise<void>;
101
117
  private lastTouchedAt;
102
118
  private touch;
103
119
  }