@powersync/service-module-mongodb 0.15.4 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/dist/replication/ChangeStream.d.ts +6 -6
  3. package/dist/replication/ChangeStream.js +300 -322
  4. package/dist/replication/ChangeStream.js.map +1 -1
  5. package/dist/replication/ChangeStreamReplicationJob.js +2 -2
  6. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  7. package/dist/replication/JsonBufferWriter.d.ts +80 -0
  8. package/dist/replication/JsonBufferWriter.js +342 -0
  9. package/dist/replication/JsonBufferWriter.js.map +1 -0
  10. package/dist/replication/MongoRelation.js +4 -0
  11. package/dist/replication/MongoRelation.js.map +1 -1
  12. package/dist/replication/MongoSnapshotQuery.d.ts +1 -1
  13. package/dist/replication/MongoSnapshotQuery.js +6 -3
  14. package/dist/replication/MongoSnapshotQuery.js.map +1 -1
  15. package/dist/replication/RawChangeStream.d.ts +55 -0
  16. package/dist/replication/RawChangeStream.js +322 -0
  17. package/dist/replication/RawChangeStream.js.map +1 -0
  18. package/dist/replication/SourceRowConverter.d.ts +46 -0
  19. package/dist/replication/SourceRowConverter.js +42 -0
  20. package/dist/replication/SourceRowConverter.js.map +1 -0
  21. package/dist/replication/bufferToSqlite.d.ts +43 -0
  22. package/dist/replication/bufferToSqlite.js +740 -0
  23. package/dist/replication/bufferToSqlite.js.map +1 -0
  24. package/dist/replication/internal-mongodb-utils.d.ts +0 -12
  25. package/dist/replication/internal-mongodb-utils.js +0 -54
  26. package/dist/replication/internal-mongodb-utils.js.map +1 -1
  27. package/dist/replication/replication-index.d.ts +2 -0
  28. package/dist/replication/replication-index.js +2 -0
  29. package/dist/replication/replication-index.js.map +1 -1
  30. package/package.json +11 -11
  31. package/scripts/benchmark-change-document-json.mts +358 -0
  32. package/scripts/benchmark-change-document.mts +370 -0
  33. package/src/replication/ChangeStream.ts +348 -371
  34. package/src/replication/ChangeStreamReplicationJob.ts +2 -2
  35. package/src/replication/JsonBufferWriter.ts +390 -0
  36. package/src/replication/MongoRelation.ts +3 -0
  37. package/src/replication/MongoSnapshotQuery.ts +8 -5
  38. package/src/replication/RawChangeStream.ts +460 -0
  39. package/src/replication/SourceRowConverter.ts +65 -0
  40. package/src/replication/bufferToSqlite.ts +944 -0
  41. package/src/replication/internal-mongodb-utils.ts +0 -65
  42. package/src/replication/replication-index.ts +2 -0
  43. package/test/src/buffer_to_sqlite.test.ts +1146 -0
  44. package/test/src/change_stream.test.ts +49 -2
  45. package/test/src/change_stream_utils.ts +4 -10
  46. package/test/src/mongo_test.test.ts +66 -64
  47. package/test/src/parse_document_id.test.ts +54 -0
  48. package/test/src/raw_change_stream.test.ts +547 -0
  49. package/test/src/resume.test.ts +12 -2
  50. package/test/src/util.ts +56 -3
  51. package/test/tsconfig.json +0 -1
  52. package/tsconfig.scripts.json +13 -0
  53. package/tsconfig.tsbuildinfo +1 -1
  54. package/test/src/internal_mongodb_utils.test.ts +0 -103
package/CHANGELOG.md CHANGED
@@ -1,5 +1,39 @@
1
1
  # @powersync/service-module-mongodb
2
2
 
3
+ ## 0.16.0
4
+
5
+ ### Minor Changes
6
+
7
+ - d564c23: Use custom MongoDB change stream implementation to get better low-level control.
8
+
9
+ ### Patch Changes
10
+
11
+ - 7c7b525: [MongoDB] Log replication timing info per batch
12
+ - 040fffd: Improve consistency of logs and error messages
13
+ - 176885c: Use adaptive batchSize for better recovery in PSYNC_S1345 change stream timeouts.
14
+ - 2b19fc3: Update first-party uuid dependencies to v14.
15
+ - Updated dependencies [f20f318]
16
+ - Updated dependencies [9add445]
17
+ - Updated dependencies [17503d1]
18
+ - Updated dependencies [ad9ea06]
19
+ - Updated dependencies [01c29c3]
20
+ - Updated dependencies [8afe719]
21
+ - Updated dependencies [b8f0195]
22
+ - Updated dependencies [cdb8993]
23
+ - Updated dependencies [7c7b525]
24
+ - Updated dependencies [824e229]
25
+ - Updated dependencies [6304a21]
26
+ - Updated dependencies [040fffd]
27
+ - Updated dependencies [9e474d3]
28
+ - Updated dependencies [423822c]
29
+ - Updated dependencies [2b19fc3]
30
+ - @powersync/service-core@1.21.0
31
+ - @powersync/service-sync-rules@0.36.0
32
+ - @powersync/lib-services-framework@0.9.4
33
+ - @powersync/service-types@0.15.2
34
+ - @powersync/lib-service-mongodb@0.6.25
35
+ - @powersync/service-jsonbig@0.17.13
36
+
3
37
  ## 0.15.4
4
38
 
5
39
  ### Patch Changes
@@ -1,8 +1,9 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { DatabaseConnectionError, Logger } from '@powersync/lib-services-framework';
3
3
  import { MetricsEngine, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core';
4
- import { DatabaseInputRow, HydratedSyncRules, SqliteInputRow, TablePattern } from '@powersync/service-sync-rules';
4
+ import { HydratedSyncRules, TablePattern } from '@powersync/service-sync-rules';
5
5
  import { MongoManager } from './MongoManager.js';
6
+ import { ProjectedChangeStreamDocument } from './RawChangeStream.js';
6
7
  export interface ChangeStreamOptions {
7
8
  connections: MongoManager;
8
9
  storage: storage.SyncRulesBucketStorage;
@@ -53,6 +54,7 @@ export declare class ChangeStream {
53
54
  private logger;
54
55
  private snapshotChunkLength;
55
56
  private changeStreamTimeout;
57
+ private readonly sourceRowConverter;
56
58
  constructor(options: ChangeStreamOptions);
57
59
  get stopped(): boolean;
58
60
  private get usePostImages();
@@ -82,7 +84,6 @@ export declare class ChangeStream {
82
84
  }>;
83
85
  private setupCheckpointsCollection;
84
86
  private getSourceNamespaceFilters;
85
- static getQueryData(results: Iterable<DatabaseInputRow>): Generator<SqliteInputRow>;
86
87
  private snapshotTable;
87
88
  private getRelation;
88
89
  private getCollectionInfo;
@@ -91,13 +92,12 @@ export declare class ChangeStream {
91
92
  snapshot: boolean;
92
93
  collectionInfo: mongo.CollectionInfo | undefined;
93
94
  }): Promise<SourceTable>;
94
- private constructAfterRecord;
95
- writeChange(batch: storage.BucketStorageBatch, table: storage.SourceTable, change: mongo.ChangeStreamDocument): Promise<storage.FlushedResult | null>;
95
+ writeChange(batch: storage.BucketStorageBatch, table: storage.SourceTable, change: ProjectedChangeStreamDocument): Promise<storage.FlushedResult | null>;
96
96
  replicate(): Promise<void>;
97
97
  initReplication(): Promise<void>;
98
98
  streamChanges(): Promise<void>;
99
- private openChangeStream;
100
- private getBufferedChangeCount;
99
+ private rawChangeStreamBatches;
100
+ private rawToSqliteRow;
101
101
  streamChangesInternal(): Promise<void>;
102
102
  getReplicationLagMillis(): number | undefined;
103
103
  private lastTouchedAt;