@powersync/service-module-mongodb-storage 0.9.4 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/dist/migrations/db/migrations/1749720702136-checkpoint-events.d.ts +3 -0
- package/dist/migrations/db/migrations/1749720702136-checkpoint-events.js +34 -0
- package/dist/migrations/db/migrations/1749720702136-checkpoint-events.js.map +1 -0
- package/dist/storage/MongoBucketStorage.js +5 -0
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +9 -3
- package/dist/storage/implementation/MongoBucketBatch.js +117 -37
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +1 -0
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +2 -0
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +23 -1
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +14 -5
- package/dist/storage/implementation/MongoSyncBucketStorage.js +161 -159
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoTestStorageFactoryGenerator.js +2 -0
- package/dist/storage/implementation/MongoTestStorageFactoryGenerator.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -15
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +55 -191
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/PersistedBatch.d.ts +6 -2
- package/dist/storage/implementation/PersistedBatch.js +39 -7
- package/dist/storage/implementation/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +12 -1
- package/dist/storage/implementation/db.js +39 -0
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +29 -1
- package/package.json +7 -7
- package/src/migrations/db/migrations/1749720702136-checkpoint-events.ts +50 -0
- package/src/storage/MongoBucketStorage.ts +5 -0
- package/src/storage/implementation/MongoBucketBatch.ts +160 -49
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +2 -0
- package/src/storage/implementation/MongoStorageProvider.ts +27 -1
- package/src/storage/implementation/MongoSyncBucketStorage.ts +187 -200
- package/src/storage/implementation/MongoTestStorageFactoryGenerator.ts +3 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +66 -255
- package/src/storage/implementation/PersistedBatch.ts +50 -11
- package/src/storage/implementation/db.ts +42 -0
- package/src/storage/implementation/models.ts +32 -1
- package/test/src/__snapshots__/storage_sync.test.ts.snap +147 -0
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,39 @@
|
|
|
1
1
|
# @powersync/service-module-mongodb-storage
|
|
2
2
|
|
|
3
|
+
## 0.10.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 0ccd470: Add powersync_replication_lag_seconds metric
|
|
8
|
+
- 951b010: Implement resuming of initial replication snapshots.
|
|
9
|
+
- d235f7b: [MongoDB Storage] Remove change streams on bucket storage database due to performance overhead.
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 1907356: Cleanly interrupt clearing of storage when the process is stopped/restarted.
|
|
14
|
+
- f9e8673: [MongoDB Storage] Handle connection errors on startup
|
|
15
|
+
- Updated dependencies [08b7aa9]
|
|
16
|
+
- Updated dependencies [0ccd470]
|
|
17
|
+
- Updated dependencies [1907356]
|
|
18
|
+
- Updated dependencies [951b010]
|
|
19
|
+
- Updated dependencies [d235f7b]
|
|
20
|
+
- Updated dependencies [f9e8673]
|
|
21
|
+
- @powersync/service-core@1.13.0
|
|
22
|
+
- @powersync/service-types@0.12.0
|
|
23
|
+
- @powersync/lib-services-framework@0.7.0
|
|
24
|
+
- @powersync/lib-service-mongodb@0.6.1
|
|
25
|
+
|
|
26
|
+
## 0.9.5
|
|
27
|
+
|
|
28
|
+
### Patch Changes
|
|
29
|
+
|
|
30
|
+
- 5b39039: Cleanup on internal sync rules implementation and APIs.
|
|
31
|
+
- Updated dependencies [100ccec]
|
|
32
|
+
- Updated dependencies [b57f938]
|
|
33
|
+
- Updated dependencies [5b39039]
|
|
34
|
+
- @powersync/service-core@1.12.1
|
|
35
|
+
- @powersync/service-sync-rules@0.27.0
|
|
36
|
+
|
|
3
37
|
## 0.9.4
|
|
4
38
|
|
|
5
39
|
### Patch Changes
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import * as storage from '../../../storage/storage-index.js';
|
|
2
|
+
export const up = async (context) => {
|
|
3
|
+
const { service_context: { configuration } } = context;
|
|
4
|
+
const db = storage.createPowerSyncMongo(configuration.storage);
|
|
5
|
+
try {
|
|
6
|
+
await db.createCheckpointEventsCollection();
|
|
7
|
+
await db.write_checkpoints.createIndex({
|
|
8
|
+
processed_at_lsn: 1
|
|
9
|
+
}, { name: 'processed_at_lsn' });
|
|
10
|
+
await db.custom_write_checkpoints.createIndex({
|
|
11
|
+
op_id: 1
|
|
12
|
+
}, { name: 'op_id' });
|
|
13
|
+
}
|
|
14
|
+
finally {
|
|
15
|
+
await db.client.close();
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
export const down = async (context) => {
|
|
19
|
+
const { service_context: { configuration } } = context;
|
|
20
|
+
const db = storage.createPowerSyncMongo(configuration.storage);
|
|
21
|
+
try {
|
|
22
|
+
if (await db.write_checkpoints.indexExists('processed_at_lsn')) {
|
|
23
|
+
await db.write_checkpoints.dropIndex('processed_at_lsn');
|
|
24
|
+
}
|
|
25
|
+
if (await db.custom_write_checkpoints.indexExists('op_id')) {
|
|
26
|
+
await db.custom_write_checkpoints.dropIndex('op_id');
|
|
27
|
+
}
|
|
28
|
+
await db.db.dropCollection('checkpoint_events');
|
|
29
|
+
}
|
|
30
|
+
finally {
|
|
31
|
+
await db.client.close();
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
//# sourceMappingURL=1749720702136-checkpoint-events.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"1749720702136-checkpoint-events.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1749720702136-checkpoint-events.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,mCAAmC,CAAC;AAG7D,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAErF,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,gCAAgC,EAAE,CAAC;QAE5C,MAAM,EAAE,CAAC,iBAAiB,CAAC,WAAW,CACpC;YACE,gBAAgB,EAAE,CAAC;SACpB,EACD,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAC7B,CAAC;QAEF,MAAM,EAAE,CAAC,wBAAwB,CAAC,WAAW,CAC3C;YACE,KAAK,EAAE,CAAC;SACT,EACD,EAAE,IAAI,EAAE,OAAO,EAAE,CAClB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IAEZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAErF,IAAI,CAAC;QACH,IAAI,MAAM,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC,kBAAkB,CAAC,EAAE,CAAC;YAC/D,MAAM,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,MAAM,EAAE,CAAC,wBAAwB,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3D,MAAM,EAAE,CAAC,wBAAwB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,EAAE,CAAC,EAAE,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;IAClD,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -88,6 +88,7 @@ export class MongoBucketStorage extends BaseObserver {
|
|
|
88
88
|
state: storage.SyncRuleState.STOP
|
|
89
89
|
}
|
|
90
90
|
});
|
|
91
|
+
await this.db.notifyCheckpoint();
|
|
91
92
|
}
|
|
92
93
|
else if (next == null && active?.id == sync_rules_group_id) {
|
|
93
94
|
// Slot removed for "active" sync rules, while there is no "next" one.
|
|
@@ -106,6 +107,7 @@ export class MongoBucketStorage extends BaseObserver {
|
|
|
106
107
|
state: storage.SyncRuleState.ERRORED
|
|
107
108
|
}
|
|
108
109
|
});
|
|
110
|
+
await this.db.notifyCheckpoint();
|
|
109
111
|
}
|
|
110
112
|
else if (next != null && active?.id == sync_rules_group_id) {
|
|
111
113
|
// Already have next sync rules, but need to stop replicating the active one.
|
|
@@ -117,6 +119,7 @@ export class MongoBucketStorage extends BaseObserver {
|
|
|
117
119
|
state: storage.SyncRuleState.ERRORED
|
|
118
120
|
}
|
|
119
121
|
});
|
|
122
|
+
await this.db.notifyCheckpoint();
|
|
120
123
|
}
|
|
121
124
|
}
|
|
122
125
|
async updateSyncRules(options) {
|
|
@@ -159,6 +162,7 @@ export class MongoBucketStorage extends BaseObserver {
|
|
|
159
162
|
no_checkpoint_before: null,
|
|
160
163
|
keepalive_op: null,
|
|
161
164
|
snapshot_done: false,
|
|
165
|
+
snapshot_lsn: undefined,
|
|
162
166
|
state: storage.SyncRuleState.PROCESSING,
|
|
163
167
|
slot_name: slot_name,
|
|
164
168
|
last_checkpoint_ts: null,
|
|
@@ -166,6 +170,7 @@ export class MongoBucketStorage extends BaseObserver {
|
|
|
166
170
|
last_keepalive_ts: null
|
|
167
171
|
};
|
|
168
172
|
await this.db.sync_rules.insertOne(doc);
|
|
173
|
+
await this.db.notifyCheckpoint();
|
|
169
174
|
rules = new MongoPersistedSyncRulesContent(this.db, doc);
|
|
170
175
|
if (options.lock) {
|
|
171
176
|
const lock = await rules.lock();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoBucketStorage.js","sourceRoot":"","sources":["../../src/storage/MongoBucketStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAE7D,OAAO,EAAqB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAClG,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAK5D,OAAO,EAAE,8BAA8B,EAAE,MAAM,oDAAoD,CAAC;AACpG,OAAO,EAAE,sBAAsB,EAAE,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAE5D,MAAM,OAAO,kBACX,SAAQ,YAAkD;IAGzC,MAAM,CAAoB;IAC1B,OAAO,CAAsB;IAC9C,iEAAiE;IACjD,gBAAgB,CAAS;IAEjC,kBAAkB,CAAqC;IAE/C,EAAE,CAAiB;IAEnC,YACE,EAAkB,EAClB,OAEC;QAED,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,QAAQ;IACV,CAAC;IAED,WAAW,CAAC,SAA4C,EAAE,OAA2B;QACnF,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC;QAClC,IAAK,OAAO,EAAU,IAAI,QAAQ,EAAE,CAAC;YACnC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,sBAAsB,CAAC,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,EAAE,kBAAkB,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,CAAC,gBAAgB,CAAC;YACvB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;gBACtB,KAAK,CAAC,gBAAgB,CAAC;oBACrB,gBAAgB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,CAAC;iBAC7F,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC;YAC/C,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,oEAAoE,CACrE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE;YACF,IAAI,EAAE,SAAS,CAAC,qBAAqB;SACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAuC;QAC9D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,EAAE,kBAAkB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,kBAAkB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACzE,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YACrD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;QACvG,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,mBAA2B;QAClD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,EAAE,IAAI,mBAAmB,EAAE,CAAC;YACnD,wCAAwC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,IAAI,CAAC,kBAAkB;gBAChC,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YACH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,IAAI,CAAC,EAAE;gBACZ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;iBAClC;aACF,CACF,CAAC;
|
|
1
|
+
{"version":3,"file":"MongoBucketStorage.js","sourceRoot":"","sources":["../../src/storage/MongoBucketStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAE7D,OAAO,EAAqB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAClG,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAK5D,OAAO,EAAE,8BAA8B,EAAE,MAAM,oDAAoD,CAAC;AACpG,OAAO,EAAE,sBAAsB,EAAE,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAE5D,MAAM,OAAO,kBACX,SAAQ,YAAkD;IAGzC,MAAM,CAAoB;IAC1B,OAAO,CAAsB;IAC9C,iEAAiE;IACjD,gBAAgB,CAAS;IAEjC,kBAAkB,CAAqC;IAE/C,EAAE,CAAiB;IAEnC,YACE,EAAkB,EAClB,OAEC;QAED,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,QAAQ;IACV,CAAC;IAED,WAAW,CAAC,SAA4C,EAAE,OAA2B;QACnF,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC;QAClC,IAAK,OAAO,EAAU,IAAI,QAAQ,EAAE,CAAC;YACnC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,sBAAsB,CAAC,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,EAAE,kBAAkB,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,CAAC,gBAAgB,CAAC;YACvB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;gBACtB,KAAK,CAAC,gBAAgB,CAAC;oBACrB,gBAAgB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,CAAC;iBAC7F,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC;YAC/C,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,oEAAoE,CACrE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE;YACF,IAAI,EAAE,SAAS,CAAC,qBAAqB;SACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAuC;QAC9D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,EAAE,kBAAkB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,kBAAkB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACzE,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YACrD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;QACvG,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,mBAA2B;QAClD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,EAAE,IAAI,mBAAmB,EAAE,CAAC;YACnD,wCAAwC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,IAAI,CAAC,kBAAkB;gBAChC,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YACH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,IAAI,CAAC,EAAE;gBACZ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;iBAClC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,EAAE,IAAI,mBAAmB,EAAE,CAAC;YAC7D,sEAAsE;YACtE,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC,kBAAkB;gBAClC,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YAEH,wFAAwF;YACxF,iEAAiE;YACjE,yCAAyC;YAEzC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO;iBACrC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,EAAE,IAAI,mBAAmB,EAAE,CAAC;YAC7D,6EAA6E;YAE7E,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO;iBACrC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAuC;QAC3D,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,iDAAiD;YACjD,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE;gBACrC,2CAA2C;gBAC3C,MAAM,EAAE,SAAS;gBACjB,aAAa,EAAE,gBAAgB,EAAE,4BAA4B;gBAC7D,YAAY,EAAE,IAAI;aACnB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,+CAA+C;YAC/C,6FAA6F;QAC/F,CAAC;QAED,IAAI,KAAK,GAA+C,SAAS,CAAC;QAElE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YAC5C,wDAAwD;YACxD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;gBACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAChD,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAC1D;gBACE,GAAG,EAAE,YAAY;aAClB,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,EAAE;iBACV;aACF,EACD;gBACE,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,OAAO;aACxB,CACF,CAAC;YAEF,MAAM,EAAE,GAAG,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC;YACjC,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE9D,MAAM,GAAG,GAAqB;gBAC5B,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,eAAe,EAAE,IAAI;gBACrB,mBAAmB,EAAE,IAAI;gBACzB,oBAAoB,EAAE,IAAI;gBAC1B,YAAY,EAAE,IAAI;gBAClB,aAAa,EAAE,KAAK;gBACpB,YAAY,EAAE,SAAS;gBACvB,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;gBACvC,SAAS,EAAE,SAAS;gBACpB,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,IAAI;gBACtB,iBAAiB,EAAE,IAAI;aACxB,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,KAAK,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,KAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,yBAAyB;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;SAC9E,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAsC;QAC7D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;SACxC,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAsC;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACrD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;SACjF,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;SAClC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,gDAAgD;QAChD,+FAA+F;QAC/F,wBAAwB;QACxB,IAAI,IAAI,CAAC,kBAAkB,EAAE,QAAQ,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;YACnC,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,iBAAiB,GAAG,CAAC,CAAU,EAAE,EAAE;YACvC,IAAI,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,mBAAmB,EAAE,CAAC;gBACzE,sCAAsC;gBACtC,OAAO,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrF,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,OAAO;gBACL,qBAAqB,EAAE,CAAC;gBACxB,qBAAqB,EAAE,CAAC;gBACxB,sBAAsB,EAAE,CAAC;aAC1B,CAAC;QACJ,CAAC;QACD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;aAEnD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aACzD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;aACrD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,OAAO;YACL,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,sBAAsB,EAAE,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;SAC3E,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,IAAI,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC5C,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC;gBACnD,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK;gBACzB,IAAI,EAAE,4BAA4B;aACnC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;oBAC/B,GAAG,EAAE,IAAI,EAAE;iBACZ,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACxC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;aACvB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAS,CAAC,GAAG,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { SqlEventDescriptor, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
3
|
-
import { BaseObserver } from '@powersync/lib-services-framework';
|
|
4
|
-
import { InternalOpId, storage } from '@powersync/service-core';
|
|
3
|
+
import { BaseObserver, Logger } from '@powersync/lib-services-framework';
|
|
4
|
+
import { BucketStorageMarkRecordUnavailable, InternalOpId, storage } from '@powersync/service-core';
|
|
5
5
|
import { PowerSyncMongo } from './db.js';
|
|
6
6
|
import { CurrentBucket } from './models.js';
|
|
7
7
|
/**
|
|
@@ -21,8 +21,11 @@ export interface MongoBucketBatchOptions {
|
|
|
21
21
|
* Set to true for initial replication.
|
|
22
22
|
*/
|
|
23
23
|
skipExistingRows: boolean;
|
|
24
|
+
markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
|
|
25
|
+
logger?: Logger;
|
|
24
26
|
}
|
|
25
27
|
export declare class MongoBucketBatch extends BaseObserver<storage.BucketBatchStorageListener> implements storage.BucketStorageBatch {
|
|
28
|
+
private logger;
|
|
26
29
|
private readonly client;
|
|
27
30
|
readonly db: PowerSyncMongo;
|
|
28
31
|
readonly session: mongo.ClientSession;
|
|
@@ -33,6 +36,7 @@ export declare class MongoBucketBatch extends BaseObserver<storage.BucketBatchSt
|
|
|
33
36
|
private readonly skipExistingRows;
|
|
34
37
|
private batch;
|
|
35
38
|
private write_checkpoint_batch;
|
|
39
|
+
private markRecordUnavailable;
|
|
36
40
|
/**
|
|
37
41
|
* Last LSN received associated with a checkpoint.
|
|
38
42
|
*
|
|
@@ -50,7 +54,7 @@ export declare class MongoBucketBatch extends BaseObserver<storage.BucketBatchSt
|
|
|
50
54
|
constructor(options: MongoBucketBatchOptions);
|
|
51
55
|
addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void;
|
|
52
56
|
get lastCheckpointLsn(): string | null;
|
|
53
|
-
flush(): Promise<storage.FlushedResult | null>;
|
|
57
|
+
flush(options?: storage.BatchBucketFlushOptions): Promise<storage.FlushedResult | null>;
|
|
54
58
|
private flushInner;
|
|
55
59
|
private replicateBatch;
|
|
56
60
|
private saveOperation;
|
|
@@ -60,6 +64,7 @@ export declare class MongoBucketBatch extends BaseObserver<storage.BucketBatchSt
|
|
|
60
64
|
private lastWaitingLogThottled;
|
|
61
65
|
commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<boolean>;
|
|
62
66
|
keepalive(lsn: string): Promise<boolean>;
|
|
67
|
+
setSnapshotLsn(lsn: string): Promise<void>;
|
|
63
68
|
save(record: storage.SaveOptions): Promise<storage.FlushedResult | null>;
|
|
64
69
|
/**
|
|
65
70
|
* Drop is equivalent to TRUNCATE, plus removing our record of the table.
|
|
@@ -67,6 +72,7 @@ export declare class MongoBucketBatch extends BaseObserver<storage.BucketBatchSt
|
|
|
67
72
|
drop(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
|
|
68
73
|
truncate(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
|
|
69
74
|
truncateSingle(sourceTable: storage.SourceTable): Promise<InternalOpId>;
|
|
75
|
+
updateTableProgress(table: storage.SourceTable, progress: Partial<storage.TableSnapshotStatus>): Promise<storage.SourceTable>;
|
|
70
76
|
markSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn: string): Promise<storage.SourceTable[]>;
|
|
71
77
|
/**
|
|
72
78
|
* Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import * as bson from 'bson';
|
|
3
|
-
import { BaseObserver, container, ErrorCode, errors, logger, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { BaseObserver, container, ErrorCode, errors, logger as defaultLogger, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
4
4
|
import { deserializeBson, SaveOperationTag, storage, utils } from '@powersync/service-core';
|
|
5
5
|
import * as timers from 'node:timers/promises';
|
|
6
6
|
import { MongoIdSequence } from './MongoIdSequence.js';
|
|
@@ -19,6 +19,7 @@ export const MAX_ROW_SIZE = 15 * 1024 * 1024;
|
|
|
19
19
|
// In the future, we can investigate allowing multiple replication streams operating independently.
|
|
20
20
|
const replicationMutex = new utils.Mutex();
|
|
21
21
|
export class MongoBucketBatch extends BaseObserver {
|
|
22
|
+
logger;
|
|
22
23
|
client;
|
|
23
24
|
db;
|
|
24
25
|
session;
|
|
@@ -29,6 +30,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
29
30
|
skipExistingRows;
|
|
30
31
|
batch = null;
|
|
31
32
|
write_checkpoint_batch = [];
|
|
33
|
+
markRecordUnavailable;
|
|
32
34
|
/**
|
|
33
35
|
* Last LSN received associated with a checkpoint.
|
|
34
36
|
*
|
|
@@ -45,6 +47,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
45
47
|
last_flushed_op = null;
|
|
46
48
|
constructor(options) {
|
|
47
49
|
super();
|
|
50
|
+
this.logger = options.logger ?? defaultLogger;
|
|
48
51
|
this.client = options.db.client;
|
|
49
52
|
this.db = options.db;
|
|
50
53
|
this.group_id = options.groupId;
|
|
@@ -55,6 +58,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
55
58
|
this.sync_rules = options.syncRules;
|
|
56
59
|
this.storeCurrentData = options.storeCurrentData;
|
|
57
60
|
this.skipExistingRows = options.skipExistingRows;
|
|
61
|
+
this.markRecordUnavailable = options.markRecordUnavailable;
|
|
58
62
|
this.batch = new OperationBatch();
|
|
59
63
|
this.persisted_op = options.keepaliveOp ?? null;
|
|
60
64
|
}
|
|
@@ -67,29 +71,31 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
67
71
|
get lastCheckpointLsn() {
|
|
68
72
|
return this.last_checkpoint_lsn;
|
|
69
73
|
}
|
|
70
|
-
async flush() {
|
|
74
|
+
async flush(options) {
|
|
71
75
|
let result = null;
|
|
72
76
|
// One flush may be split over multiple transactions.
|
|
73
77
|
// Each flushInner() is one transaction.
|
|
74
|
-
while (this.batch != null) {
|
|
75
|
-
let r = await this.flushInner();
|
|
78
|
+
while (this.batch != null || this.write_checkpoint_batch.length > 0) {
|
|
79
|
+
let r = await this.flushInner(options);
|
|
76
80
|
if (r) {
|
|
77
81
|
result = r;
|
|
78
82
|
}
|
|
79
83
|
}
|
|
80
|
-
await batchCreateCustomWriteCheckpoints(this.db, this.write_checkpoint_batch);
|
|
81
|
-
this.write_checkpoint_batch = [];
|
|
82
84
|
return result;
|
|
83
85
|
}
|
|
84
|
-
async flushInner() {
|
|
86
|
+
async flushInner(options) {
|
|
85
87
|
const batch = this.batch;
|
|
86
|
-
if (batch == null) {
|
|
87
|
-
return null;
|
|
88
|
-
}
|
|
89
88
|
let last_op = null;
|
|
90
89
|
let resumeBatch = null;
|
|
91
|
-
await this.withReplicationTransaction(`Flushing ${batch
|
|
92
|
-
|
|
90
|
+
await this.withReplicationTransaction(`Flushing ${batch?.length ?? 0} ops`, async (session, opSeq) => {
|
|
91
|
+
if (batch != null) {
|
|
92
|
+
resumeBatch = await this.replicateBatch(session, batch, opSeq, options);
|
|
93
|
+
}
|
|
94
|
+
if (this.write_checkpoint_batch.length > 0) {
|
|
95
|
+
this.logger.info(`Writing ${this.write_checkpoint_batch.length} custom write checkpoints`);
|
|
96
|
+
await batchCreateCustomWriteCheckpoints(this.db, session, this.write_checkpoint_batch, opSeq.next());
|
|
97
|
+
this.write_checkpoint_batch = [];
|
|
98
|
+
}
|
|
93
99
|
last_op = opSeq.last();
|
|
94
100
|
});
|
|
95
101
|
// null if done, set if we need another flush
|
|
@@ -101,7 +107,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
101
107
|
this.last_flushed_op = last_op;
|
|
102
108
|
return { flushed_op: last_op };
|
|
103
109
|
}
|
|
104
|
-
async replicateBatch(session, batch, op_seq) {
|
|
110
|
+
async replicateBatch(session, batch, op_seq, options) {
|
|
105
111
|
let sizes = undefined;
|
|
106
112
|
if (this.storeCurrentData && !this.skipExistingRows) {
|
|
107
113
|
// We skip this step if we don't store current_data, since the sizes will
|
|
@@ -161,7 +167,9 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
161
167
|
for await (let doc of cursor.stream()) {
|
|
162
168
|
current_data_lookup.set(cacheKey(doc._id.t, doc._id.k), doc);
|
|
163
169
|
}
|
|
164
|
-
let persistedBatch = new PersistedBatch(this.group_id, transactionSize
|
|
170
|
+
let persistedBatch = new PersistedBatch(this.group_id, transactionSize, {
|
|
171
|
+
logger: this.logger
|
|
172
|
+
});
|
|
165
173
|
for (let op of b) {
|
|
166
174
|
if (resumeBatch) {
|
|
167
175
|
resumeBatch.push(op);
|
|
@@ -181,7 +189,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
181
189
|
if (persistedBatch.shouldFlushTransaction()) {
|
|
182
190
|
// Transaction is getting big.
|
|
183
191
|
// Flush, and resume in a new transaction.
|
|
184
|
-
await persistedBatch.flush(this.db, this.session);
|
|
192
|
+
await persistedBatch.flush(this.db, this.session, options);
|
|
185
193
|
persistedBatch = null;
|
|
186
194
|
// Computing our current progress is a little tricky here, since
|
|
187
195
|
// we're stopping in the middle of a batch.
|
|
@@ -191,7 +199,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
191
199
|
}
|
|
192
200
|
if (persistedBatch) {
|
|
193
201
|
transactionSize = persistedBatch.currentSize;
|
|
194
|
-
await persistedBatch.flush(this.db, this.session);
|
|
202
|
+
await persistedBatch.flush(this.db, this.session, options);
|
|
195
203
|
}
|
|
196
204
|
}
|
|
197
205
|
return resumeBatch?.hasData() ? resumeBatch : null;
|
|
@@ -227,9 +235,17 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
227
235
|
// Not an error if we re-apply a transaction
|
|
228
236
|
existing_buckets = [];
|
|
229
237
|
existing_lookups = [];
|
|
230
|
-
// Log to help with debugging if there was a consistency issue
|
|
231
238
|
if (this.storeCurrentData) {
|
|
232
|
-
|
|
239
|
+
if (this.markRecordUnavailable != null) {
|
|
240
|
+
// This will trigger a "resnapshot" of the record.
|
|
241
|
+
// This is not relevant if storeCurrentData is false, since we'll get the full row
|
|
242
|
+
// directly in the replication stream.
|
|
243
|
+
this.markRecordUnavailable(record);
|
|
244
|
+
}
|
|
245
|
+
else {
|
|
246
|
+
// Log to help with debugging if there was a consistency issue
|
|
247
|
+
this.logger.warn(`Cannot find previous record for update on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`);
|
|
248
|
+
}
|
|
233
249
|
}
|
|
234
250
|
}
|
|
235
251
|
else {
|
|
@@ -248,8 +264,8 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
248
264
|
existing_buckets = [];
|
|
249
265
|
existing_lookups = [];
|
|
250
266
|
// Log to help with debugging if there was a consistency issue
|
|
251
|
-
if (this.storeCurrentData) {
|
|
252
|
-
logger.warn(`Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`);
|
|
267
|
+
if (this.storeCurrentData && this.markRecordUnavailable == null) {
|
|
268
|
+
this.logger.warn(`Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`);
|
|
253
269
|
}
|
|
254
270
|
}
|
|
255
271
|
else {
|
|
@@ -333,7 +349,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
333
349
|
table: record.sourceTable.qualifiedName
|
|
334
350
|
}
|
|
335
351
|
});
|
|
336
|
-
logger.error(`Failed to evaluate data query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}`);
|
|
352
|
+
this.logger.error(`Failed to evaluate data query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}`);
|
|
337
353
|
}
|
|
338
354
|
// Save new one
|
|
339
355
|
batch.saveBucketData({
|
|
@@ -362,7 +378,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
362
378
|
table: record.sourceTable.qualifiedName
|
|
363
379
|
}
|
|
364
380
|
});
|
|
365
|
-
logger.error(`Failed to evaluate parameter query on ${record.sourceTable.qualifiedName}.${after.id}: ${error.error}`);
|
|
381
|
+
this.logger.error(`Failed to evaluate parameter query on ${record.sourceTable.qualifiedName}.${after.id}: ${error.error}`);
|
|
366
382
|
}
|
|
367
383
|
batch.saveParameterData({
|
|
368
384
|
op_seq: opSeq,
|
|
@@ -410,7 +426,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
410
426
|
// Likely write conflict caused by concurrent write stream replicating
|
|
411
427
|
}
|
|
412
428
|
else {
|
|
413
|
-
logger.warn('Transaction error', e);
|
|
429
|
+
this.logger.warn('Transaction error', e);
|
|
414
430
|
}
|
|
415
431
|
await timers.setTimeout(Math.random() * 50);
|
|
416
432
|
throw e;
|
|
@@ -426,7 +442,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
426
442
|
await this.withTransaction(async () => {
|
|
427
443
|
flushTry += 1;
|
|
428
444
|
if (flushTry % 10 == 0) {
|
|
429
|
-
logger.info(`${
|
|
445
|
+
this.logger.info(`${description} - try ${flushTry}`);
|
|
430
446
|
}
|
|
431
447
|
if (flushTry > 20 && Date.now() > lastTry) {
|
|
432
448
|
throw new ServiceError(ErrorCode.PSYNC_S1402, 'Max transaction tries exceeded');
|
|
@@ -462,6 +478,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
462
478
|
last_keepalive_ts: new Date()
|
|
463
479
|
}
|
|
464
480
|
}, { session });
|
|
481
|
+
// We don't notify checkpoint here - we don't make any checkpoint updates directly
|
|
465
482
|
});
|
|
466
483
|
}
|
|
467
484
|
async [Symbol.asyncDispose]() {
|
|
@@ -471,16 +488,17 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
471
488
|
lastWaitingLogThottled = 0;
|
|
472
489
|
async commit(lsn, options) {
|
|
473
490
|
const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
|
|
474
|
-
await this.flush();
|
|
491
|
+
await this.flush(options);
|
|
475
492
|
if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) {
|
|
476
493
|
// When re-applying transactions, don't create a new checkpoint until
|
|
477
494
|
// we are past the last transaction.
|
|
478
|
-
logger.info(`Re-applied transaction ${lsn} - skipping checkpoint`);
|
|
495
|
+
this.logger.info(`Re-applied transaction ${lsn} - skipping checkpoint`);
|
|
496
|
+
// Cannot create a checkpoint yet - return false
|
|
479
497
|
return false;
|
|
480
498
|
}
|
|
481
499
|
if (lsn < this.no_checkpoint_before_lsn) {
|
|
482
500
|
if (Date.now() - this.lastWaitingLogThottled > 5_000) {
|
|
483
|
-
logger.info(`Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}. Persisted op: ${this.persisted_op}`);
|
|
501
|
+
this.logger.info(`Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}. Persisted op: ${this.persisted_op}`);
|
|
484
502
|
this.lastWaitingLogThottled = Date.now();
|
|
485
503
|
}
|
|
486
504
|
// Edge case: During initial replication, we have a no_checkpoint_before_lsn set,
|
|
@@ -495,10 +513,13 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
495
513
|
keepalive_op: this.persisted_op == null ? null : String(this.persisted_op)
|
|
496
514
|
}
|
|
497
515
|
}, { session: this.session });
|
|
516
|
+
await this.db.notifyCheckpoint();
|
|
517
|
+
// Cannot create a checkpoint yet - return false
|
|
498
518
|
return false;
|
|
499
519
|
}
|
|
500
520
|
if (!createEmptyCheckpoints && this.persisted_op == null) {
|
|
501
|
-
return
|
|
521
|
+
// Nothing to commit - also return true
|
|
522
|
+
return true;
|
|
502
523
|
}
|
|
503
524
|
const now = new Date();
|
|
504
525
|
const update = {
|
|
@@ -512,11 +533,25 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
512
533
|
if (this.persisted_op != null) {
|
|
513
534
|
update.last_checkpoint = this.persisted_op;
|
|
514
535
|
}
|
|
536
|
+
// Mark relevant write checkpoints as "processed".
|
|
537
|
+
// This makes it easier to identify write checkpoints that are "valid" in order.
|
|
538
|
+
await this.db.write_checkpoints.updateMany({
|
|
539
|
+
processed_at_lsn: null,
|
|
540
|
+
'lsns.1': { $lte: lsn }
|
|
541
|
+
}, {
|
|
542
|
+
$set: {
|
|
543
|
+
processed_at_lsn: lsn
|
|
544
|
+
}
|
|
545
|
+
}, {
|
|
546
|
+
session: this.session
|
|
547
|
+
});
|
|
515
548
|
await this.db.sync_rules.updateOne({
|
|
516
549
|
_id: this.group_id
|
|
517
550
|
}, {
|
|
518
|
-
$set: update
|
|
551
|
+
$set: update,
|
|
552
|
+
$unset: { snapshot_lsn: 1 }
|
|
519
553
|
}, { session: this.session });
|
|
554
|
+
await this.db.notifyCheckpoint();
|
|
520
555
|
this.persisted_op = null;
|
|
521
556
|
this.last_checkpoint_lsn = lsn;
|
|
522
557
|
return true;
|
|
@@ -532,9 +567,19 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
532
567
|
if (this.persisted_op != null) {
|
|
533
568
|
// The commit may have been skipped due to "no_checkpoint_before_lsn".
|
|
534
569
|
// Apply it now if relevant
|
|
535
|
-
logger.info(`Commit due to keepalive at ${lsn} / ${this.persisted_op}`);
|
|
570
|
+
this.logger.info(`Commit due to keepalive at ${lsn} / ${this.persisted_op}`);
|
|
536
571
|
return await this.commit(lsn);
|
|
537
572
|
}
|
|
573
|
+
await this.db.write_checkpoints.updateMany({
|
|
574
|
+
processed_at_lsn: null,
|
|
575
|
+
'lsns.1': { $lte: lsn }
|
|
576
|
+
}, {
|
|
577
|
+
$set: {
|
|
578
|
+
processed_at_lsn: lsn
|
|
579
|
+
}
|
|
580
|
+
}, {
|
|
581
|
+
session: this.session
|
|
582
|
+
});
|
|
538
583
|
await this.db.sync_rules.updateOne({
|
|
539
584
|
_id: this.group_id
|
|
540
585
|
}, {
|
|
@@ -543,11 +588,23 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
543
588
|
snapshot_done: true,
|
|
544
589
|
last_fatal_error: null,
|
|
545
590
|
last_keepalive_ts: new Date()
|
|
546
|
-
}
|
|
591
|
+
},
|
|
592
|
+
$unset: { snapshot_lsn: 1 }
|
|
547
593
|
}, { session: this.session });
|
|
594
|
+
await this.db.notifyCheckpoint();
|
|
548
595
|
this.last_checkpoint_lsn = lsn;
|
|
549
596
|
return true;
|
|
550
597
|
}
|
|
598
|
+
async setSnapshotLsn(lsn) {
|
|
599
|
+
const update = {
|
|
600
|
+
snapshot_lsn: lsn
|
|
601
|
+
};
|
|
602
|
+
await this.db.sync_rules.updateOne({
|
|
603
|
+
_id: this.group_id
|
|
604
|
+
}, {
|
|
605
|
+
$set: update
|
|
606
|
+
}, { session: this.session });
|
|
607
|
+
}
|
|
551
608
|
async save(record) {
|
|
552
609
|
const { after, before, sourceTable, tag } = record;
|
|
553
610
|
for (const event of this.getTableEvents(sourceTable)) {
|
|
@@ -568,7 +625,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
568
625
|
if (!sourceTable.syncData && !sourceTable.syncParameters) {
|
|
569
626
|
return null;
|
|
570
627
|
}
|
|
571
|
-
logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`);
|
|
628
|
+
this.logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`);
|
|
572
629
|
this.batch ??= new OperationBatch();
|
|
573
630
|
this.batch.push(new RecordOperation(record));
|
|
574
631
|
if (this.batch.shouldFlush()) {
|
|
@@ -629,7 +686,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
629
686
|
session: session
|
|
630
687
|
});
|
|
631
688
|
const batch = await cursor.toArray();
|
|
632
|
-
const persistedBatch = new PersistedBatch(this.group_id, 0);
|
|
689
|
+
const persistedBatch = new PersistedBatch(this.group_id, 0, { logger: this.logger });
|
|
633
690
|
for (let value of batch) {
|
|
634
691
|
persistedBatch.saveBucketData({
|
|
635
692
|
op_seq: opSeq,
|
|
@@ -654,6 +711,27 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
654
711
|
}
|
|
655
712
|
return last_op;
|
|
656
713
|
}
|
|
714
|
+
async updateTableProgress(table, progress) {
|
|
715
|
+
const copy = table.clone();
|
|
716
|
+
const snapshotStatus = {
|
|
717
|
+
totalEstimatedCount: progress.totalEstimatedCount ?? copy.snapshotStatus?.totalEstimatedCount ?? 0,
|
|
718
|
+
replicatedCount: progress.replicatedCount ?? copy.snapshotStatus?.replicatedCount ?? 0,
|
|
719
|
+
lastKey: progress.lastKey ?? copy.snapshotStatus?.lastKey ?? null
|
|
720
|
+
};
|
|
721
|
+
copy.snapshotStatus = snapshotStatus;
|
|
722
|
+
await this.withTransaction(async () => {
|
|
723
|
+
await this.db.source_tables.updateOne({ _id: table.id }, {
|
|
724
|
+
$set: {
|
|
725
|
+
snapshot_status: {
|
|
726
|
+
last_key: snapshotStatus.lastKey == null ? null : new bson.Binary(snapshotStatus.lastKey),
|
|
727
|
+
total_estimated_count: snapshotStatus.totalEstimatedCount,
|
|
728
|
+
replicated_count: snapshotStatus.replicatedCount
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
}, { session: this.session });
|
|
732
|
+
});
|
|
733
|
+
return copy;
|
|
734
|
+
}
|
|
657
735
|
async markSnapshotDone(tables, no_checkpoint_before_lsn) {
|
|
658
736
|
const session = this.session;
|
|
659
737
|
const ids = tables.map((table) => table.id);
|
|
@@ -661,6 +739,9 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
661
739
|
await this.db.source_tables.updateMany({ _id: { $in: ids } }, {
|
|
662
740
|
$set: {
|
|
663
741
|
snapshot_done: true
|
|
742
|
+
},
|
|
743
|
+
$unset: {
|
|
744
|
+
snapshot_status: 1
|
|
664
745
|
}
|
|
665
746
|
}, { session });
|
|
666
747
|
if (no_checkpoint_before_lsn > this.no_checkpoint_before_lsn) {
|
|
@@ -676,9 +757,8 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
676
757
|
}
|
|
677
758
|
});
|
|
678
759
|
return tables.map((table) => {
|
|
679
|
-
const copy =
|
|
680
|
-
copy.
|
|
681
|
-
copy.syncParameters = table.syncParameters;
|
|
760
|
+
const copy = table.clone();
|
|
761
|
+
copy.snapshotComplete = true;
|
|
682
762
|
return copy;
|
|
683
763
|
});
|
|
684
764
|
}
|
|
@@ -686,7 +766,7 @@ export class MongoBucketBatch extends BaseObserver {
|
|
|
686
766
|
* Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
|
|
687
767
|
*/
|
|
688
768
|
getTableEvents(table) {
|
|
689
|
-
return this.sync_rules.
|
|
769
|
+
return this.sync_rules.eventDescriptors.filter((evt) => [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table)));
|
|
690
770
|
}
|
|
691
771
|
}
|
|
692
772
|
export function currentBucketKey(b) {
|