@powersync/service-core 1.7.0 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/replication/AbstractReplicator.js +7 -12
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +10 -2
- package/dist/storage/BucketStorage.js +9 -1
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +13 -4
- package/dist/storage/SourceEntity.d.ts +6 -2
- package/dist/storage/SourceTable.d.ts +2 -2
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +6 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/bson.d.ts +4 -1
- package/dist/storage/bson.js +6 -12
- package/dist/storage/bson.js.map +1 -1
- package/dist/sync/sync.js +13 -5
- package/dist/sync/sync.js.map +1 -1
- package/package.json +6 -6
- package/src/replication/AbstractReplicator.ts +7 -11
- package/src/storage/BucketStorage.ts +11 -2
- package/src/storage/BucketStorageFactory.ts +14 -4
- package/src/storage/SourceEntity.ts +6 -2
- package/src/storage/SourceTable.ts +1 -1
- package/src/storage/SyncRulesBucketStorage.ts +9 -0
- package/src/storage/bson.ts +6 -12
- package/src/sync/sync.ts +15 -6
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# @powersync/service-core
|
|
2
2
|
|
|
3
|
+
## 1.7.2
|
|
4
|
+
|
|
5
|
+
## 1.7.1
|
|
6
|
+
|
|
7
|
+
### Patch Changes
|
|
8
|
+
|
|
9
|
+
- b4fe4ae: Upgrade mongodb and bson packages, removing the need for some workarounds.
|
|
10
|
+
- 88ab679: Keep serving current data when restarting replication due to errors.
|
|
11
|
+
- 2f75fd7: Improve handling of some edge cases which could trigger truncating of synced tables.
|
|
12
|
+
- 346382e: Unified compacting options between storage providers.
|
|
13
|
+
- 9b1868d: Fix missing checkpoint complete line for empty sync iterations.
|
|
14
|
+
- Updated dependencies [b4fe4ae]
|
|
15
|
+
- @powersync/service-rsocket-router@0.0.20
|
|
16
|
+
- @powersync/lib-services-framework@0.5.3
|
|
17
|
+
|
|
3
18
|
## 1.7.0
|
|
4
19
|
|
|
5
20
|
### Minor Changes
|
|
@@ -151,23 +151,23 @@ export class AbstractReplicator {
|
|
|
151
151
|
}
|
|
152
152
|
}
|
|
153
153
|
this.replicationJobs = newJobs;
|
|
154
|
-
//
|
|
154
|
+
// Stop any orphaned jobs that no longer have sync rules.
|
|
155
|
+
// Termination happens below
|
|
155
156
|
for (let job of existingJobs.values()) {
|
|
156
157
|
// Old - stop and clean up
|
|
157
158
|
try {
|
|
158
159
|
await job.stop();
|
|
159
|
-
await this.terminateSyncRules(job.storage);
|
|
160
160
|
}
|
|
161
161
|
catch (e) {
|
|
162
162
|
// This will be retried
|
|
163
|
-
this.logger.warn('Failed to
|
|
163
|
+
this.logger.warn('Failed to stop old replication job}', e);
|
|
164
164
|
}
|
|
165
165
|
}
|
|
166
166
|
// Sync rules stopped previously or by a different process.
|
|
167
167
|
const stopped = await this.storage.getStoppedSyncRules();
|
|
168
168
|
for (let syncRules of stopped) {
|
|
169
169
|
try {
|
|
170
|
-
const syncRuleStorage = this.storage.getInstance(syncRules);
|
|
170
|
+
const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
|
|
171
171
|
await this.terminateSyncRules(syncRuleStorage);
|
|
172
172
|
}
|
|
173
173
|
catch (e) {
|
|
@@ -180,14 +180,9 @@ export class AbstractReplicator {
|
|
|
180
180
|
}
|
|
181
181
|
async terminateSyncRules(syncRuleStorage) {
|
|
182
182
|
this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
|
|
187
|
-
}
|
|
188
|
-
catch (e) {
|
|
189
|
-
this.logger.warn(`Failed clean up replication config for sync rules: ${syncRuleStorage.group_id}`, e);
|
|
190
|
-
}
|
|
183
|
+
await this.cleanUp(syncRuleStorage);
|
|
184
|
+
await syncRuleStorage.terminate();
|
|
185
|
+
this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
|
|
191
186
|
}
|
|
192
187
|
}
|
|
193
188
|
//# sourceMappingURL=AbstractReplicator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractReplicator.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAStC,YAAY;AACZ,MAAM,aAAa,GAAG,WAAc,GAAG,IAAI,CAAC;AAiB5C;;;;GAIG;AACH,MAAM,OAAgB,kBAAkB;IAaR;IAZpB,MAAM,CAAiB;IACjC;;;;OAIG;IACK,eAAe,GAAG,IAAI,GAAG,EAAa,CAAC;IACvC,OAAO,GAAG,KAAK,CAAC;IAExB,wDAAwD;IAChD,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IAEnC,YAA8B,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QAC9D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAUD,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC;IACxD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,CAAC,CAAC,CAAC;YAC5D,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACvC,UAAU,CAAC,GAAG,EAAE;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5B,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC;QAEpD,IAAI,cAAc,GAAwC,SAAS,CAAC;QACpE,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACtC,IAAI,CAAC;gBACH,kDAAkD;gBAClD,2GAA2G;gBAE3G,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACrD,OAAO,EAAE,SAAS;oBAClB,IAAI,EAAE,IAAI;oBACV,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,WAAW;iBAC5C,CAAC,CAAC;gBACH,IAAI,IAAI,EAAE,CAAC;oBACT,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,4BAA4B;gBAC5B,4EAA4E;gBAC5E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;gBACvE,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC;gBACxD,+CAA+C;gBAC/C,cAAc,GAAG,SAAS,CAAC;gBAE3B,gEAAgE;gBAChE,6EAA6E;gBAC7E,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC5B,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,aAAa,EAAE,CAAC;wBACzC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;4BACtD,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC;wBAC9B,CAAC;wBAED,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAuD;QAC3E,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,EAAE,eAAe,CAAC;QAE9C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAY,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAC;QACrC,KAAK,IAAI,SAAS,IAAI,oBAAoB,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC1C,YAAY;gBACZ,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,WAAW,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;gBAChD,8BAA8B;gBAC9B,gEAAgE;gBAChE,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,IAAI,CAAC;oBACH,IAAI,IAA6B,CAAC;oBAClC,IAAI,cAAc,EAAE,aAAa,IAAI,SAAS,CAAC,EAAE,EAAE,CAAC;wBAClD,IAAI,GAAG,cAAc,CAAC;oBACxB,CAAC;yBAAM,CAAC;wBACN,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;oBAChC,CAAC;oBACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;oBACpD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;wBAC5B,IAAI,EAAE,IAAI;wBACV,OAAO,EAAE,OAAO;qBACjB,CAAC,CAAC;oBAEH,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBAClC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,qCAAqC;oBACrC,uDAAuD;oBACvD,gDAAgD;oBAChD,gFAAgF;oBAChF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QAE/B,
|
|
1
|
+
{"version":3,"file":"AbstractReplicator.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAStC,YAAY;AACZ,MAAM,aAAa,GAAG,WAAc,GAAG,IAAI,CAAC;AAiB5C;;;;GAIG;AACH,MAAM,OAAgB,kBAAkB;IAaR;IAZpB,MAAM,CAAiB;IACjC;;;;OAIG;IACK,eAAe,GAAG,IAAI,GAAG,EAAa,CAAC;IACvC,OAAO,GAAG,KAAK,CAAC;IAExB,wDAAwD;IAChD,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IAEnC,YAA8B,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QAC9D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAUD,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC;IACxD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,CAAC,CAAC,CAAC;YAC5D,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACvC,UAAU,CAAC,GAAG,EAAE;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5B,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC;QAEpD,IAAI,cAAc,GAAwC,SAAS,CAAC;QACpE,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACtC,IAAI,CAAC;gBACH,kDAAkD;gBAClD,2GAA2G;gBAE3G,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACrD,OAAO,EAAE,SAAS;oBAClB,IAAI,EAAE,IAAI;oBACV,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,WAAW;iBAC5C,CAAC,CAAC;gBACH,IAAI,IAAI,EAAE,CAAC;oBACT,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,4BAA4B;gBAC5B,4EAA4E;gBAC5E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;gBACvE,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC;gBACxD,+CAA+C;gBAC/C,cAAc,GAAG,SAAS,CAAC;gBAE3B,gEAAgE;gBAChE,6EAA6E;gBAC7E,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC5B,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,aAAa,EAAE,CAAC;wBACzC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;4BACtD,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC;wBAC9B,CAAC;wBAED,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAuD;QAC3E,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,EAAE,eAAe,CAAC;QAE9C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAY,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAC;QACrC,KAAK,IAAI,SAAS,IAAI,oBAAoB,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC1C,YAAY;gBACZ,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,WAAW,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;gBAChD,8BAA8B;gBAC9B,gEAAgE;gBAChE,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,IAAI,CAAC;oBACH,IAAI,IAA6B,CAAC;oBAClC,IAAI,cAAc,EAAE,aAAa,IAAI,SAAS,CAAC,EAAE,EAAE,CAAC;wBAClD,IAAI,GAAG,cAAc,CAAC;oBACxB,CAAC;yBAAM,CAAC;wBACN,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;oBAChC,CAAC;oBACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;oBACpD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;wBAC5B,IAAI,EAAE,IAAI;wBACV,OAAO,EAAE,OAAO;qBACjB,CAAC,CAAC;oBAEH,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBAClC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,qCAAqC;oBACrC,uDAAuD;oBACvD,gDAAgD;oBAChD,gFAAgF;oBAChF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QAE/B,yDAAyD;QACzD,4BAA4B;QAC5B,KAAK,IAAI,GAAG,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,0BAA0B;YAC1B,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,uBAAuB;gBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,2DAA2D;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;QACzD,KAAK,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1F,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;IACH,CAAC;IAES,WAAW,CAAC,UAAkB;QACtC,OAAO,GAAG,IAAI,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,eAA+C;QAChF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,eAAe,CAAC,QAAQ,KAAK,CAAC,CAAC;QAC3E,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QACpC,MAAM,eAAe,CAAC,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;IACtF,CAAC;CAGF"}
|
|
@@ -10,7 +10,7 @@ export declare enum SyncRuleState {
|
|
|
10
10
|
/**
|
|
11
11
|
* Sync rule processing is done, and can be used for sync.
|
|
12
12
|
*
|
|
13
|
-
* Only one set of sync rules should be in ACTIVE state.
|
|
13
|
+
* Only one set of sync rules should be in ACTIVE or ERRORED state.
|
|
14
14
|
*/
|
|
15
15
|
ACTIVE = "ACTIVE",
|
|
16
16
|
/**
|
|
@@ -22,7 +22,15 @@ export declare enum SyncRuleState {
|
|
|
22
22
|
* After sync rules have been stopped, the data needs to be
|
|
23
23
|
* deleted. Once deleted, the state is TERMINATED.
|
|
24
24
|
*/
|
|
25
|
-
TERMINATED = "TERMINATED"
|
|
25
|
+
TERMINATED = "TERMINATED",
|
|
26
|
+
/**
|
|
27
|
+
* Sync rules has run into a permanent replication error. It
|
|
28
|
+
* is still the "active" sync rules for syncing to users,
|
|
29
|
+
* but should not replicate anymore.
|
|
30
|
+
*
|
|
31
|
+
* It will transition to STOP when a new sync rules is activated.
|
|
32
|
+
*/
|
|
33
|
+
ERRORED = "ERRORED"
|
|
26
34
|
}
|
|
27
35
|
export declare const DEFAULT_DOCUMENT_BATCH_LIMIT = 1000;
|
|
28
36
|
export declare const DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES: number;
|
|
@@ -10,7 +10,7 @@ export var SyncRuleState;
|
|
|
10
10
|
/**
|
|
11
11
|
* Sync rule processing is done, and can be used for sync.
|
|
12
12
|
*
|
|
13
|
-
* Only one set of sync rules should be in ACTIVE state.
|
|
13
|
+
* Only one set of sync rules should be in ACTIVE or ERRORED state.
|
|
14
14
|
*/
|
|
15
15
|
SyncRuleState["ACTIVE"] = "ACTIVE";
|
|
16
16
|
/**
|
|
@@ -23,6 +23,14 @@ export var SyncRuleState;
|
|
|
23
23
|
* deleted. Once deleted, the state is TERMINATED.
|
|
24
24
|
*/
|
|
25
25
|
SyncRuleState["TERMINATED"] = "TERMINATED";
|
|
26
|
+
/**
|
|
27
|
+
* Sync rules has run into a permanent replication error. It
|
|
28
|
+
* is still the "active" sync rules for syncing to users,
|
|
29
|
+
* but should not replicate anymore.
|
|
30
|
+
*
|
|
31
|
+
* It will transition to STOP when a new sync rules is activated.
|
|
32
|
+
*/
|
|
33
|
+
SyncRuleState["ERRORED"] = "ERRORED";
|
|
26
34
|
})(SyncRuleState || (SyncRuleState = {}));
|
|
27
35
|
export const DEFAULT_DOCUMENT_BATCH_LIMIT = 1000;
|
|
28
36
|
export const DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES = 1 * 1024 * 1024;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BucketStorage.js","sourceRoot":"","sources":["../../src/storage/BucketStorage.ts"],"names":[],"mappings":"AAEA,MAAM,CAAN,IAAY,
|
|
1
|
+
{"version":3,"file":"BucketStorage.js","sourceRoot":"","sources":["../../src/storage/BucketStorage.ts"],"names":[],"mappings":"AAEA,MAAM,CAAN,IAAY,aAkCX;AAlCD,WAAY,aAAa;IACvB;;;;;OAKG;IACH,0CAAyB,CAAA;IAEzB;;;;OAIG;IACH,kCAAiB,CAAA;IACjB;;;OAGG;IACH,8BAAa,CAAA;IACb;;;OAGG;IACH,0CAAyB,CAAA;IAEzB;;;;;;OAMG;IACH,oCAAmB,CAAA;AACrB,CAAC,EAlCW,aAAa,KAAb,aAAa,QAkCxB;AAED,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AACjD,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAElE,MAAM,UAAU,UAAU,CAAC,MAA0B,EAAE,SAA6B;IAClF,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACtC,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -23,7 +23,7 @@ export interface BucketStorageFactory extends ObserverClient<BucketStorageFactor
|
|
|
23
23
|
/**
|
|
24
24
|
* Get a storage instance to query sync data for specific sync rules.
|
|
25
25
|
*/
|
|
26
|
-
getInstance(
|
|
26
|
+
getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
|
|
27
27
|
/**
|
|
28
28
|
* Deploy new sync rules.
|
|
29
29
|
*/
|
|
@@ -37,10 +37,8 @@ export interface BucketStorageFactory extends ObserverClient<BucketStorageFactor
|
|
|
37
37
|
* the latest ones.
|
|
38
38
|
*
|
|
39
39
|
* Replication should be restarted after this.
|
|
40
|
-
*
|
|
41
|
-
* @param slot_name The removed slot
|
|
42
40
|
*/
|
|
43
|
-
|
|
41
|
+
restartReplication(sync_rules_group_id: number): Promise<void>;
|
|
44
42
|
/**
|
|
45
43
|
* Get the sync rules used for querying.
|
|
46
44
|
*/
|
|
@@ -107,6 +105,17 @@ export interface UpdateSyncRulesOptions {
|
|
|
107
105
|
lock?: boolean;
|
|
108
106
|
validate?: boolean;
|
|
109
107
|
}
|
|
108
|
+
export interface GetIntanceOptions {
|
|
109
|
+
/**
|
|
110
|
+
* Set to true to skip trigger any events for creating the instance.
|
|
111
|
+
*
|
|
112
|
+
* This is used when creating the instance only for clearing data.
|
|
113
|
+
*
|
|
114
|
+
* When this is used, note that some functionality such as write checkpoint mode
|
|
115
|
+
* may not be configured correctly.
|
|
116
|
+
*/
|
|
117
|
+
skipLifecycleHooks?: boolean;
|
|
118
|
+
}
|
|
110
119
|
export interface BucketStorageSystemIdentifier {
|
|
111
120
|
/**
|
|
112
121
|
* A unique identifier for the system used for storage.
|
|
@@ -11,9 +11,13 @@ export interface ColumnDescriptor {
|
|
|
11
11
|
}
|
|
12
12
|
export interface SourceEntityDescriptor {
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
14
|
+
* The internal id of the data source structure in the database.
|
|
15
|
+
*
|
|
16
|
+
* If undefined, the schema and name are used as the identifier.
|
|
17
|
+
*
|
|
18
|
+
* If specified, this is specifically used to detect renames.
|
|
15
19
|
*/
|
|
16
|
-
objectId: number | string;
|
|
20
|
+
objectId: number | string | undefined;
|
|
17
21
|
schema: string;
|
|
18
22
|
name: string;
|
|
19
23
|
replicationColumns: ColumnDescriptor[];
|
|
@@ -2,7 +2,7 @@ import { ColumnDescriptor } from './SourceEntity.js';
|
|
|
2
2
|
export declare class SourceTable {
|
|
3
3
|
readonly id: any;
|
|
4
4
|
readonly connectionTag: string;
|
|
5
|
-
readonly objectId: number | string;
|
|
5
|
+
readonly objectId: number | string | undefined;
|
|
6
6
|
readonly schema: string;
|
|
7
7
|
readonly table: string;
|
|
8
8
|
readonly replicaIdColumns: ColumnDescriptor[];
|
|
@@ -32,7 +32,7 @@ export declare class SourceTable {
|
|
|
32
32
|
* Defaults to true for tests.
|
|
33
33
|
*/
|
|
34
34
|
syncEvent: boolean;
|
|
35
|
-
constructor(id: any, connectionTag: string, objectId: number | string, schema: string, table: string, replicaIdColumns: ColumnDescriptor[], snapshotComplete: boolean);
|
|
35
|
+
constructor(id: any, connectionTag: string, objectId: number | string | undefined, schema: string, table: string, replicaIdColumns: ColumnDescriptor[], snapshotComplete: boolean);
|
|
36
36
|
get hasReplicaIdentity(): boolean;
|
|
37
37
|
/**
|
|
38
38
|
* Use for postgres only.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SourceTable.js","sourceRoot":"","sources":["../../src/storage/SourceTable.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAG9C,MAAM,OAAO,WAAW;IA+BJ;IACA;IACA;IACA;IACA;IAEA;IACA;IArClB,MAAM,CAAU,WAAW,GAAG,WAAW,CAAC;IAE1C;;;;;;OAMG;IACI,QAAQ,GAAG,IAAI,CAAC;IAEvB;;;;;;OAMG;IACI,cAAc,GAAG,IAAI,CAAC;IAE7B;;;;;;OAMG;IACI,SAAS,GAAG,IAAI,CAAC;IAExB,YACkB,EAAO,EACP,aAAqB,EACrB,
|
|
1
|
+
{"version":3,"file":"SourceTable.js","sourceRoot":"","sources":["../../src/storage/SourceTable.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAG9C,MAAM,OAAO,WAAW;IA+BJ;IACA;IACA;IACA;IACA;IAEA;IACA;IArClB,MAAM,CAAU,WAAW,GAAG,WAAW,CAAC;IAE1C;;;;;;OAMG;IACI,QAAQ,GAAG,IAAI,CAAC;IAEvB;;;;;;OAMG;IACI,cAAc,GAAG,IAAI,CAAC;IAE7B;;;;;;OAMG;IACI,SAAS,GAAG,IAAI,CAAC;IAExB,YACkB,EAAO,EACP,aAAqB,EACrB,QAAqC,EACrC,MAAc,EACd,KAAa,EAEb,gBAAoC,EACpC,gBAAyB;QAPzB,OAAE,GAAF,EAAE,CAAK;QACP,kBAAa,GAAb,aAAa,CAAQ;QACrB,aAAQ,GAAR,QAAQ,CAA6B;QACrC,WAAM,GAAN,MAAM,CAAQ;QACd,UAAK,GAAL,KAAK,CAAQ;QAEb,qBAAgB,GAAhB,gBAAgB,CAAoB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAS;IACxC,CAAC;IAEJ,IAAI,kBAAkB;QACpB,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,IAAI,iBAAiB;QACnB,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;IACtF,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,SAAS,CAAC;IAChE,CAAC"}
|
|
@@ -142,6 +142,12 @@ export interface CompactOptions {
|
|
|
142
142
|
* These can be individual bucket names, or bucket definition names.
|
|
143
143
|
*/
|
|
144
144
|
compactBuckets?: string[];
|
|
145
|
+
/** Minimum of 2 */
|
|
146
|
+
clearBatchLimit?: number;
|
|
147
|
+
/** Minimum of 1 */
|
|
148
|
+
moveBatchLimit?: number;
|
|
149
|
+
/** Minimum of 1 */
|
|
150
|
+
moveBatchQueryLimit?: number;
|
|
145
151
|
}
|
|
146
152
|
export interface TerminateOptions {
|
|
147
153
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"AAmQA,MAAM,CAAC,MAAM,yBAAyB,GAAsB;IAC1D,kBAAkB,EAAE,EAAE;IACtB,qBAAqB,EAAE,IAAI;IAC3B,iCAAiC,EAAE,EAAE;IACrC,0BAA0B,EAAE,IAAI;CACjC,CAAC"}
|
package/dist/storage/bson.d.ts
CHANGED
|
@@ -7,7 +7,7 @@ type NodeBuffer = Buffer<ArrayBuffer>;
|
|
|
7
7
|
*/
|
|
8
8
|
export declare const BSON_DESERIALIZE_INTERNAL_OPTIONS: bson.DeserializeOptions;
|
|
9
9
|
/**
|
|
10
|
-
* Use for data from external sources.
|
|
10
|
+
* Use for data from external sources, which could contain arbitrary fields.
|
|
11
11
|
*/
|
|
12
12
|
export declare const BSON_DESERIALIZE_DATA_OPTIONS: bson.DeserializeOptions;
|
|
13
13
|
/**
|
|
@@ -25,6 +25,9 @@ export declare const getLookupBucketDefinitionName: (lookup: bson.Binary) => str
|
|
|
25
25
|
export declare const isUUID: (value: any) => value is bson.UUID;
|
|
26
26
|
export declare const serializeReplicaId: (id: ReplicaId) => NodeBuffer;
|
|
27
27
|
export declare const deserializeReplicaId: (id: Buffer) => ReplicaId;
|
|
28
|
+
/**
|
|
29
|
+
* Deserialize BSON - can be used for BSON containing arbitrary user data.
|
|
30
|
+
*/
|
|
28
31
|
export declare const deserializeBson: (buffer: Uint8Array) => bson.Document;
|
|
29
32
|
export declare const serializeBson: (document: any) => NodeBuffer;
|
|
30
33
|
/**
|
package/dist/storage/bson.js
CHANGED
|
@@ -7,11 +7,10 @@ export const BSON_DESERIALIZE_INTERNAL_OPTIONS = {
|
|
|
7
7
|
useBigInt64: true
|
|
8
8
|
};
|
|
9
9
|
/**
|
|
10
|
-
* Use for data from external sources.
|
|
10
|
+
* Use for data from external sources, which could contain arbitrary fields.
|
|
11
11
|
*/
|
|
12
12
|
export const BSON_DESERIALIZE_DATA_OPTIONS = {
|
|
13
|
-
|
|
14
|
-
useBigInt64: false
|
|
13
|
+
useBigInt64: true
|
|
15
14
|
};
|
|
16
15
|
/**
|
|
17
16
|
* Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
|
|
@@ -54,16 +53,11 @@ export const deserializeReplicaId = (id) => {
|
|
|
54
53
|
const deserialized = deserializeBson(id);
|
|
55
54
|
return deserialized.id;
|
|
56
55
|
};
|
|
56
|
+
/**
|
|
57
|
+
* Deserialize BSON - can be used for BSON containing arbitrary user data.
|
|
58
|
+
*/
|
|
57
59
|
export const deserializeBson = (buffer) => {
|
|
58
|
-
|
|
59
|
-
// Temporary workaround due to https://jira.mongodb.org/browse/NODE-6764
|
|
60
|
-
for (let key in doc) {
|
|
61
|
-
const value = doc[key];
|
|
62
|
-
if (value instanceof bson.Long) {
|
|
63
|
-
doc[key] = value.toBigInt();
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
return doc;
|
|
60
|
+
return bson.deserialize(buffer, BSON_DESERIALIZE_DATA_OPTIONS);
|
|
67
61
|
};
|
|
68
62
|
export const serializeBson = (document) => {
|
|
69
63
|
return bson.serialize(document);
|
package/dist/storage/bson.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bson.js","sourceRoot":"","sources":["../../src/storage/bson.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAO7B;;GAEG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAA4B;IACxE,6BAA6B;IAC7B,WAAW,EAAE,IAAI;CAClB,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAA4B;IACpE,
|
|
1
|
+
{"version":3,"file":"bson.js","sourceRoot":"","sources":["../../src/storage/bson.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAO7B;;GAEG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAA4B;IACxE,6BAA6B;IAC7B,WAAW,EAAE,IAAI;CAClB,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAA4B;IACpE,WAAW,EAAE,IAAI;CAClB,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,MAAyB,EAAc,EAAE;IAC7E,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACtC,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,CAAe,CAAC;AACzD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAyB,EAAE,EAAE;IAC3D,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC;AACxD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,MAAmB,EAAE,EAAE;IACnE,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,iCAAiC,CAAC,CAAC,CAAsB,CAAC;IACzG,OAAO,MAAM,CAAC,CAAC,CAAW,CAAC;AAC7B,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,KAAU,EAAsB,EAAE;IACvD,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,KAAkB,CAAC;IAChC,OAAO,IAAI,CAAC,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;AACjF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,EAAa,EAAc,EAAE;IAC9D,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAe,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,EAAU,EAAa,EAAE;IAC5D,MAAM,YAAY,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;IACzC,OAAO,YAAY,CAAC,EAAE,CAAC;AACzB,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAkB,EAAiB,EAAE;IACnE,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,6BAA6B,CAAC,CAAC;AACjE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,QAAa,EAAc,EAAE;IACzD,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAe,CAAC;AAChD,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAY,EAAE,CAAY,EAAE,EAAE;IAC5D,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,CAAC,IAAI,QAAQ,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;SAAM,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,iEAAiE;QACjE,OAAO,kBAAkB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC,CAAC"}
|
package/dist/sync/sync.js
CHANGED
|
@@ -97,6 +97,14 @@ async function* streamResponseInner(bucketStorage, syncRules, params, syncParams
|
|
|
97
97
|
const bucketsByPriority = [...Map.groupBy(bucketsToFetch, (bucket) => bucket.priority).entries()];
|
|
98
98
|
bucketsByPriority.sort((a, b) => a[0] - b[0]); // Sort from high to lower priorities
|
|
99
99
|
const lowestPriority = bucketsByPriority.at(-1)?.[0];
|
|
100
|
+
// Ensure that we have at least one priority batch: After sending the checkpoint line, clients expect to
|
|
101
|
+
// receive a sync complete message after the synchronization is done (which happens in the last
|
|
102
|
+
// bucketDataInBatches iteration). Without any batch, the line is missing and clients might not complete their
|
|
103
|
+
// sync properly.
|
|
104
|
+
const priorityBatches = bucketsByPriority;
|
|
105
|
+
if (priorityBatches.length == 0) {
|
|
106
|
+
priorityBatches.push([null, []]);
|
|
107
|
+
}
|
|
100
108
|
function maybeRaceForNewCheckpoint() {
|
|
101
109
|
if (syncedOperations >= 1000 && nextCheckpointPromise === undefined) {
|
|
102
110
|
nextCheckpointPromise = (async () => {
|
|
@@ -119,7 +127,7 @@ async function* streamResponseInner(bucketStorage, syncRules, params, syncParams
|
|
|
119
127
|
}
|
|
120
128
|
// This incrementally updates dataBuckets with each individual bucket position.
|
|
121
129
|
// At the end of this, we can be sure that all buckets have data up to the checkpoint.
|
|
122
|
-
for (const [priority, buckets] of
|
|
130
|
+
for (const [priority, buckets] of priorityBatches) {
|
|
123
131
|
const isLast = priority === lowestPriority;
|
|
124
132
|
if (abortCheckpointSignal.aborted) {
|
|
125
133
|
break;
|
|
@@ -135,9 +143,9 @@ async function* streamResponseInner(bucketStorage, syncRules, params, syncParams
|
|
|
135
143
|
abort_connection: signal,
|
|
136
144
|
abort_batch: abortCheckpointSignal,
|
|
137
145
|
user_id: syncParams.user_id,
|
|
138
|
-
// Passing
|
|
139
|
-
// sync complete message.
|
|
140
|
-
forPriority: !isLast ? priority :
|
|
146
|
+
// Passing null here will emit a full sync complete message at the end. If we pass a priority, we'll emit a partial
|
|
147
|
+
// sync complete message instead.
|
|
148
|
+
forPriority: !isLast ? priority : null
|
|
141
149
|
});
|
|
142
150
|
}
|
|
143
151
|
if (!abortCheckpointSignal.aborted) {
|
|
@@ -270,7 +278,7 @@ async function* bucketDataBatch(request) {
|
|
|
270
278
|
yield { data: null, done: true };
|
|
271
279
|
}
|
|
272
280
|
else {
|
|
273
|
-
if (request.forPriority
|
|
281
|
+
if (request.forPriority != null) {
|
|
274
282
|
const line = {
|
|
275
283
|
partial_checkpoint_complete: {
|
|
276
284
|
last_op_id: checkpoint,
|
package/dist/sync/sync.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAE,yBAAyB,EAAE,cAAc,EAAE,WAAW,EAAsB,MAAM,WAAW,CAAC;AAEvG;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAElC;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAErC,MAAM,aAAa,GAAG,WAAW,CAC/B,IAAI,SAAS,CAAC,sBAAsB,CAAC,EACrC,qBAAqB,EACrB,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAC5C,CAAC;AAiBF,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC7G,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC7G,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,KAAK,CAAC,CAAC,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,UAAU,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;YAAS,CAAC;QACT,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAOD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,aAA6C,EAC7C,SAAuB,EACvB,MAAiC,EACjC,UAA6B,EAC7B,OAAuB,EACvB,MAAmB;IAEnB,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEzC,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,gBAAgB,CAAC,OAAiB,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAEhH,MAAM,aAAa,GAAG,IAAI,mBAAmB,CAAC;QAC5C,aAAa;QACb,SAAS;QACT,UAAU;QACV,sBAAsB,EAAE,MAAM,CAAC,OAAO;KACvC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,aAAa,CAAC,oBAAoB,CAAC;QAChD,OAAO,EAAE,gBAAgB;QACzB,MAAM;KACP,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IAEtD,IAAI,CAAC;QACH,IAAI,qBAES,CAAC;QAEd,GAAG,CAAC;YACF,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBAC3B,oFAAoF;gBACpF,sCAAsC;gBACtC,qBAAqB,GAAG,cAAc,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;YAChE,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC;YACzC,qBAAqB,GAAG,SAAS,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,MAAM,IAAI,CAAC,MAAM,CAAC;YACpB,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBACpB,MAAM;YACR,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3E,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,oBAAoB;gBACpB,SAAS;YACX,CAAC;YAED,MAAM,EAAE,cAAc,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC;YAEhD,MAAM,cAAc,CAAC;YACrB,8GAA8G;YAC9G,kHAAkH;YAClH,iHAAiH;YACjH,sBAAsB;YACtB,MAAM,yBAAyB,GAAG,IAAI,eAAe,EAAE,CAAC;YACxD,IAAI,gBAAgB,GAAG,CAAC,CAAC;YAEzB,MAAM,qBAAqB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,yBAAyB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;YAE1F,MAAM,iBAAiB,GAAG,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAClG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qCAAqC;YACpF,MAAM,cAAc,GAAG,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAErD,SAAS,yBAAyB;gBAChC,IAAI,gBAAgB,IAAI,IAAI,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;oBACpE,qBAAqB,GAAG,CAAC,KAAK,IAAI,EAAE;wBAClC,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;wBACzD,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;4BAC9B,yBAAyB,CAAC,KAAK,EAAE,CAAC;wBACpC,CAAC;6BAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;4BAC5B,oGAAoG;4BACpG,yBAAyB,CAAC,KAAK,EAAE,CAAC;wBACpC,CAAC;wBAED,OAAO,IAAI,CAAC;oBACd,CAAC,CAAC,EAAE,CAAC;gBACP,CAAC;YACH,CAAC;YAED,SAAS,kBAAkB,CAAC,UAAkB;gBAC5C,gBAAgB,IAAI,UAAU,CAAC;gBAC/B,OAAO,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;gBACxC,yBAAyB,EAAE,CAAC;YAC9B,CAAC;YAED,+EAA+E;YAC/E,sFAAsF;YACtF,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAE,yBAAyB,EAAE,cAAc,EAAE,WAAW,EAAsB,MAAM,WAAW,CAAC;AAEvG;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAElC;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAErC,MAAM,aAAa,GAAG,WAAW,CAC/B,IAAI,SAAS,CAAC,sBAAsB,CAAC,EACrC,qBAAqB,EACrB,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAC5C,CAAC;AAiBF,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC7G,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC7G,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,KAAK,CAAC,CAAC,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,UAAU,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;YAAS,CAAC;QACT,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAOD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,aAA6C,EAC7C,SAAuB,EACvB,MAAiC,EACjC,UAA6B,EAC7B,OAAuB,EACvB,MAAmB;IAEnB,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEzC,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,gBAAgB,CAAC,OAAiB,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAEhH,MAAM,aAAa,GAAG,IAAI,mBAAmB,CAAC;QAC5C,aAAa;QACb,SAAS;QACT,UAAU;QACV,sBAAsB,EAAE,MAAM,CAAC,OAAO;KACvC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,aAAa,CAAC,oBAAoB,CAAC;QAChD,OAAO,EAAE,gBAAgB;QACzB,MAAM;KACP,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IAEtD,IAAI,CAAC;QACH,IAAI,qBAES,CAAC;QAEd,GAAG,CAAC;YACF,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBAC3B,oFAAoF;gBACpF,sCAAsC;gBACtC,qBAAqB,GAAG,cAAc,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;YAChE,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC;YACzC,qBAAqB,GAAG,SAAS,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,MAAM,IAAI,CAAC,MAAM,CAAC;YACpB,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBACpB,MAAM;YACR,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3E,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,oBAAoB;gBACpB,SAAS;YACX,CAAC;YAED,MAAM,EAAE,cAAc,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC;YAEhD,MAAM,cAAc,CAAC;YACrB,8GAA8G;YAC9G,kHAAkH;YAClH,iHAAiH;YACjH,sBAAsB;YACtB,MAAM,yBAAyB,GAAG,IAAI,eAAe,EAAE,CAAC;YACxD,IAAI,gBAAgB,GAAG,CAAC,CAAC;YAEzB,MAAM,qBAAqB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,yBAAyB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;YAE1F,MAAM,iBAAiB,GAAG,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAClG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qCAAqC;YACpF,MAAM,cAAc,GAAG,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAErD,wGAAwG;YACxG,+FAA+F;YAC/F,8GAA8G;YAC9G,iBAAiB;YACjB,MAAM,eAAe,GAAmD,iBAAiB,CAAC;YAC1F,IAAI,eAAe,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAChC,eAAe,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;YACnC,CAAC;YAED,SAAS,yBAAyB;gBAChC,IAAI,gBAAgB,IAAI,IAAI,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;oBACpE,qBAAqB,GAAG,CAAC,KAAK,IAAI,EAAE;wBAClC,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;wBACzD,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;4BAC9B,yBAAyB,CAAC,KAAK,EAAE,CAAC;wBACpC,CAAC;6BAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;4BAC5B,oGAAoG;4BACpG,yBAAyB,CAAC,KAAK,EAAE,CAAC;wBACpC,CAAC;wBAED,OAAO,IAAI,CAAC;oBACd,CAAC,CAAC,EAAE,CAAC;gBACP,CAAC;YACH,CAAC;YAED,SAAS,kBAAkB,CAAC,UAAkB;gBAC5C,gBAAgB,IAAI,UAAU,CAAC;gBAC/B,OAAO,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;gBACxC,yBAAyB,EAAE,CAAC;YAC9B,CAAC;YAED,+EAA+E;YAC/E,sFAAsF;YACtF,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC;gBAClD,MAAM,MAAM,GAAG,QAAQ,KAAK,cAAc,CAAC;gBAC3C,IAAI,qBAAqB,CAAC,OAAO,EAAE,CAAC;oBAClC,MAAM;gBACR,CAAC;gBAED,KAAK,CAAC,CAAC,mBAAmB,CAAC;oBACzB,aAAa,EAAE,aAAa;oBAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU;oBAC5C,cAAc,EAAE,OAAO;oBACvB,aAAa;oBACb,QAAQ;oBACR,WAAW;oBACX,UAAU,EAAE,kBAAkB;oBAC9B,gBAAgB,EAAE,MAAM;oBACxB,WAAW,EAAE,qBAAqB;oBAClC,OAAO,EAAE,UAAU,CAAC,OAAO;oBAC3B,mHAAmH;oBACnH,iCAAiC;oBACjC,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;iBACvC,CAAC,CAAC;YACL,CAAC;YAED,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,CAAC;gBACnC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE;IAC5B,CAAC;YAAS,CAAC;QACT,MAAM,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC;IAClC,CAAC;AACH,CAAC;AAsBD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAC,OAA0B;IAC5D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/C,wEAAwE;QACxE,6EAA6E;QAC7E,4GAA4G;QAC5G,gBAAgB;QAChB,gBAAgB;QAChB,qBAAqB;QACrB,MAAM;QACN,WAAW;QACX,IAAI;QACJ,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC;YACH,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM;gBACR,CAAC;qBAAM,CAAC;oBACN,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;oBAC7B,MAAM,IAAI,CAAC;oBACX,IAAI,IAAI,EAAE,CAAC;wBACT,MAAM,GAAG,IAAI,CAAC;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;AACH,CAAC;AAOD;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAA0B;IACxD,MAAM,EACJ,aAAa,EAAE,OAAO,EACtB,UAAU,EACV,cAAc,EACd,aAAa,EACb,QAAQ,EACR,WAAW,EACX,gBAAgB,EAChB,WAAW,EACX,UAAU,EACX,GAAG,OAAO,CAAC;IAEZ,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,qBAAqB,GAAG,KAAK,CAAC;IAElC,IAAI,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,kDAAkD,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IAChG,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO;IACT,CAAC;IAED,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,QAAQ,CAAC;IAClC,IAAI,CAAC;QACH,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,6DAA6D;YAC7D,qBAAqB;YACrB,MAAM,CAAC,IAAI,CAAC,mCAAmC,KAAK,GAAG,CAAC,EAAE,EAAE;gBAC1D,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,eAAe,EAAE,KAAK,GAAG,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC;QACD,kGAAkG;QAClG,iDAAiD;QACjD,MAAM,eAAe,GAAG,aAAa,CAAC,0BAA0B,CAAC,cAAc,CAAC,CAAC;QACjF,MAAM,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAE5E,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,WAAW,EAAE,CAAC;YACrD,qDAAqD;YACrD,IAAI,gBAAgB,CAAC,OAAO,EAAE,CAAC;gBAC7B,OAAO;YACT,CAAC;YACD,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACf,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;YACD,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,GAAG,YAAY,EAAE,CAAC;gBAChD,qBAAqB,GAAG,IAAI,CAAC;YAC/B,CAAC;YACD,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACvB,SAAS;YACX,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAE7C,IAAI,SAAc,CAAC;YACnB,IAAI,WAAW,EAAE,CAAC;gBAChB,wEAAwE;gBACxE,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1B,CAAC;iBAAM,IAAI,QAAQ,EAAE,CAAC;gBACpB;;mBAEG;gBACH,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,CAAC;iBACR,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,8EAA8E;gBAC9E,4BAA4B;gBAC5B,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC;iBACjC,CAAC;gBACF,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YAC1C,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACvC,IAAI,SAAS,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBAC9B,0EAA0E;gBAC1E,8CAA8C;gBAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACpC,CAAC;YACD,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE1B,aAAa,CAAC,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEvG,6EAA6E;YAC7E,gBAAgB;YAChB,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;gBACxB,OAAO;YACT,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,IAAI,qBAAqB,EAAE,CAAC;gBAC1B,gDAAgD;gBAChD,wDAAwD;gBACxD,kEAAkE;gBAClE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;oBAChC,MAAM,IAAI,GAAkD;wBAC1D,2BAA2B,EAAE;4BAC3B,UAAU,EAAE,UAAU;4BACtB,QAAQ,EAAE,OAAO,CAAC,WAAW;yBAC9B;qBACF,CAAC;oBACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,GAAyC;wBACjD,mBAAmB,EAAE;4BACnB,UAAU,EAAE,UAAU;yBACvB;qBACF,CAAC;oBACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACnC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,6DAA6D;YAC7D,qBAAqB;YACrB,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;gBACjC,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,UAA+B;IAC9D,OAAO;QACL,GAAG,UAAU;QACb,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO;gBACL,GAAG,KAAK;gBACR,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;gBACzE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;aACjC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "1.7.
|
|
8
|
+
"version": "1.7.2",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"@opentelemetry/sdk-metrics": "1.24.1",
|
|
19
19
|
"async": "^3.2.4",
|
|
20
20
|
"async-mutex": "^0.5.0",
|
|
21
|
-
"bson": "^6.
|
|
21
|
+
"bson": "^6.10.3",
|
|
22
22
|
"commander": "^12.0.0",
|
|
23
23
|
"cors": "^2.8.5",
|
|
24
24
|
"ipaddr.js": "^2.1.0",
|
|
@@ -32,11 +32,11 @@
|
|
|
32
32
|
"uuid": "^9.0.1",
|
|
33
33
|
"winston": "^3.13.0",
|
|
34
34
|
"yaml": "^2.3.2",
|
|
35
|
-
"@powersync/lib-services-framework": "0.5.
|
|
35
|
+
"@powersync/lib-services-framework": "0.5.3",
|
|
36
36
|
"@powersync/service-jsonbig": "0.17.10",
|
|
37
|
-
"@powersync/service-
|
|
38
|
-
"@powersync/service-
|
|
39
|
-
"@powersync/service-
|
|
37
|
+
"@powersync/service-types": "0.8.0",
|
|
38
|
+
"@powersync/service-rsocket-router": "0.0.20",
|
|
39
|
+
"@powersync/service-sync-rules": "0.24.0"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@types/async": "^3.2.24",
|
|
@@ -193,15 +193,15 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
193
193
|
|
|
194
194
|
this.replicationJobs = newJobs;
|
|
195
195
|
|
|
196
|
-
//
|
|
196
|
+
// Stop any orphaned jobs that no longer have sync rules.
|
|
197
|
+
// Termination happens below
|
|
197
198
|
for (let job of existingJobs.values()) {
|
|
198
199
|
// Old - stop and clean up
|
|
199
200
|
try {
|
|
200
201
|
await job.stop();
|
|
201
|
-
await this.terminateSyncRules(job.storage);
|
|
202
202
|
} catch (e) {
|
|
203
203
|
// This will be retried
|
|
204
|
-
this.logger.warn('Failed to
|
|
204
|
+
this.logger.warn('Failed to stop old replication job}', e);
|
|
205
205
|
}
|
|
206
206
|
}
|
|
207
207
|
|
|
@@ -209,7 +209,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
209
209
|
const stopped = await this.storage.getStoppedSyncRules();
|
|
210
210
|
for (let syncRules of stopped) {
|
|
211
211
|
try {
|
|
212
|
-
const syncRuleStorage = this.storage.getInstance(syncRules);
|
|
212
|
+
const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
|
|
213
213
|
await this.terminateSyncRules(syncRuleStorage);
|
|
214
214
|
} catch (e) {
|
|
215
215
|
this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
|
|
@@ -223,13 +223,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
223
223
|
|
|
224
224
|
protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
|
|
225
225
|
this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
|
|
230
|
-
} catch (e) {
|
|
231
|
-
this.logger.warn(`Failed clean up replication config for sync rules: ${syncRuleStorage.group_id}`, e);
|
|
232
|
-
}
|
|
226
|
+
await this.cleanUp(syncRuleStorage);
|
|
227
|
+
await syncRuleStorage.terminate();
|
|
228
|
+
this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
|
|
233
229
|
}
|
|
234
230
|
|
|
235
231
|
abstract testConnection(): Promise<ConnectionTestResult>;
|
|
@@ -12,7 +12,7 @@ export enum SyncRuleState {
|
|
|
12
12
|
/**
|
|
13
13
|
* Sync rule processing is done, and can be used for sync.
|
|
14
14
|
*
|
|
15
|
-
* Only one set of sync rules should be in ACTIVE state.
|
|
15
|
+
* Only one set of sync rules should be in ACTIVE or ERRORED state.
|
|
16
16
|
*/
|
|
17
17
|
ACTIVE = 'ACTIVE',
|
|
18
18
|
/**
|
|
@@ -24,7 +24,16 @@ export enum SyncRuleState {
|
|
|
24
24
|
* After sync rules have been stopped, the data needs to be
|
|
25
25
|
* deleted. Once deleted, the state is TERMINATED.
|
|
26
26
|
*/
|
|
27
|
-
TERMINATED = 'TERMINATED'
|
|
27
|
+
TERMINATED = 'TERMINATED',
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Sync rules has run into a permanent replication error. It
|
|
31
|
+
* is still the "active" sync rules for syncing to users,
|
|
32
|
+
* but should not replicate anymore.
|
|
33
|
+
*
|
|
34
|
+
* It will transition to STOP when a new sync rules is activated.
|
|
35
|
+
*/
|
|
36
|
+
ERRORED = 'ERRORED'
|
|
28
37
|
}
|
|
29
38
|
|
|
30
39
|
export const DEFAULT_DOCUMENT_BATCH_LIMIT = 1000;
|
|
@@ -23,7 +23,7 @@ export interface BucketStorageFactory extends ObserverClient<BucketStorageFactor
|
|
|
23
23
|
/**
|
|
24
24
|
* Get a storage instance to query sync data for specific sync rules.
|
|
25
25
|
*/
|
|
26
|
-
getInstance(
|
|
26
|
+
getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
29
|
* Deploy new sync rules.
|
|
@@ -39,10 +39,8 @@ export interface BucketStorageFactory extends ObserverClient<BucketStorageFactor
|
|
|
39
39
|
* the latest ones.
|
|
40
40
|
*
|
|
41
41
|
* Replication should be restarted after this.
|
|
42
|
-
*
|
|
43
|
-
* @param slot_name The removed slot
|
|
44
42
|
*/
|
|
45
|
-
|
|
43
|
+
restartReplication(sync_rules_group_id: number): Promise<void>;
|
|
46
44
|
|
|
47
45
|
/**
|
|
48
46
|
* Get the sync rules used for querying.
|
|
@@ -125,6 +123,18 @@ export interface UpdateSyncRulesOptions {
|
|
|
125
123
|
validate?: boolean;
|
|
126
124
|
}
|
|
127
125
|
|
|
126
|
+
export interface GetIntanceOptions {
|
|
127
|
+
/**
|
|
128
|
+
* Set to true to skip trigger any events for creating the instance.
|
|
129
|
+
*
|
|
130
|
+
* This is used when creating the instance only for clearing data.
|
|
131
|
+
*
|
|
132
|
+
* When this is used, note that some functionality such as write checkpoint mode
|
|
133
|
+
* may not be configured correctly.
|
|
134
|
+
*/
|
|
135
|
+
skipLifecycleHooks?: boolean;
|
|
136
|
+
}
|
|
137
|
+
|
|
128
138
|
export interface BucketStorageSystemIdentifier {
|
|
129
139
|
/**
|
|
130
140
|
* A unique identifier for the system used for storage.
|
|
@@ -13,9 +13,13 @@ export interface ColumnDescriptor {
|
|
|
13
13
|
// TODO: This needs to be consolidated with SourceTable into something new.
|
|
14
14
|
export interface SourceEntityDescriptor {
|
|
15
15
|
/**
|
|
16
|
-
*
|
|
16
|
+
* The internal id of the data source structure in the database.
|
|
17
|
+
*
|
|
18
|
+
* If undefined, the schema and name are used as the identifier.
|
|
19
|
+
*
|
|
20
|
+
* If specified, this is specifically used to detect renames.
|
|
17
21
|
*/
|
|
18
|
-
objectId: number | string;
|
|
22
|
+
objectId: number | string | undefined;
|
|
19
23
|
schema: string;
|
|
20
24
|
name: string;
|
|
21
25
|
replicationColumns: ColumnDescriptor[];
|
|
@@ -35,7 +35,7 @@ export class SourceTable {
|
|
|
35
35
|
constructor(
|
|
36
36
|
public readonly id: any,
|
|
37
37
|
public readonly connectionTag: string,
|
|
38
|
-
public readonly objectId: number | string,
|
|
38
|
+
public readonly objectId: number | string | undefined,
|
|
39
39
|
public readonly schema: string,
|
|
40
40
|
public readonly table: string,
|
|
41
41
|
|
|
@@ -179,6 +179,15 @@ export interface CompactOptions {
|
|
|
179
179
|
* These can be individual bucket names, or bucket definition names.
|
|
180
180
|
*/
|
|
181
181
|
compactBuckets?: string[];
|
|
182
|
+
|
|
183
|
+
/** Minimum of 2 */
|
|
184
|
+
clearBatchLimit?: number;
|
|
185
|
+
|
|
186
|
+
/** Minimum of 1 */
|
|
187
|
+
moveBatchLimit?: number;
|
|
188
|
+
|
|
189
|
+
/** Minimum of 1 */
|
|
190
|
+
moveBatchQueryLimit?: number;
|
|
182
191
|
}
|
|
183
192
|
|
|
184
193
|
export interface TerminateOptions {
|