@powersync/service-core 0.0.0-dev-20250122110924 → 0.0.0-dev-20250214100224
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -9
- package/dist/api/RouteAPI.d.ts +8 -0
- package/dist/auth/CachedKeyCollector.js +26 -25
- package/dist/auth/CachedKeyCollector.js.map +1 -1
- package/dist/auth/CompoundKeyCollector.js +1 -0
- package/dist/auth/CompoundKeyCollector.js.map +1 -1
- package/dist/auth/KeySpec.js +3 -0
- package/dist/auth/KeySpec.js.map +1 -1
- package/dist/auth/KeyStore.js +4 -0
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/LeakyBucket.js +5 -0
- package/dist/auth/LeakyBucket.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.js +4 -1
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/auth/StaticKeyCollector.js +1 -0
- package/dist/auth/StaticKeyCollector.js.map +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.js +1 -0
- package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
- package/dist/metrics/Metrics.js +35 -1
- package/dist/metrics/Metrics.js.map +1 -1
- package/dist/modules/AbstractModule.js +2 -0
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/modules/ModuleManager.js +1 -3
- package/dist/modules/ModuleManager.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.js +4 -2
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +18 -12
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationEngine.js +1 -3
- package/dist/replication/ReplicationEngine.js.map +1 -1
- package/dist/replication/ReplicationModule.js +3 -0
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/RouterEngine.js +8 -0
- package/dist/routes/RouterEngine.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +3 -3
- package/dist/routes/endpoints/admin.d.ts +6 -6
- package/dist/routes/endpoints/admin.js +4 -1
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +14 -84
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +7 -2
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/runner/teardown.js +2 -2
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +32 -5
- package/dist/storage/BucketStorage.js +3 -0
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/ChecksumCache.js +12 -7
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/SourceTable.js +32 -25
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageEngine.js +4 -3
- package/dist/storage/StorageEngine.js.map +1 -1
- package/dist/storage/bson.d.ts +5 -3
- package/dist/storage/bson.js.map +1 -1
- package/dist/sync/BroadcastIterable.js +4 -3
- package/dist/sync/BroadcastIterable.js.map +1 -1
- package/dist/sync/LastValueSink.js +2 -0
- package/dist/sync/LastValueSink.js.map +1 -1
- package/dist/sync/RequestTracker.js +2 -4
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/merge.js +4 -0
- package/dist/sync/merge.js.map +1 -1
- package/dist/sync/sync.js +2 -2
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js +2 -2
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.js +3 -0
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/util/Mutex.js +5 -0
- package/dist/util/Mutex.js.map +1 -1
- package/dist/util/checkpointing.d.ts +13 -0
- package/dist/util/checkpointing.js +92 -0
- package/dist/util/checkpointing.js.map +1 -0
- package/dist/util/config/compound-config-collector.js +3 -1
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +4 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -0
- package/dist/util/memory-tracking.js +1 -1
- package/dist/util/memory-tracking.js.map +1 -1
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +0 -1
- package/dist/util/utils.js +0 -9
- package/dist/util/utils.js.map +1 -1
- package/package.json +5 -5
- package/src/api/RouteAPI.ts +10 -0
- package/src/replication/AbstractReplicator.ts +9 -3
- package/src/routes/endpoints/admin.ts +4 -1
- package/src/routes/endpoints/checkpointing.ts +8 -19
- package/src/routes/endpoints/sync-rules.ts +7 -2
- package/src/storage/BucketStorage.ts +39 -4
- package/src/storage/bson.ts +9 -7
- package/src/util/checkpointing.ts +43 -0
- package/src/util/config/compound-config-collector.ts +2 -1
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/sync-rules-provider.ts +6 -0
- package/src/util/config/types.ts +1 -0
- package/src/util/memory-tracking.ts +2 -2
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +0 -10
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-tracking.js","sourceRoot":"","sources":["../../src/util/memory-tracking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB;IAC9B,IAAI,OAAO,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAEpC,IAAI,YAAY,GAAG;QACjB,KAAK,EAAE,CAAC;QACR,WAAW,EAAE,CAAC;QACd,eAAe,EAAE,CAAC;QAClB,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,CAAC;KACV,CAAC;IAEF,MAAM,cAAc,GAAG,IAAI,oBAAoB,CAAsC,CAAC,CAAC,EAAE,EAAE;QACzF,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,YAAY,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,GAAgC,CAAC;QAClD,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAA+B,CAAC;QACxE,MAAM,CAAC,QAAQ,CAAC,GAAG,
|
|
1
|
+
{"version":3,"file":"memory-tracking.js","sourceRoot":"","sources":["../../src/util/memory-tracking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB;IAC9B,IAAI,OAAO,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAEpC,IAAI,YAAY,GAAG;QACjB,KAAK,EAAE,CAAC;QACR,WAAW,EAAE,CAAC;QACd,eAAe,EAAE,CAAC;QAClB,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,CAAC;KACV,CAAC;IAEF,MAAM,cAAc,GAAG,IAAI,oBAAoB,CAAsC,CAAC,CAAC,EAAE,EAAE;QACzF,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,YAAY,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,GAAgC,CAAC;QAClD,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAA+B,CAAC;QACxE,MAAM,CAAC,QAAQ,CAAC,GAAG,UAAkE,GAAG,IAAW;YACjG,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAyB,CAAC;YAC1E,YAAY,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC;YAC5C,cAAc,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/D,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC;IACJ,CAAC;IAED,WAAW,CAAC,GAAG,EAAE;QACf,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAElC,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;YACtB,IAAI,IAAI,CAAC,GAAG,CAAE,GAAW,CAAC,GAAG,CAAC,GAAI,OAAe,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,CAAC;gBACpE,WAAW,GAAG,IAAI,CAAC;YACrB,CAAC;QACH,CAAC;QAED,IAAI,WAAW,EAAE,CAAC;YAChB,OAAO,GAAG,GAAG,CAAC;YAEd,MAAM,MAAM,GAAG,OAAO,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC;cACzB,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;eAChB,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;;kBAEb,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC;kBACpB,EAAE,CAAC,YAAY,CAAC,KAAK,GAAG,YAAY,CAAC,WAAW,GAAG,YAAY,CAAC,eAAe,CAAC;iBACjF,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC;mBACnB,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;;QAElC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAE3B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,EAAE,EAAE,CAAC,CAAC;AACT,CAAC;AAED,SAAS,EAAE,CAAC,CAAS;IACnB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;AAC5C,CAAC"}
|
|
@@ -5,6 +5,7 @@ export * from './Mutex.js';
|
|
|
5
5
|
export * from './protocol-types.js';
|
|
6
6
|
export * from './secs.js';
|
|
7
7
|
export * from './utils.js';
|
|
8
|
+
export * from './checkpointing.js';
|
|
8
9
|
export * from './config.js';
|
|
9
10
|
export * from './config/compound-config-collector.js';
|
|
10
11
|
export * from './config/types.js';
|
package/dist/util/util-index.js
CHANGED
|
@@ -5,6 +5,7 @@ export * from './Mutex.js';
|
|
|
5
5
|
export * from './protocol-types.js';
|
|
6
6
|
export * from './secs.js';
|
|
7
7
|
export * from './utils.js';
|
|
8
|
+
export * from './checkpointing.js';
|
|
8
9
|
export * from './config.js';
|
|
9
10
|
export * from './config/compound-config-collector.js';
|
|
10
11
|
export * from './config/types.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util-index.js","sourceRoot":"","sources":["../../src/util/util-index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,UAAU,CAAC;AACzB,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,qBAAqB,CAAC;AACpC,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"util-index.js","sourceRoot":"","sources":["../../src/util/util-index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,UAAU,CAAC;AACzB,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,qBAAqB,CAAC;AACpC,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AAEnC,cAAc,aAAa,CAAC;AAC5B,cAAc,uCAAuC,CAAC;AACtD,cAAc,mBAAmB,CAAC;AAElC,cAAc,yCAAyC,CAAC;AACxD,cAAc,qDAAqD,CAAC;AACpE,cAAc,uDAAuD,CAAC;AACtE,cAAc,yDAAyD,CAAC;AAExE,cAAc,yDAAyD,CAAC;AACxE,cAAc,6DAA6D,CAAC;AAC5E,cAAc,yDAAyD,CAAC;AACxE,cAAc,uCAAuC,CAAC;AACtD,cAAc,4CAA4C,CAAC"}
|
package/dist/util/utils.d.ts
CHANGED
|
@@ -24,7 +24,6 @@ export declare function hasToastedValues(row: sync_rules.ToastableSqliteRow): bo
|
|
|
24
24
|
* If we don't store data, we assume we always have a complete row.
|
|
25
25
|
*/
|
|
26
26
|
export declare function isCompleteRow(storeData: boolean, row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow;
|
|
27
|
-
export declare function checkpointUserId(user_id: string | undefined, client_id: string | undefined): string;
|
|
28
27
|
/**
|
|
29
28
|
* Reduce a bucket to the final state as stored on the client.
|
|
30
29
|
*
|
package/dist/util/utils.js
CHANGED
|
@@ -117,15 +117,6 @@ export function isCompleteRow(storeData, row) {
|
|
|
117
117
|
}
|
|
118
118
|
return !hasToastedValues(row);
|
|
119
119
|
}
|
|
120
|
-
export function checkpointUserId(user_id, client_id) {
|
|
121
|
-
if (user_id == null) {
|
|
122
|
-
throw new Error('user_id is required');
|
|
123
|
-
}
|
|
124
|
-
if (client_id == null) {
|
|
125
|
-
return user_id;
|
|
126
|
-
}
|
|
127
|
-
return `${user_id}/${client_id}`;
|
|
128
|
-
}
|
|
129
120
|
/**
|
|
130
121
|
* Reduce a bucket to the final state as stored on the client.
|
|
131
122
|
*
|
package/dist/util/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAM7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAI1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,SAAkB,EAAE,GAAkC;IAClF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAM7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAI1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,SAAkB,EAAE,GAAkC;IAClF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,UAAwB;IACnD,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,aAAa,GAAG,EAAE,CAAC,QAAkB,CAAC;QACxC,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YAC3B,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAiB;QAC7B,wDAAwD;QACxD,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;QACpD,GAAG,IAAI;KACR,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiD;IAC/E,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.0.0-dev-
|
|
8
|
+
"version": "0.0.0-dev-20250214100224",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -32,11 +32,11 @@
|
|
|
32
32
|
"uuid": "^9.0.1",
|
|
33
33
|
"winston": "^3.13.0",
|
|
34
34
|
"yaml": "^2.3.2",
|
|
35
|
-
"@powersync/lib-services-framework": "0.
|
|
35
|
+
"@powersync/lib-services-framework": "0.5.1",
|
|
36
36
|
"@powersync/service-jsonbig": "0.17.10",
|
|
37
|
-
"@powersync/service-rsocket-router": "0.0.
|
|
38
|
-
"@powersync/service-sync-rules": "0.
|
|
39
|
-
"@powersync/service-types": "0.
|
|
37
|
+
"@powersync/service-rsocket-router": "0.0.18",
|
|
38
|
+
"@powersync/service-sync-rules": "0.23.4",
|
|
39
|
+
"@powersync/service-types": "0.8.0"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@types/async": "^3.2.24",
|
package/src/api/RouteAPI.ts
CHANGED
|
@@ -54,6 +54,14 @@ export interface RouteAPI {
|
|
|
54
54
|
*/
|
|
55
55
|
getReplicationHead(): Promise<string>;
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Get the current LSN or equivalent replication HEAD position identifier.
|
|
59
|
+
*
|
|
60
|
+
* The position is provided to the callback. After the callback returns,
|
|
61
|
+
* the replication head or a greater one will be streamed on the replication stream.
|
|
62
|
+
*/
|
|
63
|
+
createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
|
|
64
|
+
|
|
57
65
|
/**
|
|
58
66
|
* @returns The schema for tables inside the connected database. This is typically
|
|
59
67
|
* used to validate sync rules.
|
|
@@ -76,3 +84,5 @@ export interface RouteAPI {
|
|
|
76
84
|
*/
|
|
77
85
|
getParseSyncRulesOptions(): ParseSyncRulesOptions;
|
|
78
86
|
}
|
|
87
|
+
|
|
88
|
+
export type ReplicationHeadCallback<T> = (head: string) => Promise<T>;
|
|
@@ -93,21 +93,27 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
93
93
|
|
|
94
94
|
private async runLoop() {
|
|
95
95
|
const syncRules = await this.syncRuleProvider.get();
|
|
96
|
+
|
|
96
97
|
let configuredLock: storage.ReplicationLock | undefined = undefined;
|
|
97
98
|
if (syncRules != null) {
|
|
98
99
|
this.logger.info('Loaded sync rules');
|
|
99
100
|
try {
|
|
100
101
|
// Configure new sync rules, if they have changed.
|
|
101
102
|
// In that case, also immediately take out a lock, so that another process doesn't start replication on it.
|
|
102
|
-
|
|
103
|
-
|
|
103
|
+
|
|
104
|
+
const { lock } = await this.storage.configureSyncRules({
|
|
105
|
+
content: syncRules,
|
|
106
|
+
lock: true,
|
|
107
|
+
validate: this.syncRuleProvider.exitOnError
|
|
104
108
|
});
|
|
105
109
|
if (lock) {
|
|
106
110
|
configuredLock = lock;
|
|
107
111
|
}
|
|
108
112
|
} catch (e) {
|
|
109
|
-
// Log
|
|
113
|
+
// Log and re-raise to exit.
|
|
114
|
+
// Should only reach this due to validation errors if exit_on_error is true.
|
|
110
115
|
this.logger.error(`Failed to update sync rules from configuration`, e);
|
|
116
|
+
throw e;
|
|
111
117
|
}
|
|
112
118
|
} else {
|
|
113
119
|
this.logger.info('No sync rules configured - configure via API');
|
|
@@ -128,7 +128,10 @@ export const reprocess = routeDefinition({
|
|
|
128
128
|
}
|
|
129
129
|
|
|
130
130
|
const new_rules = await activeBucketStorage.updateSyncRules({
|
|
131
|
-
content: active.sync_rules.content
|
|
131
|
+
content: active.sync_rules.content,
|
|
132
|
+
// These sync rules already passed validation. But if the rules are not valid anymore due
|
|
133
|
+
// to a service change, we do want to report the error here.
|
|
134
|
+
validate: true
|
|
132
135
|
});
|
|
133
136
|
|
|
134
137
|
const baseConfig = await apiHandler.getSourceConfig();
|
|
@@ -25,7 +25,7 @@ export const writeCheckpoint = routeDefinition({
|
|
|
25
25
|
// Since we don't use LSNs anymore, the only way to get that is to wait.
|
|
26
26
|
const start = Date.now();
|
|
27
27
|
|
|
28
|
-
const head = await apiHandler.
|
|
28
|
+
const head = await apiHandler.createReplicationHead(async (head) => head);
|
|
29
29
|
|
|
30
30
|
const timeout = 50_000;
|
|
31
31
|
|
|
@@ -56,25 +56,14 @@ export const writeCheckpoint2 = routeDefinition({
|
|
|
56
56
|
|
|
57
57
|
const apiHandler = service_context.routerEngine!.getAPI();
|
|
58
58
|
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
storageEngine: { activeBucketStorage }
|
|
65
|
-
} = service_context;
|
|
66
|
-
|
|
67
|
-
const activeSyncRules = await activeBucketStorage.getActiveSyncRulesContent();
|
|
68
|
-
if (!activeSyncRules) {
|
|
69
|
-
throw new framework.errors.ValidationError(`Cannot create Write Checkpoint since no sync rules are active.`);
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
using syncBucketStorage = activeBucketStorage.getInstance(activeSyncRules);
|
|
73
|
-
const writeCheckpoint = await syncBucketStorage.createManagedWriteCheckpoint({
|
|
74
|
-
user_id: full_user_id,
|
|
75
|
-
heads: { '1': currentCheckpoint }
|
|
59
|
+
const { replicationHead, writeCheckpoint } = await util.createWriteCheckpoint({
|
|
60
|
+
userId: user_id,
|
|
61
|
+
clientId: payload.params.client_id,
|
|
62
|
+
api: apiHandler,
|
|
63
|
+
storage: service_context.storageEngine.activeBucketStorage
|
|
76
64
|
});
|
|
77
|
-
|
|
65
|
+
|
|
66
|
+
logger.info(`Write checkpoint for ${user_id}/${payload.params.client_id}: ${writeCheckpoint} | ${replicationHead}`);
|
|
78
67
|
|
|
79
68
|
return {
|
|
80
69
|
write_checkpoint: String(writeCheckpoint)
|
|
@@ -69,7 +69,9 @@ export const deploySyncRules = routeDefinition({
|
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules({
|
|
72
|
-
content: content
|
|
72
|
+
content: content,
|
|
73
|
+
// Aready validated above
|
|
74
|
+
validate: false
|
|
73
75
|
});
|
|
74
76
|
|
|
75
77
|
return {
|
|
@@ -167,7 +169,10 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
167
169
|
}
|
|
168
170
|
|
|
169
171
|
const new_rules = await activeBucketStorage.updateSyncRules({
|
|
170
|
-
content: sync_rules.sync_rules.content
|
|
172
|
+
content: sync_rules.sync_rules.content,
|
|
173
|
+
// These sync rules already passed validation. But if the rules are not valid anymore due
|
|
174
|
+
// to a service change, we do want to report the error here.
|
|
175
|
+
validate: true
|
|
171
176
|
});
|
|
172
177
|
return {
|
|
173
178
|
slot_name: new_rules.slot_name
|
|
@@ -60,13 +60,26 @@ export interface BucketStorageFactoryListener extends DisposableListener {
|
|
|
60
60
|
replicationEvent: (event: ReplicationEventPayload) => void;
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
+
export interface BucketStorageSystemIdentifier {
|
|
64
|
+
/**
|
|
65
|
+
* A unique identifier for the system used for storage.
|
|
66
|
+
* For Postgres this can be the cluster `system_identifier` and database name.
|
|
67
|
+
* For MongoDB this can be the replica set name.
|
|
68
|
+
*/
|
|
69
|
+
id: string;
|
|
70
|
+
/**
|
|
71
|
+
* A unique type for the storage implementation.
|
|
72
|
+
* e.g. `mongodb`, `postgresql`.
|
|
73
|
+
*/
|
|
74
|
+
type: string;
|
|
75
|
+
}
|
|
76
|
+
|
|
63
77
|
export interface BucketStorageFactory extends AsyncDisposableObserverClient<BucketStorageFactoryListener> {
|
|
64
78
|
/**
|
|
65
79
|
* Update sync rules from configuration, if changed.
|
|
66
80
|
*/
|
|
67
81
|
configureSyncRules(
|
|
68
|
-
|
|
69
|
-
options?: { lock?: boolean }
|
|
82
|
+
options: UpdateSyncRulesOptions
|
|
70
83
|
): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }>;
|
|
71
84
|
|
|
72
85
|
/**
|
|
@@ -76,6 +89,8 @@ export interface BucketStorageFactory extends AsyncDisposableObserverClient<Buck
|
|
|
76
89
|
|
|
77
90
|
/**
|
|
78
91
|
* Deploy new sync rules.
|
|
92
|
+
*
|
|
93
|
+
* Similar to configureSyncRules, but applies the update unconditionally.
|
|
79
94
|
*/
|
|
80
95
|
updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
|
|
81
96
|
|
|
@@ -143,6 +158,11 @@ export interface BucketStorageFactory extends AsyncDisposableObserverClient<Buck
|
|
|
143
158
|
* Get the unique identifier for this instance of Powersync
|
|
144
159
|
*/
|
|
145
160
|
getPowerSyncInstanceId(): Promise<string>;
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Get a unique identifier for the system used for storage.
|
|
164
|
+
*/
|
|
165
|
+
getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
|
|
146
166
|
}
|
|
147
167
|
|
|
148
168
|
export interface ReplicationCheckpoint {
|
|
@@ -213,6 +233,7 @@ export interface PersistedSyncRules {
|
|
|
213
233
|
export interface UpdateSyncRulesOptions {
|
|
214
234
|
content: string;
|
|
215
235
|
lock?: boolean;
|
|
236
|
+
validate?: boolean;
|
|
216
237
|
}
|
|
217
238
|
|
|
218
239
|
export interface SyncRulesBucketStorageOptions {
|
|
@@ -367,6 +388,20 @@ export interface BucketBatchStorageListener extends DisposableListener {
|
|
|
367
388
|
replicationEvent: (payload: ReplicationEventPayload) => void;
|
|
368
389
|
}
|
|
369
390
|
|
|
391
|
+
export interface BucketBatchCommitOptions {
|
|
392
|
+
/**
|
|
393
|
+
* Creates a new checkpoint even if there were no persisted operations.
|
|
394
|
+
* Defaults to true.
|
|
395
|
+
*/
|
|
396
|
+
createEmptyCheckpoints?: boolean;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
export type ResolvedBucketBatchCommitOptions = Required<BucketBatchCommitOptions>;
|
|
400
|
+
|
|
401
|
+
export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
|
|
402
|
+
createEmptyCheckpoints: true
|
|
403
|
+
};
|
|
404
|
+
|
|
370
405
|
export interface BucketStorageBatch extends DisposableObserverClient<BucketBatchStorageListener> {
|
|
371
406
|
/**
|
|
372
407
|
* Save an op, and potentially flush.
|
|
@@ -398,11 +433,11 @@ export interface BucketStorageBatch extends DisposableObserverClient<BucketBatch
|
|
|
398
433
|
flush(): Promise<FlushedResult | null>;
|
|
399
434
|
|
|
400
435
|
/**
|
|
401
|
-
* Flush and commit any saved ops. This creates a new checkpoint.
|
|
436
|
+
* Flush and commit any saved ops. This creates a new checkpoint by default.
|
|
402
437
|
*
|
|
403
438
|
* Only call this after a transaction.
|
|
404
439
|
*/
|
|
405
|
-
commit(lsn: string): Promise<boolean>;
|
|
440
|
+
commit(lsn: string, options?: BucketBatchCommitOptions): Promise<boolean>;
|
|
406
441
|
|
|
407
442
|
/**
|
|
408
443
|
* Advance the checkpoint LSN position, without any associated op.
|
package/src/storage/bson.ts
CHANGED
|
@@ -3,6 +3,8 @@ import * as bson from 'bson';
|
|
|
3
3
|
import { SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
4
4
|
import { ReplicaId } from './BucketStorage.js';
|
|
5
5
|
|
|
6
|
+
type NodeBuffer = Buffer<ArrayBuffer>;
|
|
7
|
+
|
|
6
8
|
export const BSON_DESERIALIZE_OPTIONS: bson.DeserializeOptions = {
|
|
7
9
|
// use bigint instead of Long
|
|
8
10
|
useBigInt64: true
|
|
@@ -12,7 +14,7 @@ export const BSON_DESERIALIZE_OPTIONS: bson.DeserializeOptions = {
|
|
|
12
14
|
* Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
|
|
13
15
|
* @param lookup
|
|
14
16
|
*/
|
|
15
|
-
export const serializeLookupBuffer = (lookup: SqliteJsonValue[]):
|
|
17
|
+
export const serializeLookupBuffer = (lookup: SqliteJsonValue[]): NodeBuffer => {
|
|
16
18
|
const normalized = lookup.map((value) => {
|
|
17
19
|
if (typeof value == 'number' && Number.isInteger(value)) {
|
|
18
20
|
return BigInt(value);
|
|
@@ -20,7 +22,7 @@ export const serializeLookupBuffer = (lookup: SqliteJsonValue[]): Buffer => {
|
|
|
20
22
|
return value;
|
|
21
23
|
}
|
|
22
24
|
});
|
|
23
|
-
return bson.serialize({ l: normalized }) as
|
|
25
|
+
return bson.serialize({ l: normalized }) as NodeBuffer;
|
|
24
26
|
};
|
|
25
27
|
|
|
26
28
|
export const serializeLookup = (lookup: SqliteJsonValue[]) => {
|
|
@@ -40,8 +42,8 @@ export const isUUID = (value: any): value is bson.UUID => {
|
|
|
40
42
|
return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
|
|
41
43
|
};
|
|
42
44
|
|
|
43
|
-
export const serializeReplicaId = (id: ReplicaId):
|
|
44
|
-
return bson.serialize({ id }) as
|
|
45
|
+
export const serializeReplicaId = (id: ReplicaId): NodeBuffer => {
|
|
46
|
+
return bson.serialize({ id }) as NodeBuffer;
|
|
45
47
|
};
|
|
46
48
|
|
|
47
49
|
export const deserializeReplicaId = (id: Buffer): ReplicaId => {
|
|
@@ -53,8 +55,8 @@ export const deserializeBson = (buffer: Buffer) => {
|
|
|
53
55
|
return bson.deserialize(buffer, BSON_DESERIALIZE_OPTIONS);
|
|
54
56
|
};
|
|
55
57
|
|
|
56
|
-
export const serializeBson = (document: any):
|
|
57
|
-
return bson.serialize(document) as
|
|
58
|
+
export const serializeBson = (document: any): NodeBuffer => {
|
|
59
|
+
return bson.serialize(document) as NodeBuffer;
|
|
58
60
|
};
|
|
59
61
|
|
|
60
62
|
/**
|
|
@@ -73,6 +75,6 @@ export const replicaIdEquals = (a: ReplicaId, b: ReplicaId) => {
|
|
|
73
75
|
return false;
|
|
74
76
|
} else {
|
|
75
77
|
// There are many possible primitive values, this covers them all
|
|
76
|
-
return serializeReplicaId(a).equals(serializeReplicaId(b)
|
|
78
|
+
return serializeReplicaId(a).equals(serializeReplicaId(b));
|
|
77
79
|
}
|
|
78
80
|
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { RouteAPI } from '../api/RouteAPI.js';
|
|
3
|
+
import { BucketStorageFactory } from '../storage/BucketStorage.js';
|
|
4
|
+
|
|
5
|
+
export interface CreateWriteCheckpointOptions {
|
|
6
|
+
userId: string | undefined;
|
|
7
|
+
clientId: string | undefined;
|
|
8
|
+
api: RouteAPI;
|
|
9
|
+
storage: BucketStorageFactory;
|
|
10
|
+
}
|
|
11
|
+
export async function createWriteCheckpoint(options: CreateWriteCheckpointOptions) {
|
|
12
|
+
const full_user_id = checkpointUserId(options.userId, options.clientId);
|
|
13
|
+
|
|
14
|
+
const activeSyncRules = await options.storage.getActiveSyncRulesContent();
|
|
15
|
+
if (!activeSyncRules) {
|
|
16
|
+
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync rules are active.`);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
using syncBucketStorage = options.storage.getInstance(activeSyncRules);
|
|
20
|
+
|
|
21
|
+
const { writeCheckpoint, currentCheckpoint } = await options.api.createReplicationHead(async (currentCheckpoint) => {
|
|
22
|
+
const writeCheckpoint = await syncBucketStorage.createManagedWriteCheckpoint({
|
|
23
|
+
user_id: full_user_id,
|
|
24
|
+
heads: { '1': currentCheckpoint }
|
|
25
|
+
});
|
|
26
|
+
return { writeCheckpoint, currentCheckpoint };
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
return {
|
|
30
|
+
writeCheckpoint: String(writeCheckpoint),
|
|
31
|
+
replicationHead: currentCheckpoint
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function checkpointUserId(user_id: string | undefined, client_id: string | undefined) {
|
|
36
|
+
if (user_id == null) {
|
|
37
|
+
throw new Error('user_id is required');
|
|
38
|
+
}
|
|
39
|
+
if (client_id == null) {
|
|
40
|
+
return user_id;
|
|
41
|
+
}
|
|
42
|
+
return `${user_id}/${client_id}`;
|
|
43
|
+
}
|
|
@@ -20,6 +20,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
|
|
|
20
20
|
// Only persist the path here, and load on demand using `loadSyncRules()`.
|
|
21
21
|
return {
|
|
22
22
|
present: true,
|
|
23
|
+
exit_on_error: baseConfig.sync_rules?.exit_on_error ?? true,
|
|
23
24
|
path: config_path ? path.resolve(path.dirname(config_path), sync_path) : sync_path
|
|
24
25
|
};
|
|
25
26
|
}
|
|
@@ -3,6 +3,8 @@ import fs from 'fs/promises';
|
|
|
3
3
|
|
|
4
4
|
export interface SyncRulesProvider {
|
|
5
5
|
get(): Promise<string | undefined>;
|
|
6
|
+
|
|
7
|
+
readonly exitOnError: boolean;
|
|
6
8
|
}
|
|
7
9
|
|
|
8
10
|
export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
|
|
@@ -15,4 +17,8 @@ export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
|
|
|
15
17
|
return await fs.readFile(this.config.path, 'utf-8');
|
|
16
18
|
}
|
|
17
19
|
}
|
|
20
|
+
|
|
21
|
+
get exitOnError() {
|
|
22
|
+
return this.config.exit_on_error;
|
|
23
|
+
}
|
|
18
24
|
}
|
package/src/util/config/types.ts
CHANGED
|
@@ -25,8 +25,8 @@ export function trackMemoryUsage() {
|
|
|
25
25
|
for (let key of Object.keys(bufferMemory)) {
|
|
26
26
|
const typedKey = key as keyof typeof bufferMemory;
|
|
27
27
|
const originalFunction = Buffer[typedKey] as (...args: any[]) => Buffer;
|
|
28
|
-
Buffer[typedKey] = function (...args: any[]) {
|
|
29
|
-
const buffer = originalFunction.apply(this, args)
|
|
28
|
+
Buffer[typedKey] = function <TArrayBuffer extends ArrayBufferLike = ArrayBufferLike>(...args: any[]) {
|
|
29
|
+
const buffer = originalFunction.apply(this, args) as Buffer<TArrayBuffer>;
|
|
30
30
|
bufferMemory[typedKey] += buffer.byteLength;
|
|
31
31
|
bufferRegistry.register(buffer, [typedKey, buffer.byteLength]);
|
|
32
32
|
return buffer;
|
package/src/util/util-index.ts
CHANGED
package/src/util/utils.ts
CHANGED
|
@@ -145,16 +145,6 @@ export function isCompleteRow(storeData: boolean, row: sync_rules.ToastableSqlit
|
|
|
145
145
|
return !hasToastedValues(row);
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
-
export function checkpointUserId(user_id: string | undefined, client_id: string | undefined) {
|
|
149
|
-
if (user_id == null) {
|
|
150
|
-
throw new Error('user_id is required');
|
|
151
|
-
}
|
|
152
|
-
if (client_id == null) {
|
|
153
|
-
return user_id;
|
|
154
|
-
}
|
|
155
|
-
return `${user_id}/${client_id}`;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
148
|
/**
|
|
159
149
|
* Reduce a bucket to the final state as stored on the client.
|
|
160
150
|
*
|