@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240725112650
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -6
- package/dist/entry/cli-entry.js +2 -1
- package/dist/entry/cli-entry.js.map +1 -1
- package/dist/entry/commands/compact-action.d.ts +2 -0
- package/dist/entry/commands/compact-action.js +48 -0
- package/dist/entry/commands/compact-action.js.map +1 -0
- package/dist/entry/entry-index.d.ts +1 -0
- package/dist/entry/entry-index.js +1 -0
- package/dist/entry/entry-index.js.map +1 -1
- package/dist/metrics/Metrics.d.ts +4 -3
- package/dist/metrics/Metrics.js +51 -0
- package/dist/metrics/Metrics.js.map +1 -1
- package/dist/replication/WalStream.js +6 -8
- package/dist/replication/WalStream.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +883 -0
- package/dist/routes/configure-fastify.js +58 -0
- package/dist/routes/configure-fastify.js.map +1 -0
- package/dist/routes/configure-rsocket.d.ts +13 -0
- package/dist/routes/configure-rsocket.js +46 -0
- package/dist/routes/configure-rsocket.js.map +1 -0
- package/dist/routes/endpoints/socket-route.js +6 -14
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +4 -5
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/route-register.d.ts +1 -1
- package/dist/routes/route-register.js +1 -1
- package/dist/routes/route-register.js.map +1 -1
- package/dist/routes/router-socket.d.ts +4 -4
- package/dist/routes/router-socket.js.map +1 -1
- package/dist/routes/router.d.ts +1 -0
- package/dist/routes/router.js.map +1 -1
- package/dist/routes/routes-index.d.ts +2 -0
- package/dist/routes/routes-index.js +2 -0
- package/dist/routes/routes-index.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +31 -1
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoCompactor.d.ts +40 -0
- package/dist/storage/mongo/MongoCompactor.js +292 -0
- package/dist/storage/mongo/MongoCompactor.js.map +1 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +3 -2
- package/dist/storage/mongo/MongoSyncBucketStorage.js +19 -13
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/mongo/models.d.ts +5 -4
- package/dist/storage/mongo/models.js.map +1 -1
- package/dist/storage/mongo/util.d.ts +3 -0
- package/dist/storage/mongo/util.js +22 -0
- package/dist/storage/mongo/util.js.map +1 -1
- package/dist/sync/RequestTracker.js +2 -3
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync-index.d.ts +1 -0
- package/dist/sync/sync-index.js +1 -0
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.js +20 -7
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/util/config/collectors/config-collector.d.ts +12 -0
- package/dist/util/config/collectors/config-collector.js +43 -0
- package/dist/util/config/collectors/config-collector.js.map +1 -1
- package/dist/util/config/compound-config-collector.d.ts +3 -29
- package/dist/util/config/compound-config-collector.js +22 -69
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/package.json +6 -4
- package/src/entry/cli-entry.ts +2 -1
- package/src/entry/commands/compact-action.ts +54 -0
- package/src/entry/entry-index.ts +1 -0
- package/src/metrics/Metrics.ts +67 -2
- package/src/replication/WalStream.ts +6 -10
- package/src/routes/configure-fastify.ts +102 -0
- package/src/routes/configure-rsocket.ts +59 -0
- package/src/routes/endpoints/socket-route.ts +6 -15
- package/src/routes/endpoints/sync-stream.ts +4 -5
- package/src/routes/route-register.ts +2 -2
- package/src/routes/router-socket.ts +5 -5
- package/src/routes/router.ts +2 -0
- package/src/routes/routes-index.ts +2 -0
- package/src/storage/BucketStorage.ts +36 -1
- package/src/storage/mongo/MongoCompactor.ts +371 -0
- package/src/storage/mongo/MongoSyncBucketStorage.ts +25 -14
- package/src/storage/mongo/models.ts +5 -4
- package/src/storage/mongo/util.ts +25 -0
- package/src/sync/RequestTracker.ts +3 -3
- package/src/sync/sync-index.ts +1 -0
- package/src/sync/sync.ts +21 -7
- package/src/sync/util.ts +1 -0
- package/src/util/config/collectors/config-collector.ts +48 -0
- package/src/util/config/compound-config-collector.ts +23 -87
- package/test/src/__snapshots__/sync.test.ts.snap +85 -0
- package/test/src/bucket_validation.test.ts +142 -0
- package/test/src/bucket_validation.ts +116 -0
- package/test/src/compacting.test.ts +207 -0
- package/test/src/data_storage.test.ts +19 -60
- package/test/src/slow_tests.test.ts +144 -102
- package/test/src/sync.test.ts +169 -29
- package/test/src/util.ts +71 -13
- package/test/src/wal_stream.test.ts +21 -16
- package/test/src/wal_stream_utils.ts +13 -4
- package/tsconfig.tsbuildinfo +1 -1
package/dist/sync/sync.js
CHANGED
|
@@ -189,6 +189,8 @@ async function* bucketDataInBatches(request) {
|
|
|
189
189
|
*/
|
|
190
190
|
async function* bucketDataBatch(request) {
|
|
191
191
|
const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, tracker, signal } = request;
|
|
192
|
+
const checkpointOp = BigInt(checkpoint);
|
|
193
|
+
let checkpointInvalidated = false;
|
|
192
194
|
const [_, release] = await syncSemaphore.acquire();
|
|
193
195
|
try {
|
|
194
196
|
// Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
|
|
@@ -196,13 +198,16 @@ async function* bucketDataBatch(request) {
|
|
|
196
198
|
const filteredBuckets = new Map(bucketsToFetch.map((bucket) => [bucket, dataBuckets.get(bucket)]));
|
|
197
199
|
const data = storage.getBucketDataBatch(checkpoint, filteredBuckets);
|
|
198
200
|
let has_more = false;
|
|
199
|
-
for await (let r of data) {
|
|
201
|
+
for await (let { batch: r, targetOp } of data) {
|
|
200
202
|
if (signal.aborted) {
|
|
201
203
|
return;
|
|
202
204
|
}
|
|
203
205
|
if (r.has_more) {
|
|
204
206
|
has_more = true;
|
|
205
207
|
}
|
|
208
|
+
if (targetOp != null && targetOp > checkpointOp) {
|
|
209
|
+
checkpointInvalidated = true;
|
|
210
|
+
}
|
|
206
211
|
if (r.data.length == 0) {
|
|
207
212
|
continue;
|
|
208
213
|
}
|
|
@@ -237,12 +242,20 @@ async function* bucketDataBatch(request) {
|
|
|
237
242
|
dataBuckets.set(r.bucket, r.next_after);
|
|
238
243
|
}
|
|
239
244
|
if (!has_more) {
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
245
|
+
if (checkpointInvalidated) {
|
|
246
|
+
// Checkpoint invalidated by a CLEAR or MOVE op.
|
|
247
|
+
// Don't send the checkpoint_complete line in this case.
|
|
248
|
+
// More data should be available immediately for a new checkpoint.
|
|
249
|
+
yield { data: null, done: true };
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
const line = {
|
|
253
|
+
checkpoint_complete: {
|
|
254
|
+
last_op_id: checkpoint
|
|
255
|
+
}
|
|
256
|
+
};
|
|
257
|
+
yield { data: line, done: true };
|
|
258
|
+
}
|
|
246
259
|
}
|
|
247
260
|
}
|
|
248
261
|
finally {
|
package/dist/sync/sync.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAsB,WAAW,EAAE,MAAM,WAAW,CAAC;AAG5D;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;AAgB5D,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC5F,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE;QACV,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE;YAClB,UAAU,CAAC,KAAK,EAAE,CAAC;SACpB;KACF;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5F,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI;QACF,KAAK,CAAC,CAAC,MAAM,CAAC;KACf;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,YAAY,UAAU,EAAE;YAC3B,OAAO;SACR;aAAM;YACL,MAAM,CAAC,CAAC;SACT;KACF;YAAS;QACR,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;KACpB;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAAqC,EACrC,MAAiC,EACjC,UAA6B,EAC7B,OAAuB,EACvB,MAAmB;IAEnB,sCAAsC;IACtC,iHAAiH;IACjH,IAAI,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,IAAI,aAAa,GAA4B,IAAI,CAAC;IAClD,IAAI,mBAAmB,GAAkB,IAAI,CAAC;IAE9C,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEzC,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,KAAK,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE;YACjD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SAC9B;KACF;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,UAAU,CAAC,gBAAgB,CAAC,OAAiB,EAAE,MAAM,CAAC,CAAC;IACnG,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE;QAC/B,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,IAAI,OAAO,IAAI,IAAI,EAAE;YACnB,2EAA2E;YAC3E,SAAS;SACV;QACD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QAEtC,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC;YACjD,gBAAgB,CAAC,OAAO;gBACtB,OAAO,OAAO,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,UAAU,EAAE,UAAU;SACvB,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE;YAC5B,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE;gBAC/B,UAAU;gBACV,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,OAAO,EAAE,UAAU,CAAC,MAAM;aAC3B,CAAC,CAAC;YACH,iEAAiE;YACjE,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;SAC3D;QAED,IAAI,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE;YAC7B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;SAC5D;QACD,WAAW,GAAG,cAAc,CAAC;QAE7B,MAAM,UAAU,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACvE,mEAAmE;QACnE,IAAI,cAAwB,CAAC;QAE7B,IAAI,aAAa,EAAE;YACjB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAE5D,IACE,mBAAmB,IAAI,eAAe;gBACtC,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,EAC/B;gBACA,iDAAiD;gBACjD,SAAS;aACV;YACD,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAE1D,IAAI,OAAO,GAAG,uBAAuB,UAAU,KAAK,CAAC;YACrD,OAAO,IAAI,UAAU,eAAe,KAAK,CAAC;YAC1C,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,KAAK,CAAC;YAC9C,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBACnB,UAAU;gBACV,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,OAAO,EAAE,UAAU,CAAC,MAAM;gBAC1B,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;gBACnC,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;aACpC,CAAC,CAAC;YAEH,MAAM,aAAa,GAAqC;gBACtD,eAAe,EAAE;oBACf,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,eAAe,EAAE,IAAI,CAAC,cAAc;oBACpC,eAAe,EAAE,IAAI,CAAC,cAAc;iBACrC;aACF,CAAC;YAEF,MAAM,aAAa,CAAC;SACrB;aAAM;YACL,IAAI,OAAO,GAAG,mBAAmB,UAAU,aAAa,eAAe,KAAK,CAAC;YAC7E,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC;YAC7E,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9F,cAAc,GAAG,UAAU,CAAC;YAC5B,MAAM,aAAa,GAAiC;gBAClD,UAAU,EAAE;oBACV,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,OAAO,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;iBACnC;aACF,CAAC;YACF,MAAM,aAAa,CAAC;SACrB;QACD,aAAa,GAAG,WAAW,CAAC;QAC5B,mBAAmB,GAAG,eAAe,CAAC;QAEtC,+EAA+E;QAC/E,sFAAsF;QACtF,KAAK,CAAC,CAAC,mBAAmB,CAAC;YACzB,OAAO;YACP,UAAU;YACV,cAAc;YACd,WAAW;YACX,QAAQ;YACR,WAAW;YACX,MAAM;YACN,OAAO;SACR,CAAC,CAAC;QAEH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;KACzD;AACH,CAAC;AAcD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAC,OAA0B;IAC5D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE;QACzC,wEAAwE;QACxE,6EAA6E;QAC7E,4GAA4G;QAC5G,gBAAgB;QAChB,gBAAgB;QAChB,qBAAqB;QACrB,MAAM;QACN,WAAW;QACX,IAAI;QACJ,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI;YACF,OAAO,IAAI,EAAE;gBACX,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,QAAQ,EAAE;oBACZ,MAAM;iBACP;qBAAM;oBACL,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;oBAC7B,MAAM,IAAI,CAAC;oBACX,IAAI,IAAI,EAAE;wBACR,MAAM,GAAG,IAAI,CAAC;qBACf;iBACF;aACF;SACF;gBAAS;YACR,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;SACrB;KACF;AACH,CAAC;AAOD;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAA0B;IACxD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE7G,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,qBAAqB,GAAG,KAAK,CAAC;IAElC,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;IACnD,IAAI;QACF,kGAAkG;QAClG,iDAAiD;QACjD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC,CAAC,CAAC;QACpG,MAAM,IAAI,GAAG,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAErE,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,IAAI,EAAE;YAC7C,IAAI,MAAM,CAAC,OAAO,EAAE;gBAClB,OAAO;aACR;YACD,IAAI,CAAC,CAAC,QAAQ,EAAE;gBACd,QAAQ,GAAG,IAAI,CAAC;aACjB;YACD,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,GAAG,YAAY,EAAE;gBAC/C,qBAAqB,GAAG,IAAI,CAAC;aAC9B;YACD,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE;gBACtB,SAAS;aACV;YACD,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAE7C,IAAI,SAAc,CAAC;YACnB,IAAI,WAAW,EAAE;gBACf,wEAAwE;gBACxE,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;aACzB;iBAAM,IAAI,QAAQ,EAAE;gBACnB,uEAAuE;gBACvE,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,CAAC;iBACR,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;aACtC;iBAAM;gBACL,8EAA8E;gBAC9E,4BAA4B;gBAC5B,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC;iBACjC,CAAC;gBACF,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;aACzC;YACD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACvC,IAAI,SAAS,CAAC,MAAM,GAAG,KAAM,EAAE;gBAC7B,0EAA0E;gBAC1E,8CAA8C;gBAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;aACnC;YACD,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE3C,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;SACzC;QAED,IAAI,CAAC,QAAQ,EAAE;YACb,IAAI,qBAAqB,EAAE;gBACzB,gDAAgD;gBAChD,wDAAwD;gBACxD,kEAAkE;gBAClE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aAClC;iBAAM;gBACL,MAAM,IAAI,GAAyC;oBACjD,mBAAmB,EAAE;wBACnB,UAAU,EAAE,UAAU;qBACvB;iBACF,CAAC;gBACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aAClC;SACF;KACF;YAAS;QACR,OAAO,EAAE,CAAC;KACX;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,UAA+B;IAC9D,OAAO;QACL,GAAG,UAAU;QACb,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO;gBACL,GAAG,KAAK;gBACR,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;gBACzE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;aACjC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,OAAyC,EAAE,KAAa;IAC9E,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE;YACxB,OAAO,CAAC,CAAC,MAAM,CAAC;SACjB;aAAM;YACL,OAAO,CAAC,CAAC;SACV;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;KAChC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC;AACzC,CAAC"}
|
package/dist/sync/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/sync/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/sync/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;AAiB1C,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAElC,MAAM,4BAA4B,GAAuB;IACvD,UAAU,EAAE,IAAI;IAChB,qBAAqB,EAAE,KAAM;CAC9B,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,WAAW,CAChC,KAAsB,EACtB,MAAmB,EACnB,OAAqC;IAErC,MAAM,gBAAgB,GAAuB;QAC3C,GAAG,4BAA4B;QAC/B,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;KACnB,CAAC;IAEF,MAAM,EAAE,UAAU,EAAE,qBAAqB,EAAE,GAAG,gBAAgB,CAAC;IAE/D,8CAA8C;IAC9C,+DAA+D;IAC/D,+BAA+B;IAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC;IACpC,MAAM,iBAAiB,GAAG,UAAU,GAAG,qBAAqB,CAAC;IAE7D,IAAI,iBAAiB,GAAG,IAAI,CAAC;IAE7B,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE;QACtB,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QAC/E,IAAI,iBAAiB,IAAI,gBAAgB,GAAG,CAAC,EAAE;YAC7C,iBAAiB,GAAG,KAAK,CAAC;SAC3B;aAAM;YACL,MAAM,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,CAAC;YAC7C,IAAI,gBAAgB,IAAI,CAAC,EAAE;gBACzB,OAAO;aACR;SACF;QAED,MAAM,gBAAgB,GAAG,kBAAkB,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;QAE1E,4CAA4C;QAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9D,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7E,6CAA6C;QAC7C,MAAM,qBAAqB,GAAG,oBAAoB,IAAI,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,YAAY,CAAC;QAE9F,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC;QACrG,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YAC1D,oBAAoB;QACtB,CAAC,CAAC,CAAC;KACJ;AACH,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC,QAA4D;IACxF,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,QAAQ,EAAE;QAC/B,IAAI,IAAI,IAAI,IAAI,EAAE;YAChB,uCAAuC;YACvC,SAAS;SACV;aAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;YAClC,uBAAuB;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC;SACnB;aAAM;YACL,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;SACnC;KACF;AACH,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAC5C,QAA+B,EAC/B,OAAuB;IAEvB,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,QAAQ,EAAE;QAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1C,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,OAAO,CAAC;KACf;AACH,CAAC"}
|
|
@@ -11,6 +11,18 @@ export declare abstract class ConfigCollector {
|
|
|
11
11
|
* @returns null if this collector cannot provide a config
|
|
12
12
|
*/
|
|
13
13
|
abstract collectSerialized(runnerConfig: RunnerConfig): Promise<configFile.SerializedPowerSyncConfig | null>;
|
|
14
|
+
/**
|
|
15
|
+
* Collects the PowerSyncConfig settings.
|
|
16
|
+
* Validates and decodes the config.
|
|
17
|
+
* @returns null if this collector cannot provide a config
|
|
18
|
+
*/
|
|
19
|
+
collect(runner_config: RunnerConfig): Promise<configFile.PowerSyncConfig | null>;
|
|
20
|
+
/**
|
|
21
|
+
* Validates input config
|
|
22
|
+
* ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
23
|
+
*/
|
|
24
|
+
validate(config: configFile.PowerSyncConfig): void;
|
|
25
|
+
decode(encoded: configFile.SerializedPowerSyncConfig): configFile.PowerSyncConfig;
|
|
14
26
|
protected parseContent(content: string, contentType?: ConfigFileFormat): any;
|
|
15
27
|
protected parseYaml(content: string): any;
|
|
16
28
|
protected parseJSON(content: string): any;
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
import * as t from 'ts-codec';
|
|
1
2
|
import * as yaml from 'yaml';
|
|
3
|
+
import { configFile } from '@powersync/service-types';
|
|
4
|
+
import { schema } from '@powersync/lib-services-framework';
|
|
2
5
|
export var ConfigFileFormat;
|
|
3
6
|
(function (ConfigFileFormat) {
|
|
4
7
|
ConfigFileFormat["YAML"] = "yaml";
|
|
@@ -15,7 +18,47 @@ export var ConfigFileFormat;
|
|
|
15
18
|
* uri: !env PS_MONGO_URI
|
|
16
19
|
*/
|
|
17
20
|
const YAML_ENV_PREFIX = 'PS_';
|
|
21
|
+
// ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
22
|
+
const configSchemaValidator = schema
|
|
23
|
+
.parseJSONSchema(t.generateJSONSchema(configFile.powerSyncConfig, { allowAdditional: true, parsers: [configFile.portParser] }))
|
|
24
|
+
.validator();
|
|
18
25
|
export class ConfigCollector {
|
|
26
|
+
/**
|
|
27
|
+
* Collects the PowerSyncConfig settings.
|
|
28
|
+
* Validates and decodes the config.
|
|
29
|
+
* @returns null if this collector cannot provide a config
|
|
30
|
+
*/
|
|
31
|
+
async collect(runner_config) {
|
|
32
|
+
const serialized = await this.collectSerialized(runner_config);
|
|
33
|
+
if (!serialized) {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* After this point a serialized config has been found. Any failures to decode or validate
|
|
38
|
+
* will result in a hard stop.
|
|
39
|
+
*/
|
|
40
|
+
const decoded = this.decode(serialized);
|
|
41
|
+
this.validate(decoded);
|
|
42
|
+
return decoded;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Validates input config
|
|
46
|
+
* ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
47
|
+
*/
|
|
48
|
+
validate(config) {
|
|
49
|
+
const valid = configSchemaValidator.validate(config);
|
|
50
|
+
if (!valid.valid) {
|
|
51
|
+
throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
decode(encoded) {
|
|
55
|
+
try {
|
|
56
|
+
return configFile.powerSyncConfig.decode(encoded);
|
|
57
|
+
}
|
|
58
|
+
catch (ex) {
|
|
59
|
+
throw new Error(`Failed to decode PowerSync config: ${ex}`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
19
62
|
parseContent(content, contentType) {
|
|
20
63
|
switch (contentType) {
|
|
21
64
|
case ConfigFileFormat.YAML:
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config-collector.js","sourceRoot":"","sources":["../../../../src/util/config/collectors/config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config-collector.js","sourceRoot":"","sources":["../../../../src/util/config/collectors/config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAI3D,MAAM,CAAN,IAAY,gBAGX;AAHD,WAAY,gBAAgB;IAC1B,iCAAa,CAAA;IACb,iCAAa,CAAA;AACf,CAAC,EAHW,gBAAgB,KAAhB,gBAAgB,QAG3B;AAED;;;;;;;;;GASG;AACH,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,uFAAuF;AACvF,MAAM,qBAAqB,GAAG,MAAM;KACjC,eAAe,CACd,CAAC,CAAC,kBAAkB,CAAC,UAAU,CAAC,eAAe,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAC9G;KACA,SAAS,EAAE,CAAC;AAEf,MAAM,OAAgB,eAAe;IASnC;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,aAA2B;QACvC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QAC/D,IAAI,CAAC,UAAU,EAAE;YACf,OAAO,IAAI,CAAC;SACb;QAED;;;WAGG;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,MAAkC;QACzC,MAAM,KAAK,GAAG,qBAAqB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACpF;IACH,CAAC;IAED,MAAM,CAAC,OAA6C;QAClD,IAAI;YACF,OAAO,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SACnD;QAAC,OAAO,EAAE,EAAE;YACX,MAAM,IAAI,KAAK,CAAC,sCAAsC,EAAE,EAAE,CAAC,CAAC;SAC7D;IACH,CAAC;IAES,YAAY,CAAC,OAAe,EAAE,WAA8B;QACpE,QAAQ,WAAW,EAAE;YACnB,KAAK,gBAAgB,CAAC,IAAI;gBACxB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACjC,KAAK,gBAAgB,CAAC,IAAI;gBACxB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACjC,OAAO,CAAC,CAAC;gBACP,6CAA6C;gBAC7C,IAAI;oBACF,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;iBAChC;gBAAC,OAAO,EAAE,EAAE,GAAE;gBACf,IAAI;oBACF,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;iBAChC;gBAAC,OAAO,EAAE,EAAE;oBACX,MAAM,IAAI,KAAK,CAAC,kEAAkE,EAAE,EAAE,CAAC,CAAC;iBACzF;aACF;SACF;IACH,CAAC;IAES,SAAS,CAAC,OAAe;QACjC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;YACzC,MAAM,EAAE,MAAM;YACd,gBAAgB,EAAE,IAAI;YACtB,WAAW;YACX,UAAU,EAAE;gBACV;oBACE,GAAG,EAAE,MAAM;oBACX,OAAO,CAAC,OAAe,EAAE,OAAgC;wBACvD,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;4BACxC,OAAO,CACL,iDAAiD,OAAO,+CAA+C,eAAe,GAAG,CAC1H,CAAC;4BACF,OAAO,OAAO,CAAC;yBAChB;wBACD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;wBACnC,IAAI,OAAO,KAAK,IAAI,WAAW,EAAE;4BAC/B,OAAO,CACL,iDAAiD,OAAO,6DAA6D,CACtH,CAAC;4BACF,OAAO,OAAO,CAAC;yBAChB;wBACD,OAAO,KAAK,CAAC;oBACf,CAAC;iBACF;aACF;SACF,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,gEAAgE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjH,CAAC;SACH;QAED,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAES,SAAS,CAAC,OAAe;QACjC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;CACF"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import * as t from 'ts-codec';
|
|
2
1
|
import { configFile } from '@powersync/service-types';
|
|
3
2
|
import { ConfigCollector } from './collectors/config-collector.js';
|
|
4
3
|
import { ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
|
|
@@ -17,42 +16,17 @@ export type CompoundConfigCollectorOptions = {
|
|
|
17
16
|
*/
|
|
18
17
|
syncRulesCollectors: SyncRulesCollector[];
|
|
19
18
|
};
|
|
20
|
-
export
|
|
21
|
-
SERIALIZED: configFile.SerializedPowerSyncConfig;
|
|
22
|
-
DESERIALIZED: configFile.PowerSyncConfig;
|
|
23
|
-
RESOLVED: ResolvedPowerSyncConfig;
|
|
24
|
-
};
|
|
25
|
-
export declare class CompoundConfigCollector<Generics extends ConfigCollectorGenerics = ConfigCollectorGenerics> {
|
|
19
|
+
export declare class CompoundConfigCollector {
|
|
26
20
|
protected options: CompoundConfigCollectorOptions;
|
|
27
21
|
constructor(options?: CompoundConfigCollectorOptions);
|
|
28
|
-
/**
|
|
29
|
-
* The default ts-codec for validations and decoding
|
|
30
|
-
*/
|
|
31
|
-
get codec(): t.AnyCodec;
|
|
32
22
|
/**
|
|
33
23
|
* Collects and resolves base config
|
|
34
24
|
*/
|
|
35
|
-
collectConfig(
|
|
25
|
+
collectConfig(runner_config?: RunnerConfig): Promise<ResolvedPowerSyncConfig>;
|
|
36
26
|
/**
|
|
37
27
|
* Collects the base PowerSyncConfig from various registered collectors.
|
|
38
28
|
* @throws if no collector could return a configuration.
|
|
39
29
|
*/
|
|
40
|
-
protected collectBaseConfig(runner_config: RunnerConfig): Promise<
|
|
41
|
-
/**
|
|
42
|
-
* Performs the resolving of the common (shared) base configuration
|
|
43
|
-
*/
|
|
44
|
-
protected resolveBaseConfig(baseConfig: Generics['DESERIALIZED'], runnerConfig?: RunnerConfig): Promise<ResolvedPowerSyncConfig>;
|
|
45
|
-
/**
|
|
46
|
-
* Perform any additional resolving from {@link ResolvedPowerSyncConfig}
|
|
47
|
-
* to the extended {@link Generics['RESOLVED']}
|
|
48
|
-
*
|
|
49
|
-
*/
|
|
50
|
-
protected resolveConfig(baseConfig: Generics['DESERIALIZED'], resolvedBaseConfig: ResolvedPowerSyncConfig, runnerConfig?: RunnerConfig): Promise<Generics['RESOLVED']>;
|
|
30
|
+
protected collectBaseConfig(runner_config: RunnerConfig): Promise<configFile.PowerSyncConfig>;
|
|
51
31
|
protected collectSyncRules(baseConfig: configFile.PowerSyncConfig, runnerConfig: RunnerConfig): Promise<SyncRulesConfig>;
|
|
52
|
-
/**
|
|
53
|
-
* Validates input config
|
|
54
|
-
* ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
55
|
-
*/
|
|
56
|
-
protected validate(config: Generics['DESERIALIZED']): void;
|
|
57
|
-
protected decode(encoded: Generics['SERIALIZED']): Generics['DESERIALIZED'];
|
|
58
32
|
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { configFile, normalizeConnection } from '@powersync/service-types';
|
|
1
|
+
import { normalizeConnection } from '@powersync/service-types';
|
|
3
2
|
import * as auth from '../../auth/auth-index.js';
|
|
4
3
|
import { Base64ConfigCollector } from './collectors/impl/base64-config-collector.js';
|
|
5
4
|
import { FileSystemConfigCollector } from './collectors/impl/filesystem-config-collector.js';
|
|
@@ -7,7 +6,7 @@ import { Base64SyncRulesCollector } from './sync-rules/impl/base64-sync-rules-co
|
|
|
7
6
|
import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
|
|
8
7
|
import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-rules-collector.js';
|
|
9
8
|
import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
|
|
10
|
-
import { logger
|
|
9
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
11
10
|
const POWERSYNC_DEV_KID = 'powersync-dev';
|
|
12
11
|
const DEFAULT_COLLECTOR_OPTIONS = {
|
|
13
12
|
configCollectors: [new Base64ConfigCollector(), new FileSystemConfigCollector(), new FallbackConfigCollector()],
|
|
@@ -21,46 +20,11 @@ export class CompoundConfigCollector {
|
|
|
21
20
|
constructor(options = DEFAULT_COLLECTOR_OPTIONS) {
|
|
22
21
|
this.options = options;
|
|
23
22
|
}
|
|
24
|
-
/**
|
|
25
|
-
* The default ts-codec for validations and decoding
|
|
26
|
-
*/
|
|
27
|
-
get codec() {
|
|
28
|
-
return configFile.powerSyncConfig;
|
|
29
|
-
}
|
|
30
23
|
/**
|
|
31
24
|
* Collects and resolves base config
|
|
32
25
|
*/
|
|
33
|
-
async collectConfig(
|
|
34
|
-
const baseConfig = await this.collectBaseConfig(
|
|
35
|
-
const baseResolvedConfig = await this.resolveBaseConfig(baseConfig, runnerConfig);
|
|
36
|
-
return this.resolveConfig(baseConfig, baseResolvedConfig, runnerConfig);
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* Collects the base PowerSyncConfig from various registered collectors.
|
|
40
|
-
* @throws if no collector could return a configuration.
|
|
41
|
-
*/
|
|
42
|
-
async collectBaseConfig(runner_config) {
|
|
43
|
-
for (const collector of this.options.configCollectors) {
|
|
44
|
-
try {
|
|
45
|
-
const baseConfig = await collector.collectSerialized(runner_config);
|
|
46
|
-
if (baseConfig) {
|
|
47
|
-
const decoded = this.decode(baseConfig);
|
|
48
|
-
this.validate(decoded);
|
|
49
|
-
return decoded;
|
|
50
|
-
}
|
|
51
|
-
logger.debug(`Could not collect PowerSync config with ${collector.name} method. Moving on to next method if available.`);
|
|
52
|
-
}
|
|
53
|
-
catch (ex) {
|
|
54
|
-
// An error in a collector is a hard stop
|
|
55
|
-
throw new Error(`Could not collect config using ${collector.name} method. Caught exception: ${ex}`);
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
throw new Error('PowerSyncConfig could not be collected using any of the registered config collectors.');
|
|
59
|
-
}
|
|
60
|
-
/**
|
|
61
|
-
* Performs the resolving of the common (shared) base configuration
|
|
62
|
-
*/
|
|
63
|
-
async resolveBaseConfig(baseConfig, runnerConfig = {}) {
|
|
26
|
+
async collectConfig(runner_config = {}) {
|
|
27
|
+
const baseConfig = await this.collectBaseConfig(runner_config);
|
|
64
28
|
const connections = baseConfig.replication?.connections ?? [];
|
|
65
29
|
if (connections.length > 1) {
|
|
66
30
|
throw new Error('Only a single replication connection is supported currently');
|
|
@@ -93,7 +57,7 @@ export class CompoundConfigCollector {
|
|
|
93
57
|
if (baseConfig.dev?.demo_auth && baseDevKey != null && baseDevKey.kty == 'oct') {
|
|
94
58
|
devKey = await auth.KeySpec.importKey(baseDevKey);
|
|
95
59
|
}
|
|
96
|
-
const sync_rules = await this.collectSyncRules(baseConfig,
|
|
60
|
+
const sync_rules = await this.collectSyncRules(baseConfig, runner_config);
|
|
97
61
|
let jwt_audiences = baseConfig.client_auth?.audience ?? [];
|
|
98
62
|
let config = {
|
|
99
63
|
connection: mapped[0],
|
|
@@ -125,13 +89,24 @@ export class CompoundConfigCollector {
|
|
|
125
89
|
return config;
|
|
126
90
|
}
|
|
127
91
|
/**
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
92
|
+
* Collects the base PowerSyncConfig from various registered collectors.
|
|
93
|
+
* @throws if no collector could return a configuration.
|
|
131
94
|
*/
|
|
132
|
-
async
|
|
133
|
-
|
|
134
|
-
|
|
95
|
+
async collectBaseConfig(runner_config) {
|
|
96
|
+
for (const collector of this.options.configCollectors) {
|
|
97
|
+
try {
|
|
98
|
+
const baseConfig = await collector.collect(runner_config);
|
|
99
|
+
if (baseConfig) {
|
|
100
|
+
return baseConfig;
|
|
101
|
+
}
|
|
102
|
+
logger.debug(`Could not collect PowerSync config with ${collector.name} method. Moving on to next method if available.`);
|
|
103
|
+
}
|
|
104
|
+
catch (ex) {
|
|
105
|
+
// An error in a collector is a hard stop
|
|
106
|
+
throw new Error(`Could not collect config using ${collector.name} method. Caught exception: ${ex}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
throw new Error('PowerSyncConfig could not be collected using any of the registered config collectors.');
|
|
135
110
|
}
|
|
136
111
|
async collectSyncRules(baseConfig, runnerConfig) {
|
|
137
112
|
for (const collector of this.options.syncRulesCollectors) {
|
|
@@ -151,27 +126,5 @@ export class CompoundConfigCollector {
|
|
|
151
126
|
present: false
|
|
152
127
|
};
|
|
153
128
|
}
|
|
154
|
-
/**
|
|
155
|
-
* Validates input config
|
|
156
|
-
* ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
157
|
-
*/
|
|
158
|
-
validate(config) {
|
|
159
|
-
// ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
160
|
-
const validator = schema
|
|
161
|
-
.parseJSONSchema(t.generateJSONSchema(this.codec, { allowAdditional: true, parsers: [configFile.portParser] }))
|
|
162
|
-
.validator();
|
|
163
|
-
const valid = validator.validate(config);
|
|
164
|
-
if (!valid.valid) {
|
|
165
|
-
throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
decode(encoded) {
|
|
169
|
-
try {
|
|
170
|
-
return this.codec.decode(encoded);
|
|
171
|
-
}
|
|
172
|
-
catch (ex) {
|
|
173
|
-
throw new Error(`Failed to decode PowerSync config: ${ex}`);
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
129
|
}
|
|
177
130
|
//# sourceMappingURL=compound-config-collector.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG3E,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,4BAA4B,EAAE,MAAM,sDAAsD,CAAC;AACpG,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAiB1C,MAAM,yBAAyB,GAAmC;IAChE,gBAAgB,EAAE,CAAC,IAAI,qBAAqB,EAAE,EAAE,IAAI,yBAAyB,EAAE,EAAE,IAAI,uBAAuB,EAAE,CAAC;IAC/G,mBAAmB,EAAE;QACnB,IAAI,wBAAwB,EAAE;QAC9B,IAAI,4BAA4B,EAAE;QAClC,IAAI,wBAAwB,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,OAAO,uBAAuB;IAClC,YAAsB,UAA0C,yBAAyB;QAAnE,YAAO,GAAP,OAAO,CAA4D;IAAG,CAAC;IAE7F;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,gBAA8B,EAAE;QAClD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QAE/D,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC;QAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1B,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;SAChF;QAED,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACnC,MAAM,IAAI,GAAuB;gBAC/B,IAAI,EAAE,YAAqB;gBAC3B,GAAG,mBAAmB,CAAC,CAAC,CAAC;gBACzB,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,KAAK;aAChC,CAAC;YAEF,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE5E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAEhC,IAAI,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACzD,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACvF;QAED,IAAI,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,IAAI,QAAQ,EAAE;YAChC,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;SACzB;QAED,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE;YACzB,UAAU,CAAC,GAAG,CACZ,IAAI,IAAI,CAAC,kBAAkB,CACzB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAClG,CACF,CAAC;SACH;QAED,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QAE1G,IAAI,MAAgC,CAAC;QACrC,IAAI,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,EAAE;YAC9E,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;SACnD;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;QAE1E,IAAI,aAAa,GAAa,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QAErE,IAAI,MAAM,GAA4B;YACpC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,eAAe,EAAE,QAAQ;YACzB,+DAA+D;YAC/D,mEAAmE;YACnE,mBAAmB,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;YACvD,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE;YACxC,GAAG,EAAE;gBACH,SAAS,EAAE,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,KAAK;gBAC7C,WAAW,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,IAAI,KAAK;gBACjD,aAAa,EAAE,UAAU,CAAC,GAAG,EAAE,aAAa;gBAC5C,QAAQ,EAAE,UAAU,CAAC,GAAG,EAAE,QAAQ,IAAI,KAAK;gBAC3C,OAAO,EAAE,MAAM;aAChB;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,IAAI;YAC7B,UAAU;YACV,aAAa;YAEb,oBAAoB,EAAE,IAAI;YAC1B,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,EAAE;YACnC,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,SAAS,EAAE;gBACT,yBAAyB,EAAE,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,KAAK;gBACnF,yBAAyB,EACvB,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,0CAA0C;aAChG;YACD,gBAAgB,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,gBAAgB,IAAI,YAAY;SACnE,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB,CAAC,aAA2B;QAC3D,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE;YACrD,IAAI;gBACF,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;gBAC1D,IAAI,UAAU,EAAE;oBACd,OAAO,UAAU,CAAC;iBACnB;gBACD,MAAM,CAAC,KAAK,CACV,2CAA2C,SAAS,CAAC,IAAI,iDAAiD,CAC3G,CAAC;aACH;YAAC,OAAO,EAAE,EAAE;gBACX,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;aACrG;SACF;QACD,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;IAC3G,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC9B,UAAsC,EACtC,YAA0B;QAE1B,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE;YACxD,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;gBACjE,IAAI,MAAM,EAAE;oBACV,OAAO,MAAM,CAAC;iBACf;gBACD,MAAM,CAAC,KAAK,CACV,qCAAqC,SAAS,CAAC,IAAI,iDAAiD,CACrG,CAAC;aACH;YAAC,OAAO,EAAE,EAAE;gBACX,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,sCAAsC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;aACzG;SACF;QACD,OAAO;YACL,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -5,14 +5,16 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.0.0-dev-
|
|
8
|
+
"version": "0.0.0-dev-20240725112650",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
12
12
|
"dependencies": {
|
|
13
13
|
"@js-sdsl/ordered-set": "^4.4.2",
|
|
14
14
|
"@opentelemetry/api": "~1.8.0",
|
|
15
|
+
"@opentelemetry/exporter-metrics-otlp-http": "^0.51.1",
|
|
15
16
|
"@opentelemetry/exporter-prometheus": "^0.51.1",
|
|
17
|
+
"@opentelemetry/resources": "^1.24.1",
|
|
16
18
|
"@opentelemetry/sdk-metrics": "1.24.1",
|
|
17
19
|
"async": "^3.2.4",
|
|
18
20
|
"async-mutex": "^0.5.0",
|
|
@@ -31,11 +33,11 @@
|
|
|
31
33
|
"uuid": "^9.0.1",
|
|
32
34
|
"winston": "^3.13.0",
|
|
33
35
|
"yaml": "^2.3.2",
|
|
34
|
-
"@powersync/lib-services-framework": "0.
|
|
35
|
-
"@powersync/service-jpgwire": "0.17.13",
|
|
36
|
+
"@powersync/lib-services-framework": "0.1.1",
|
|
36
37
|
"@powersync/service-jsonbig": "0.17.10",
|
|
37
|
-
"@powersync/service-rsocket-router": "0.0.
|
|
38
|
+
"@powersync/service-rsocket-router": "0.0.10",
|
|
38
39
|
"@powersync/service-sync-rules": "0.18.1",
|
|
40
|
+
"@powersync/service-jpgwire": "0.17.13",
|
|
39
41
|
"@powersync/service-types": "0.1.0"
|
|
40
42
|
},
|
|
41
43
|
"devDependencies": {
|
package/src/entry/cli-entry.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { Command } from 'commander';
|
|
|
3
3
|
import * as utils from '../util/util-index.js';
|
|
4
4
|
import { registerMigrationAction } from './commands/migrate-action.js';
|
|
5
5
|
import { registerTearDownAction } from './commands/teardown-action.js';
|
|
6
|
-
import { registerStartAction } from './entry-index.js';
|
|
6
|
+
import { registerCompactAction, registerStartAction } from './entry-index.js';
|
|
7
7
|
import { logger } from '@powersync/lib-services-framework';
|
|
8
8
|
|
|
9
9
|
/**
|
|
@@ -18,6 +18,7 @@ export function generateEntryProgram(startHandlers?: Record<utils.ServiceRunner,
|
|
|
18
18
|
|
|
19
19
|
registerTearDownAction(entryProgram);
|
|
20
20
|
registerMigrationAction(entryProgram);
|
|
21
|
+
registerCompactAction(entryProgram);
|
|
21
22
|
|
|
22
23
|
if (startHandlers) {
|
|
23
24
|
registerStartAction(entryProgram, startHandlers);
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
|
|
3
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
4
|
+
import * as v8 from 'v8';
|
|
5
|
+
import { createPowerSyncMongo, MongoBucketStorage } from '../../storage/storage-index.js';
|
|
6
|
+
import { loadConfig } from '../../util/config.js';
|
|
7
|
+
import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
|
|
8
|
+
|
|
9
|
+
const COMMAND_NAME = 'compact';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Approximately max-old-space-size + 64MB.
|
|
13
|
+
*/
|
|
14
|
+
const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Subtract 128MB for process overhead.
|
|
18
|
+
*
|
|
19
|
+
* Limit to 1024MB overall.
|
|
20
|
+
*/
|
|
21
|
+
const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
|
|
22
|
+
|
|
23
|
+
export function registerCompactAction(program: Command) {
|
|
24
|
+
const compactCommand = program.command(COMMAND_NAME);
|
|
25
|
+
|
|
26
|
+
wrapConfigCommand(compactCommand);
|
|
27
|
+
|
|
28
|
+
return compactCommand.description('Compact storage').action(async (options) => {
|
|
29
|
+
const runnerConfig = extractRunnerOptions(options);
|
|
30
|
+
|
|
31
|
+
const config = await loadConfig(runnerConfig);
|
|
32
|
+
const { storage } = config;
|
|
33
|
+
const psdb = createPowerSyncMongo(storage);
|
|
34
|
+
const client = psdb.client;
|
|
35
|
+
await client.connect();
|
|
36
|
+
try {
|
|
37
|
+
const bucketStorage = new MongoBucketStorage(psdb, { slot_name_prefix: config.slot_name_prefix });
|
|
38
|
+
const active = await bucketStorage.getActiveSyncRules();
|
|
39
|
+
if (active == null) {
|
|
40
|
+
logger.info('No active instance to compact');
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const p = bucketStorage.getInstance(active);
|
|
44
|
+
await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
|
|
45
|
+
logger.info('done');
|
|
46
|
+
} catch (e) {
|
|
47
|
+
logger.error(`Failed to compact: ${e.toString()}`);
|
|
48
|
+
process.exit(1);
|
|
49
|
+
} finally {
|
|
50
|
+
await client.close();
|
|
51
|
+
process.exit(0);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
}
|
package/src/entry/entry-index.ts
CHANGED
package/src/metrics/Metrics.ts
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api';
|
|
2
2
|
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
|
|
3
|
-
import { MeterProvider } from '@opentelemetry/sdk-metrics';
|
|
3
|
+
import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
|
|
4
|
+
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
|
4
5
|
import * as jpgwire from '@powersync/service-jpgwire';
|
|
6
|
+
import * as util from '../util/util-index.js';
|
|
5
7
|
import * as storage from '../storage/storage-index.js';
|
|
6
8
|
import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
|
|
9
|
+
import { Resource } from '@opentelemetry/resources';
|
|
7
10
|
import { logger } from '@powersync/lib-services-framework';
|
|
8
11
|
|
|
9
12
|
export interface MetricsOptions {
|
|
@@ -13,6 +16,8 @@ export interface MetricsOptions {
|
|
|
13
16
|
}
|
|
14
17
|
|
|
15
18
|
export class Metrics {
|
|
19
|
+
private static instance: Metrics;
|
|
20
|
+
|
|
16
21
|
private prometheusExporter: PrometheusExporter;
|
|
17
22
|
private meterProvider: MeterProvider;
|
|
18
23
|
|
|
@@ -55,7 +60,7 @@ export class Metrics {
|
|
|
55
60
|
// Record on API pod
|
|
56
61
|
public concurrent_connections: UpDownCounter<Attributes>;
|
|
57
62
|
|
|
58
|
-
constructor(meterProvider: MeterProvider, prometheusExporter: PrometheusExporter) {
|
|
63
|
+
private constructor(meterProvider: MeterProvider, prometheusExporter: PrometheusExporter) {
|
|
59
64
|
this.meterProvider = meterProvider;
|
|
60
65
|
this.prometheusExporter = prometheusExporter;
|
|
61
66
|
const meter = meterProvider.getMeter('powersync');
|
|
@@ -127,6 +132,66 @@ export class Metrics {
|
|
|
127
132
|
this.concurrent_connections.add(0);
|
|
128
133
|
}
|
|
129
134
|
|
|
135
|
+
public static getInstance(): Metrics {
|
|
136
|
+
if (!Metrics.instance) {
|
|
137
|
+
throw new Error('Metrics have not been initialised');
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return Metrics.instance;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
public static async initialise(options: MetricsOptions): Promise<void> {
|
|
144
|
+
if (Metrics.instance) {
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
logger.info('Configuring telemetry.');
|
|
148
|
+
|
|
149
|
+
logger.info(
|
|
150
|
+
`
|
|
151
|
+
Attention:
|
|
152
|
+
PowerSync collects completely anonymous telemetry regarding usage.
|
|
153
|
+
This information is used to shape our roadmap to better serve our customers.
|
|
154
|
+
You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL:
|
|
155
|
+
https://docs.powersync.com/self-hosting/telemetry
|
|
156
|
+
Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disabled' : 'enabled'}
|
|
157
|
+
`.trim()
|
|
158
|
+
);
|
|
159
|
+
|
|
160
|
+
const configuredExporters: MetricReader[] = [];
|
|
161
|
+
|
|
162
|
+
const port: number = util.env.METRICS_PORT ?? 0;
|
|
163
|
+
const prometheusExporter = new PrometheusExporter({ port: port, preventServerStart: true });
|
|
164
|
+
configuredExporters.push(prometheusExporter);
|
|
165
|
+
|
|
166
|
+
if (!options.disable_telemetry_sharing) {
|
|
167
|
+
logger.info('Sharing anonymous telemetry');
|
|
168
|
+
const periodicExporter = new PeriodicExportingMetricReader({
|
|
169
|
+
exporter: new OTLPMetricExporter({
|
|
170
|
+
url: options.internal_metrics_endpoint
|
|
171
|
+
}),
|
|
172
|
+
exportIntervalMillis: 1000 * 60 * 5 // 5 minutes
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
configuredExporters.push(periodicExporter);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const meterProvider = new MeterProvider({
|
|
179
|
+
resource: new Resource({
|
|
180
|
+
['service']: 'PowerSync',
|
|
181
|
+
['instance_id']: options.powersync_instance_id
|
|
182
|
+
}),
|
|
183
|
+
readers: configuredExporters
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
if (port > 0) {
|
|
187
|
+
await prometheusExporter.startServer();
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
Metrics.instance = new Metrics(meterProvider, prometheusExporter);
|
|
191
|
+
|
|
192
|
+
logger.info('Telemetry configuration complete.');
|
|
193
|
+
}
|
|
194
|
+
|
|
130
195
|
public async shutdown(): Promise<void> {
|
|
131
196
|
await this.meterProvider.shutdown();
|
|
132
197
|
}
|