@powersync/common 1.51.0 → 1.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.cjs +510 -1129
- package/dist/bundle.cjs.map +1 -1
- package/dist/bundle.mjs +511 -1116
- package/dist/bundle.mjs.map +1 -1
- package/dist/bundle.node.cjs +508 -1129
- package/dist/bundle.node.cjs.map +1 -1
- package/dist/bundle.node.mjs +509 -1116
- package/dist/bundle.node.mjs.map +1 -1
- package/dist/index.d.cts +73 -433
- package/legacy/sync_protocol.d.ts +103 -0
- package/lib/client/AbstractPowerSyncDatabase.js +3 -3
- package/lib/client/AbstractPowerSyncDatabase.js.map +1 -1
- package/lib/client/ConnectionManager.js +1 -1
- package/lib/client/ConnectionManager.js.map +1 -1
- package/lib/client/sync/bucket/BucketStorageAdapter.d.ts +6 -64
- package/lib/client/sync/bucket/BucketStorageAdapter.js +4 -0
- package/lib/client/sync/bucket/BucketStorageAdapter.js.map +1 -1
- package/lib/client/sync/bucket/SqliteBucketStorage.d.ts +1 -28
- package/lib/client/sync/bucket/SqliteBucketStorage.js +0 -162
- package/lib/client/sync/bucket/SqliteBucketStorage.js.map +1 -1
- package/lib/client/sync/stream/AbstractRemote.d.ts +29 -18
- package/lib/client/sync/stream/AbstractRemote.js +155 -188
- package/lib/client/sync/stream/AbstractRemote.js.map +1 -1
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +13 -35
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +150 -448
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js.map +1 -1
- package/lib/client/sync/stream/JsonValue.d.ts +7 -0
- package/lib/client/sync/stream/JsonValue.js +2 -0
- package/lib/client/sync/stream/JsonValue.js.map +1 -0
- package/lib/client/sync/stream/core-instruction.d.ts +14 -9
- package/lib/client/sync/stream/core-instruction.js +3 -0
- package/lib/client/sync/stream/core-instruction.js.map +1 -1
- package/lib/db/DBAdapter.d.ts +9 -0
- package/lib/db/DBAdapter.js +8 -1
- package/lib/db/DBAdapter.js.map +1 -1
- package/lib/db/crud/SyncStatus.d.ts +3 -4
- package/lib/db/crud/SyncStatus.js +0 -4
- package/lib/db/crud/SyncStatus.js.map +1 -1
- package/lib/db/schema/RawTable.d.ts +0 -5
- package/lib/db/schema/Schema.d.ts +0 -2
- package/lib/db/schema/Schema.js +0 -2
- package/lib/db/schema/Schema.js.map +1 -1
- package/lib/index.d.ts +2 -6
- package/lib/index.js +1 -6
- package/lib/index.js.map +1 -1
- package/lib/utils/async.d.ts +0 -9
- package/lib/utils/async.js +0 -9
- package/lib/utils/async.js.map +1 -1
- package/lib/utils/stream_transform.d.ts +39 -0
- package/lib/utils/stream_transform.js +206 -0
- package/lib/utils/stream_transform.js.map +1 -0
- package/package.json +15 -10
- package/src/client/AbstractPowerSyncDatabase.ts +3 -3
- package/src/client/ConnectionManager.ts +1 -1
- package/src/client/sync/bucket/BucketStorageAdapter.ts +6 -71
- package/src/client/sync/bucket/SqliteBucketStorage.ts +1 -197
- package/src/client/sync/stream/AbstractRemote.ts +183 -229
- package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +181 -510
- package/src/client/sync/stream/JsonValue.ts +8 -0
- package/src/client/sync/stream/core-instruction.ts +15 -5
- package/src/db/DBAdapter.ts +20 -2
- package/src/db/crud/SyncStatus.ts +4 -5
- package/src/db/schema/RawTable.ts +0 -5
- package/src/db/schema/Schema.ts +0 -2
- package/src/index.ts +2 -6
- package/src/utils/async.ts +0 -11
- package/src/utils/stream_transform.ts +252 -0
- package/lib/client/sync/bucket/OpType.d.ts +0 -16
- package/lib/client/sync/bucket/OpType.js +0 -23
- package/lib/client/sync/bucket/OpType.js.map +0 -1
- package/lib/client/sync/bucket/OplogEntry.d.ts +0 -23
- package/lib/client/sync/bucket/OplogEntry.js +0 -36
- package/lib/client/sync/bucket/OplogEntry.js.map +0 -1
- package/lib/client/sync/bucket/SyncDataBatch.d.ts +0 -6
- package/lib/client/sync/bucket/SyncDataBatch.js +0 -12
- package/lib/client/sync/bucket/SyncDataBatch.js.map +0 -1
- package/lib/client/sync/bucket/SyncDataBucket.d.ts +0 -40
- package/lib/client/sync/bucket/SyncDataBucket.js +0 -40
- package/lib/client/sync/bucket/SyncDataBucket.js.map +0 -1
- package/lib/client/sync/stream/streaming-sync-types.d.ts +0 -143
- package/lib/client/sync/stream/streaming-sync-types.js +0 -26
- package/lib/client/sync/stream/streaming-sync-types.js.map +0 -1
- package/lib/utils/DataStream.d.ts +0 -62
- package/lib/utils/DataStream.js +0 -169
- package/lib/utils/DataStream.js.map +0 -1
- package/src/client/sync/bucket/OpType.ts +0 -23
- package/src/client/sync/bucket/OplogEntry.ts +0 -50
- package/src/client/sync/bucket/SyncDataBatch.ts +0 -11
- package/src/client/sync/bucket/SyncDataBucket.ts +0 -49
- package/src/client/sync/stream/streaming-sync-types.ts +0 -210
- package/src/utils/DataStream.ts +0 -222
package/dist/bundle.node.cjs
CHANGED
|
@@ -1819,8 +1819,15 @@ class BaseTransaction {
|
|
|
1819
1819
|
class TransactionImplementation extends DBGetUtilsDefaultMixin(BaseTransaction) {
|
|
1820
1820
|
static async runWith(ctx, fn) {
|
|
1821
1821
|
let tx = new TransactionImplementation(ctx);
|
|
1822
|
+
// For write transactions, use BEGIN IMMEDIATE to immediately obtain a write lock on the database (instead of doing
|
|
1823
|
+
// that on the first statement). If we have a genuine read-only connection, we also use BEGIN IMMEDIATE there: In
|
|
1824
|
+
// WAL mode, that ensures we pin the current state of the database (instead of the state at the first statement in
|
|
1825
|
+
// the transaction). But if we have a "fake" read-only connection implemented through `pragma query_only = true`, we
|
|
1826
|
+
// can't use this trick because it would attempt to lock the connection. So there, we use a regular `BEGIN`
|
|
1827
|
+
// statement.
|
|
1828
|
+
const useBeginImmediate = ctx.connectionType != 'queryOnly';
|
|
1822
1829
|
try {
|
|
1823
|
-
await ctx.execute('BEGIN IMMEDIATE');
|
|
1830
|
+
await ctx.execute(useBeginImmediate ? 'BEGIN IMMEDIATE' : 'BEGIN');
|
|
1824
1831
|
const result = await fn(tx);
|
|
1825
1832
|
await tx.commit();
|
|
1826
1833
|
return result;
|
|
@@ -1979,16 +1986,12 @@ class SyncStatus {
|
|
|
1979
1986
|
*
|
|
1980
1987
|
* This returns null when the database is currently being opened and we don't have reliable information about all
|
|
1981
1988
|
* included streams yet.
|
|
1982
|
-
*
|
|
1983
|
-
* @experimental Sync streams are currently in alpha.
|
|
1984
1989
|
*/
|
|
1985
1990
|
get syncStreams() {
|
|
1986
1991
|
return this.options.dataFlow?.internalStreamSubscriptions?.map((core) => new SyncStreamStatusView(this, core));
|
|
1987
1992
|
}
|
|
1988
1993
|
/**
|
|
1989
1994
|
* If the `stream` appears in {@link syncStreams}, returns the current status for that stream.
|
|
1990
|
-
*
|
|
1991
|
-
* @experimental Sync streams are currently in alpha.
|
|
1992
1995
|
*/
|
|
1993
1996
|
forStream(stream) {
|
|
1994
1997
|
const asJson = JSON.stringify(stream.parameters);
|
|
@@ -2257,15 +2260,6 @@ class ControlledExecutor {
|
|
|
2257
2260
|
}
|
|
2258
2261
|
}
|
|
2259
2262
|
|
|
2260
|
-
/**
|
|
2261
|
-
* A ponyfill for `Symbol.asyncIterator` that is compatible with the
|
|
2262
|
-
* [recommended polyfill](https://github.com/Azure/azure-sdk-for-js/blob/%40azure/core-asynciterator-polyfill_1.0.2/sdk/core/core-asynciterator-polyfill/src/index.ts#L4-L6)
|
|
2263
|
-
* we recommend for React Native.
|
|
2264
|
-
*
|
|
2265
|
-
* As long as we use this symbol (instead of `for await` and `async *`) in this package, we can be compatible with async
|
|
2266
|
-
* iterators without requiring them.
|
|
2267
|
-
*/
|
|
2268
|
-
const symbolAsyncIterator = Symbol.asyncIterator ?? Symbol.for('Symbol.asyncIterator');
|
|
2269
2263
|
/**
|
|
2270
2264
|
* Throttle a function to be called at most once every "wait" milliseconds,
|
|
2271
2265
|
* on the trailing edge.
|
|
@@ -2580,7 +2574,7 @@ class SyncStreamSubscriptionHandle {
|
|
|
2580
2574
|
constructor(subscription) {
|
|
2581
2575
|
this.subscription = subscription;
|
|
2582
2576
|
subscription.refcount++;
|
|
2583
|
-
_finalizer?.register(this, subscription);
|
|
2577
|
+
_finalizer?.register(this, subscription, this);
|
|
2584
2578
|
}
|
|
2585
2579
|
get name() {
|
|
2586
2580
|
return this.subscription.name;
|
|
@@ -3169,6 +3163,10 @@ exports.PowerSyncControlCommand = void 0;
|
|
|
3169
3163
|
PowerSyncControlCommand["NOTIFY_TOKEN_REFRESHED"] = "refreshed_token";
|
|
3170
3164
|
PowerSyncControlCommand["NOTIFY_CRUD_UPLOAD_COMPLETED"] = "completed_upload";
|
|
3171
3165
|
PowerSyncControlCommand["UPDATE_SUBSCRIPTIONS"] = "update_subscriptions";
|
|
3166
|
+
/**
|
|
3167
|
+
* An `established` or `end` event for response streams.
|
|
3168
|
+
*/
|
|
3169
|
+
PowerSyncControlCommand["CONNECTION_STATE"] = "connection";
|
|
3172
3170
|
})(exports.PowerSyncControlCommand || (exports.PowerSyncControlCommand = {}));
|
|
3173
3171
|
|
|
3174
3172
|
/**
|
|
@@ -3350,103 +3348,6 @@ class AbortOperation extends Error {
|
|
|
3350
3348
|
}
|
|
3351
3349
|
}
|
|
3352
3350
|
|
|
3353
|
-
exports.OpTypeEnum = void 0;
|
|
3354
|
-
(function (OpTypeEnum) {
|
|
3355
|
-
OpTypeEnum[OpTypeEnum["CLEAR"] = 1] = "CLEAR";
|
|
3356
|
-
OpTypeEnum[OpTypeEnum["MOVE"] = 2] = "MOVE";
|
|
3357
|
-
OpTypeEnum[OpTypeEnum["PUT"] = 3] = "PUT";
|
|
3358
|
-
OpTypeEnum[OpTypeEnum["REMOVE"] = 4] = "REMOVE";
|
|
3359
|
-
})(exports.OpTypeEnum || (exports.OpTypeEnum = {}));
|
|
3360
|
-
/**
|
|
3361
|
-
* Used internally for sync buckets.
|
|
3362
|
-
*/
|
|
3363
|
-
class OpType {
|
|
3364
|
-
value;
|
|
3365
|
-
static fromJSON(jsonValue) {
|
|
3366
|
-
return new OpType(exports.OpTypeEnum[jsonValue]);
|
|
3367
|
-
}
|
|
3368
|
-
constructor(value) {
|
|
3369
|
-
this.value = value;
|
|
3370
|
-
}
|
|
3371
|
-
toJSON() {
|
|
3372
|
-
return Object.entries(exports.OpTypeEnum).find(([, value]) => value === this.value)[0];
|
|
3373
|
-
}
|
|
3374
|
-
}
|
|
3375
|
-
|
|
3376
|
-
class OplogEntry {
|
|
3377
|
-
op_id;
|
|
3378
|
-
op;
|
|
3379
|
-
checksum;
|
|
3380
|
-
subkey;
|
|
3381
|
-
object_type;
|
|
3382
|
-
object_id;
|
|
3383
|
-
data;
|
|
3384
|
-
static fromRow(row) {
|
|
3385
|
-
return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, row.subkey, row.object_type, row.object_id, row.data);
|
|
3386
|
-
}
|
|
3387
|
-
constructor(op_id, op, checksum, subkey, object_type, object_id, data) {
|
|
3388
|
-
this.op_id = op_id;
|
|
3389
|
-
this.op = op;
|
|
3390
|
-
this.checksum = checksum;
|
|
3391
|
-
this.subkey = subkey;
|
|
3392
|
-
this.object_type = object_type;
|
|
3393
|
-
this.object_id = object_id;
|
|
3394
|
-
this.data = data;
|
|
3395
|
-
}
|
|
3396
|
-
toJSON(fixedKeyEncoding = false) {
|
|
3397
|
-
return {
|
|
3398
|
-
op_id: this.op_id,
|
|
3399
|
-
op: this.op.toJSON(),
|
|
3400
|
-
object_type: this.object_type,
|
|
3401
|
-
object_id: this.object_id,
|
|
3402
|
-
checksum: this.checksum,
|
|
3403
|
-
data: this.data,
|
|
3404
|
-
// Older versions of the JS SDK used to always JSON.stringify here. That has always been wrong,
|
|
3405
|
-
// but we need to migrate gradually to not break existing databases.
|
|
3406
|
-
subkey: fixedKeyEncoding ? this.subkey : JSON.stringify(this.subkey)
|
|
3407
|
-
};
|
|
3408
|
-
}
|
|
3409
|
-
}
|
|
3410
|
-
|
|
3411
|
-
class SyncDataBucket {
|
|
3412
|
-
bucket;
|
|
3413
|
-
data;
|
|
3414
|
-
has_more;
|
|
3415
|
-
after;
|
|
3416
|
-
next_after;
|
|
3417
|
-
static fromRow(row) {
|
|
3418
|
-
return new SyncDataBucket(row.bucket, row.data.map((entry) => OplogEntry.fromRow(entry)), row.has_more ?? false, row.after, row.next_after);
|
|
3419
|
-
}
|
|
3420
|
-
constructor(bucket, data,
|
|
3421
|
-
/**
|
|
3422
|
-
* True if the response does not contain all the data for this bucket, and another request must be made.
|
|
3423
|
-
*/
|
|
3424
|
-
has_more,
|
|
3425
|
-
/**
|
|
3426
|
-
* The `after` specified in the request.
|
|
3427
|
-
*/
|
|
3428
|
-
after,
|
|
3429
|
-
/**
|
|
3430
|
-
* Use this for the next request.
|
|
3431
|
-
*/
|
|
3432
|
-
next_after) {
|
|
3433
|
-
this.bucket = bucket;
|
|
3434
|
-
this.data = data;
|
|
3435
|
-
this.has_more = has_more;
|
|
3436
|
-
this.after = after;
|
|
3437
|
-
this.next_after = next_after;
|
|
3438
|
-
}
|
|
3439
|
-
toJSON(fixedKeyEncoding = false) {
|
|
3440
|
-
return {
|
|
3441
|
-
bucket: this.bucket,
|
|
3442
|
-
has_more: this.has_more,
|
|
3443
|
-
after: this.after,
|
|
3444
|
-
next_after: this.next_after,
|
|
3445
|
-
data: this.data.map((entry) => entry.toJSON(fixedKeyEncoding))
|
|
3446
|
-
};
|
|
3447
|
-
}
|
|
3448
|
-
}
|
|
3449
|
-
|
|
3450
3351
|
var dist = {};
|
|
3451
3352
|
|
|
3452
3353
|
var Codecs = {};
|
|
@@ -8231,177 +8132,10 @@ function requireDist () {
|
|
|
8231
8132
|
|
|
8232
8133
|
var distExports = requireDist();
|
|
8233
8134
|
|
|
8234
|
-
var version = "1.
|
|
8135
|
+
var version = "1.53.0";
|
|
8235
8136
|
var PACKAGE = {
|
|
8236
8137
|
version: version};
|
|
8237
8138
|
|
|
8238
|
-
const DEFAULT_PRESSURE_LIMITS = {
|
|
8239
|
-
highWater: 10,
|
|
8240
|
-
lowWater: 0
|
|
8241
|
-
};
|
|
8242
|
-
/**
|
|
8243
|
-
* A very basic implementation of a data stream with backpressure support which does not use
|
|
8244
|
-
* native JS streams or async iterators.
|
|
8245
|
-
* This is handy for environments such as React Native which need polyfills for the above.
|
|
8246
|
-
*/
|
|
8247
|
-
class DataStream extends BaseObserver {
|
|
8248
|
-
options;
|
|
8249
|
-
dataQueue;
|
|
8250
|
-
isClosed;
|
|
8251
|
-
processingPromise;
|
|
8252
|
-
notifyDataAdded;
|
|
8253
|
-
logger;
|
|
8254
|
-
mapLine;
|
|
8255
|
-
constructor(options) {
|
|
8256
|
-
super();
|
|
8257
|
-
this.options = options;
|
|
8258
|
-
this.processingPromise = null;
|
|
8259
|
-
this.isClosed = false;
|
|
8260
|
-
this.dataQueue = [];
|
|
8261
|
-
this.mapLine = options?.mapLine ?? ((line) => line);
|
|
8262
|
-
this.logger = options?.logger ?? Logger.get('DataStream');
|
|
8263
|
-
if (options?.closeOnError) {
|
|
8264
|
-
const l = this.registerListener({
|
|
8265
|
-
error: (ex) => {
|
|
8266
|
-
l?.();
|
|
8267
|
-
this.close();
|
|
8268
|
-
}
|
|
8269
|
-
});
|
|
8270
|
-
}
|
|
8271
|
-
}
|
|
8272
|
-
get highWatermark() {
|
|
8273
|
-
return this.options?.pressure?.highWaterMark ?? DEFAULT_PRESSURE_LIMITS.highWater;
|
|
8274
|
-
}
|
|
8275
|
-
get lowWatermark() {
|
|
8276
|
-
return this.options?.pressure?.lowWaterMark ?? DEFAULT_PRESSURE_LIMITS.lowWater;
|
|
8277
|
-
}
|
|
8278
|
-
get closed() {
|
|
8279
|
-
return this.isClosed;
|
|
8280
|
-
}
|
|
8281
|
-
async close() {
|
|
8282
|
-
this.isClosed = true;
|
|
8283
|
-
await this.processingPromise;
|
|
8284
|
-
this.iterateListeners((l) => l.closed?.());
|
|
8285
|
-
// Discard any data in the queue
|
|
8286
|
-
this.dataQueue = [];
|
|
8287
|
-
this.listeners.clear();
|
|
8288
|
-
}
|
|
8289
|
-
/**
|
|
8290
|
-
* Enqueues data for the consumers to read
|
|
8291
|
-
*/
|
|
8292
|
-
enqueueData(data) {
|
|
8293
|
-
if (this.isClosed) {
|
|
8294
|
-
throw new Error('Cannot enqueue data into closed stream.');
|
|
8295
|
-
}
|
|
8296
|
-
this.dataQueue.push(data);
|
|
8297
|
-
this.notifyDataAdded?.();
|
|
8298
|
-
this.processQueue();
|
|
8299
|
-
}
|
|
8300
|
-
/**
|
|
8301
|
-
* Reads data once from the data stream
|
|
8302
|
-
* @returns a Data payload or Null if the stream closed.
|
|
8303
|
-
*/
|
|
8304
|
-
async read() {
|
|
8305
|
-
if (this.closed) {
|
|
8306
|
-
return null;
|
|
8307
|
-
}
|
|
8308
|
-
// Wait for any pending processing to complete first.
|
|
8309
|
-
// This ensures we register our listener before calling processQueue(),
|
|
8310
|
-
// avoiding a race where processQueue() sees no reader and returns early.
|
|
8311
|
-
if (this.processingPromise) {
|
|
8312
|
-
await this.processingPromise;
|
|
8313
|
-
}
|
|
8314
|
-
// Re-check after await - stream may have closed while we were waiting
|
|
8315
|
-
if (this.closed) {
|
|
8316
|
-
return null;
|
|
8317
|
-
}
|
|
8318
|
-
return new Promise((resolve, reject) => {
|
|
8319
|
-
const l = this.registerListener({
|
|
8320
|
-
data: async (data) => {
|
|
8321
|
-
resolve(data);
|
|
8322
|
-
// Remove the listener
|
|
8323
|
-
l?.();
|
|
8324
|
-
},
|
|
8325
|
-
closed: () => {
|
|
8326
|
-
resolve(null);
|
|
8327
|
-
l?.();
|
|
8328
|
-
},
|
|
8329
|
-
error: (ex) => {
|
|
8330
|
-
reject(ex);
|
|
8331
|
-
l?.();
|
|
8332
|
-
}
|
|
8333
|
-
});
|
|
8334
|
-
this.processQueue();
|
|
8335
|
-
});
|
|
8336
|
-
}
|
|
8337
|
-
/**
|
|
8338
|
-
* Executes a callback for each data item in the stream
|
|
8339
|
-
*/
|
|
8340
|
-
forEach(callback) {
|
|
8341
|
-
if (this.dataQueue.length <= this.lowWatermark) {
|
|
8342
|
-
this.iterateAsyncErrored(async (l) => l.lowWater?.());
|
|
8343
|
-
}
|
|
8344
|
-
return this.registerListener({
|
|
8345
|
-
data: callback
|
|
8346
|
-
});
|
|
8347
|
-
}
|
|
8348
|
-
processQueue() {
|
|
8349
|
-
if (this.processingPromise) {
|
|
8350
|
-
return;
|
|
8351
|
-
}
|
|
8352
|
-
const promise = (this.processingPromise = this._processQueue());
|
|
8353
|
-
promise.finally(() => {
|
|
8354
|
-
this.processingPromise = null;
|
|
8355
|
-
});
|
|
8356
|
-
return promise;
|
|
8357
|
-
}
|
|
8358
|
-
hasDataReader() {
|
|
8359
|
-
return Array.from(this.listeners.values()).some((l) => !!l.data);
|
|
8360
|
-
}
|
|
8361
|
-
async _processQueue() {
|
|
8362
|
-
/**
|
|
8363
|
-
* Allow listeners to mutate the queue before processing.
|
|
8364
|
-
* This allows for operations such as dropping or compressing data
|
|
8365
|
-
* on high water or requesting more data on low water.
|
|
8366
|
-
*/
|
|
8367
|
-
if (this.dataQueue.length >= this.highWatermark) {
|
|
8368
|
-
await this.iterateAsyncErrored(async (l) => l.highWater?.());
|
|
8369
|
-
}
|
|
8370
|
-
if (this.isClosed || !this.hasDataReader()) {
|
|
8371
|
-
return;
|
|
8372
|
-
}
|
|
8373
|
-
if (this.dataQueue.length) {
|
|
8374
|
-
const data = this.dataQueue.shift();
|
|
8375
|
-
const mapped = this.mapLine(data);
|
|
8376
|
-
await this.iterateAsyncErrored(async (l) => l.data?.(mapped));
|
|
8377
|
-
}
|
|
8378
|
-
if (this.dataQueue.length <= this.lowWatermark) {
|
|
8379
|
-
const dataAdded = new Promise((resolve) => {
|
|
8380
|
-
this.notifyDataAdded = resolve;
|
|
8381
|
-
});
|
|
8382
|
-
await Promise.race([this.iterateAsyncErrored(async (l) => l.lowWater?.()), dataAdded]);
|
|
8383
|
-
this.notifyDataAdded = null;
|
|
8384
|
-
}
|
|
8385
|
-
if (this.dataQueue.length > 0) {
|
|
8386
|
-
setTimeout(() => this.processQueue());
|
|
8387
|
-
}
|
|
8388
|
-
}
|
|
8389
|
-
async iterateAsyncErrored(cb) {
|
|
8390
|
-
// Important: We need to copy the listeners, as calling a listener could result in adding another
|
|
8391
|
-
// listener, resulting in infinite loops.
|
|
8392
|
-
const listeners = Array.from(this.listeners.values());
|
|
8393
|
-
for (let i of listeners) {
|
|
8394
|
-
try {
|
|
8395
|
-
await cb(i);
|
|
8396
|
-
}
|
|
8397
|
-
catch (ex) {
|
|
8398
|
-
this.logger.error(ex);
|
|
8399
|
-
this.iterateListeners((l) => l.error?.(ex));
|
|
8400
|
-
}
|
|
8401
|
-
}
|
|
8402
|
-
}
|
|
8403
|
-
}
|
|
8404
|
-
|
|
8405
8139
|
var WebsocketDuplexConnection = {};
|
|
8406
8140
|
|
|
8407
8141
|
var hasRequiredWebsocketDuplexConnection;
|
|
@@ -8564,8 +8298,199 @@ class WebsocketClientTransport {
|
|
|
8564
8298
|
}
|
|
8565
8299
|
}
|
|
8566
8300
|
|
|
8301
|
+
const doneResult = { done: true, value: undefined };
|
|
8302
|
+
function valueResult(value) {
|
|
8303
|
+
return { done: false, value };
|
|
8304
|
+
}
|
|
8305
|
+
/**
|
|
8306
|
+
* Expands a source async iterator by allowing to inject events asynchronously.
|
|
8307
|
+
*
|
|
8308
|
+
* The resulting iterator will emit all events from its source. Additionally though, events can be injected. These
|
|
8309
|
+
* events are dropped once the main iterator completes, but are otherwise forwarded.
|
|
8310
|
+
*
|
|
8311
|
+
* The iterator completes when its source completes, and it supports backpressure by only calling `next()` on the source
|
|
8312
|
+
* in response to a `next()` call from downstream if no pending injected events can be dispatched.
|
|
8313
|
+
*/
|
|
8314
|
+
function injectable(source) {
|
|
8315
|
+
let sourceIsDone = false;
|
|
8316
|
+
let waiter = undefined; // An active, waiting next() call.
|
|
8317
|
+
// A pending upstream event that couldn't be dispatched because inject() has been called before it was resolved.
|
|
8318
|
+
let pendingSourceEvent = null;
|
|
8319
|
+
let pendingInjectedEvents = [];
|
|
8320
|
+
const consumeWaiter = () => {
|
|
8321
|
+
const pending = waiter;
|
|
8322
|
+
waiter = undefined;
|
|
8323
|
+
return pending;
|
|
8324
|
+
};
|
|
8325
|
+
const fetchFromSource = () => {
|
|
8326
|
+
const resolveWaiter = (propagate) => {
|
|
8327
|
+
const active = consumeWaiter();
|
|
8328
|
+
if (active) {
|
|
8329
|
+
propagate(active);
|
|
8330
|
+
}
|
|
8331
|
+
else {
|
|
8332
|
+
pendingSourceEvent = propagate;
|
|
8333
|
+
}
|
|
8334
|
+
};
|
|
8335
|
+
const nextFromSource = source.next();
|
|
8336
|
+
nextFromSource.then((value) => {
|
|
8337
|
+
sourceIsDone = value.done == true;
|
|
8338
|
+
resolveWaiter((w) => w.resolve(value));
|
|
8339
|
+
}, (error) => {
|
|
8340
|
+
resolveWaiter((w) => w.reject(error));
|
|
8341
|
+
});
|
|
8342
|
+
};
|
|
8343
|
+
return {
|
|
8344
|
+
next: () => {
|
|
8345
|
+
return new Promise((resolve, reject) => {
|
|
8346
|
+
// First priority: Dispatch ready upstream events.
|
|
8347
|
+
if (sourceIsDone) {
|
|
8348
|
+
return resolve(doneResult);
|
|
8349
|
+
}
|
|
8350
|
+
if (pendingSourceEvent) {
|
|
8351
|
+
pendingSourceEvent({ resolve, reject });
|
|
8352
|
+
pendingSourceEvent = null;
|
|
8353
|
+
return;
|
|
8354
|
+
}
|
|
8355
|
+
// Second priority: Dispatch injected events
|
|
8356
|
+
if (pendingInjectedEvents.length) {
|
|
8357
|
+
return resolve(valueResult(pendingInjectedEvents.shift()));
|
|
8358
|
+
}
|
|
8359
|
+
// Nothing pending? Fetch from source
|
|
8360
|
+
waiter = { resolve, reject };
|
|
8361
|
+
return fetchFromSource();
|
|
8362
|
+
});
|
|
8363
|
+
},
|
|
8364
|
+
inject: (event) => {
|
|
8365
|
+
const pending = consumeWaiter();
|
|
8366
|
+
if (pending != null) {
|
|
8367
|
+
pending.resolve(valueResult(event));
|
|
8368
|
+
}
|
|
8369
|
+
else {
|
|
8370
|
+
pendingInjectedEvents.push(event);
|
|
8371
|
+
}
|
|
8372
|
+
}
|
|
8373
|
+
};
|
|
8374
|
+
}
|
|
8375
|
+
/**
|
|
8376
|
+
* Splits a byte stream at line endings, emitting each line as a string.
|
|
8377
|
+
*/
|
|
8378
|
+
function extractJsonLines(source, decoder) {
|
|
8379
|
+
let buffer = '';
|
|
8380
|
+
const pendingLines = [];
|
|
8381
|
+
let isFinalEvent = false;
|
|
8382
|
+
return {
|
|
8383
|
+
next: async () => {
|
|
8384
|
+
while (true) {
|
|
8385
|
+
if (isFinalEvent) {
|
|
8386
|
+
return doneResult;
|
|
8387
|
+
}
|
|
8388
|
+
{
|
|
8389
|
+
const first = pendingLines.shift();
|
|
8390
|
+
if (first) {
|
|
8391
|
+
return { done: false, value: first };
|
|
8392
|
+
}
|
|
8393
|
+
}
|
|
8394
|
+
const { done, value } = await source.next();
|
|
8395
|
+
if (done) {
|
|
8396
|
+
const remaining = buffer.trim();
|
|
8397
|
+
if (remaining.length != 0) {
|
|
8398
|
+
isFinalEvent = true;
|
|
8399
|
+
return { done: false, value: remaining };
|
|
8400
|
+
}
|
|
8401
|
+
return doneResult;
|
|
8402
|
+
}
|
|
8403
|
+
const data = decoder.decode(value, { stream: true });
|
|
8404
|
+
buffer += data;
|
|
8405
|
+
const lines = buffer.split('\n');
|
|
8406
|
+
for (let i = 0; i < lines.length - 1; i++) {
|
|
8407
|
+
const l = lines[i].trim();
|
|
8408
|
+
if (l.length > 0) {
|
|
8409
|
+
pendingLines.push(l);
|
|
8410
|
+
}
|
|
8411
|
+
}
|
|
8412
|
+
buffer = lines[lines.length - 1];
|
|
8413
|
+
}
|
|
8414
|
+
}
|
|
8415
|
+
};
|
|
8416
|
+
}
|
|
8417
|
+
/**
|
|
8418
|
+
* Splits a concatenated stream of BSON objects by emitting individual objects.
|
|
8419
|
+
*/
|
|
8420
|
+
function extractBsonObjects(source) {
|
|
8421
|
+
// Fully read but not emitted yet.
|
|
8422
|
+
const completedObjects = [];
|
|
8423
|
+
// Whether source has returned { done: true }. We do the same once completed objects have been emitted.
|
|
8424
|
+
let isDone = false;
|
|
8425
|
+
const lengthBuffer = new DataView(new ArrayBuffer(4));
|
|
8426
|
+
let objectBody = null;
|
|
8427
|
+
// If we're parsing the length field, a number between 1 and 4 (inclusive) describing remaining bytes in the header.
|
|
8428
|
+
// If we're consuming a document, the bytes remaining.
|
|
8429
|
+
let remainingLength = 4;
|
|
8430
|
+
return {
|
|
8431
|
+
async next() {
|
|
8432
|
+
while (true) {
|
|
8433
|
+
// Before fetching new data from upstream, return completed objects.
|
|
8434
|
+
if (completedObjects.length) {
|
|
8435
|
+
return valueResult(completedObjects.shift());
|
|
8436
|
+
}
|
|
8437
|
+
if (isDone) {
|
|
8438
|
+
return doneResult;
|
|
8439
|
+
}
|
|
8440
|
+
const upstreamEvent = await source.next();
|
|
8441
|
+
if (upstreamEvent.done) {
|
|
8442
|
+
isDone = true;
|
|
8443
|
+
if (objectBody || remainingLength != 4) {
|
|
8444
|
+
throw new Error('illegal end of stream in BSON object');
|
|
8445
|
+
}
|
|
8446
|
+
return doneResult;
|
|
8447
|
+
}
|
|
8448
|
+
const chunk = upstreamEvent.value;
|
|
8449
|
+
for (let i = 0; i < chunk.length;) {
|
|
8450
|
+
const availableInData = chunk.length - i;
|
|
8451
|
+
if (objectBody) {
|
|
8452
|
+
// We're in the middle of reading a BSON document.
|
|
8453
|
+
const bytesToRead = Math.min(availableInData, remainingLength);
|
|
8454
|
+
const copySource = new Uint8Array(chunk.buffer, chunk.byteOffset + i, bytesToRead);
|
|
8455
|
+
objectBody.set(copySource, objectBody.length - remainingLength);
|
|
8456
|
+
i += bytesToRead;
|
|
8457
|
+
remainingLength -= bytesToRead;
|
|
8458
|
+
if (remainingLength == 0) {
|
|
8459
|
+
completedObjects.push(objectBody);
|
|
8460
|
+
// Prepare to read another document, starting with its length
|
|
8461
|
+
objectBody = null;
|
|
8462
|
+
remainingLength = 4;
|
|
8463
|
+
}
|
|
8464
|
+
}
|
|
8465
|
+
else {
|
|
8466
|
+
// Copy up to 4 bytes into lengthBuffer, depending on how many we still need.
|
|
8467
|
+
const bytesToRead = Math.min(availableInData, remainingLength);
|
|
8468
|
+
for (let j = 0; j < bytesToRead; j++) {
|
|
8469
|
+
lengthBuffer.setUint8(4 - remainingLength + j, chunk[i + j]);
|
|
8470
|
+
}
|
|
8471
|
+
i += bytesToRead;
|
|
8472
|
+
remainingLength -= bytesToRead;
|
|
8473
|
+
if (remainingLength == 0) {
|
|
8474
|
+
// Transition from reading length header to reading document. Subtracting 4 because the length of the
|
|
8475
|
+
// header is included in length.
|
|
8476
|
+
const length = lengthBuffer.getInt32(0, true /* little endian */);
|
|
8477
|
+
remainingLength = length - 4;
|
|
8478
|
+
if (remainingLength < 1) {
|
|
8479
|
+
throw new Error(`invalid length for bson: ${length}`);
|
|
8480
|
+
}
|
|
8481
|
+
objectBody = new Uint8Array(length);
|
|
8482
|
+
new DataView(objectBody.buffer).setInt32(0, length, true);
|
|
8483
|
+
}
|
|
8484
|
+
}
|
|
8485
|
+
}
|
|
8486
|
+
}
|
|
8487
|
+
}
|
|
8488
|
+
};
|
|
8489
|
+
}
|
|
8490
|
+
|
|
8567
8491
|
const POWERSYNC_TRAILING_SLASH_MATCH = /\/+$/;
|
|
8568
8492
|
const POWERSYNC_JS_VERSION = PACKAGE.version;
|
|
8493
|
+
const SYNC_QUEUE_REQUEST_HIGH_WATER = 10;
|
|
8569
8494
|
const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
|
|
8570
8495
|
// Keep alive message is sent every period
|
|
8571
8496
|
const KEEP_ALIVE_MS = 20_000;
|
|
@@ -8745,73 +8670,67 @@ class AbstractRemote {
|
|
|
8745
8670
|
return new WebSocket(url);
|
|
8746
8671
|
}
|
|
8747
8672
|
/**
|
|
8748
|
-
* Returns a data stream of sync line data.
|
|
8673
|
+
* Returns a data stream of sync line data, fetched via RSocket-over-WebSocket.
|
|
8749
8674
|
*
|
|
8750
|
-
*
|
|
8751
|
-
* @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
|
|
8752
|
-
* (required for compatibility with older sync services).
|
|
8675
|
+
* The only mechanism to abort the returned stream is to use the abort signal in {@link SocketSyncStreamOptions}.
|
|
8753
8676
|
*/
|
|
8754
|
-
async socketStreamRaw(options
|
|
8677
|
+
async socketStreamRaw(options) {
|
|
8755
8678
|
const { path, fetchStrategy = exports.FetchStrategy.Buffered } = options;
|
|
8756
|
-
const mimeType =
|
|
8679
|
+
const mimeType = 'application/json';
|
|
8757
8680
|
function toBuffer(js) {
|
|
8758
|
-
|
|
8759
|
-
if (bson != null) {
|
|
8760
|
-
contents = bson.serialize(js);
|
|
8761
|
-
}
|
|
8762
|
-
else {
|
|
8763
|
-
contents = JSON.stringify(js);
|
|
8764
|
-
}
|
|
8765
|
-
return node_buffer.Buffer.from(contents);
|
|
8681
|
+
return node_buffer.Buffer.from(JSON.stringify(js));
|
|
8766
8682
|
}
|
|
8767
8683
|
const syncQueueRequestSize = fetchStrategy == exports.FetchStrategy.Buffered ? 10 : 1;
|
|
8768
8684
|
const request = await this.buildRequest(path);
|
|
8685
|
+
const url = this.options.socketUrlTransformer(request.url);
|
|
8769
8686
|
// Add the user agent in the setup payload - we can't set custom
|
|
8770
8687
|
// headers with websockets on web. The browser userAgent is however added
|
|
8771
8688
|
// automatically as a header.
|
|
8772
8689
|
const userAgent = this.getUserAgent();
|
|
8773
|
-
|
|
8774
|
-
|
|
8775
|
-
|
|
8776
|
-
|
|
8777
|
-
|
|
8778
|
-
|
|
8779
|
-
|
|
8690
|
+
// While we're connecting (a process that can't be aborted in RSocket), the WebSocket instance to close if we wanted
|
|
8691
|
+
// to abort the connection.
|
|
8692
|
+
let pendingSocket = null;
|
|
8693
|
+
let keepAliveTimeout;
|
|
8694
|
+
let rsocket = null;
|
|
8695
|
+
let queue = null;
|
|
8696
|
+
let didClose = false;
|
|
8697
|
+
const abortRequest = () => {
|
|
8698
|
+
if (didClose) {
|
|
8699
|
+
return;
|
|
8700
|
+
}
|
|
8701
|
+
didClose = true;
|
|
8702
|
+
clearTimeout(keepAliveTimeout);
|
|
8703
|
+
if (pendingSocket) {
|
|
8704
|
+
pendingSocket.close();
|
|
8705
|
+
}
|
|
8706
|
+
if (rsocket) {
|
|
8707
|
+
rsocket.close();
|
|
8708
|
+
}
|
|
8709
|
+
if (queue) {
|
|
8710
|
+
queue.stop();
|
|
8711
|
+
}
|
|
8712
|
+
};
|
|
8780
8713
|
// Handle upstream abort
|
|
8781
|
-
if (options.abortSignal
|
|
8714
|
+
if (options.abortSignal.aborted) {
|
|
8782
8715
|
throw new AbortOperation('Connection request aborted');
|
|
8783
8716
|
}
|
|
8784
8717
|
else {
|
|
8785
|
-
options.abortSignal
|
|
8786
|
-
stream.close();
|
|
8787
|
-
}, { once: true });
|
|
8718
|
+
options.abortSignal.addEventListener('abort', abortRequest);
|
|
8788
8719
|
}
|
|
8789
|
-
let keepAliveTimeout;
|
|
8790
8720
|
const resetTimeout = () => {
|
|
8791
8721
|
clearTimeout(keepAliveTimeout);
|
|
8792
8722
|
keepAliveTimeout = setTimeout(() => {
|
|
8793
8723
|
this.logger.error(`No data received on WebSocket in ${SOCKET_TIMEOUT_MS}ms, closing connection.`);
|
|
8794
|
-
|
|
8724
|
+
abortRequest();
|
|
8795
8725
|
}, SOCKET_TIMEOUT_MS);
|
|
8796
8726
|
};
|
|
8797
8727
|
resetTimeout();
|
|
8798
|
-
// Typescript complains about this being `never` if it's not assigned here.
|
|
8799
|
-
// This is assigned in `wsCreator`.
|
|
8800
|
-
let disposeSocketConnectionTimeout = () => { };
|
|
8801
|
-
const url = this.options.socketUrlTransformer(request.url);
|
|
8802
8728
|
const connector = new distExports.RSocketConnector({
|
|
8803
8729
|
transport: new WebsocketClientTransport({
|
|
8804
8730
|
url,
|
|
8805
8731
|
wsCreator: (url) => {
|
|
8806
|
-
const socket = this.createSocket(url);
|
|
8807
|
-
|
|
8808
|
-
closed: () => {
|
|
8809
|
-
// Allow closing the underlying WebSocket if the stream was closed before the
|
|
8810
|
-
// RSocket connect completed. This should effectively abort the request.
|
|
8811
|
-
socket.close();
|
|
8812
|
-
}
|
|
8813
|
-
});
|
|
8814
|
-
socket.addEventListener('message', (event) => {
|
|
8732
|
+
const socket = (pendingSocket = this.createSocket(url));
|
|
8733
|
+
socket.addEventListener('message', () => {
|
|
8815
8734
|
resetTimeout();
|
|
8816
8735
|
});
|
|
8817
8736
|
return socket;
|
|
@@ -8831,43 +8750,40 @@ class AbstractRemote {
|
|
|
8831
8750
|
}
|
|
8832
8751
|
}
|
|
8833
8752
|
});
|
|
8834
|
-
let rsocket;
|
|
8835
8753
|
try {
|
|
8836
8754
|
rsocket = await connector.connect();
|
|
8837
8755
|
// The connection is established, we no longer need to monitor the initial timeout
|
|
8838
|
-
|
|
8756
|
+
pendingSocket = null;
|
|
8839
8757
|
}
|
|
8840
8758
|
catch (ex) {
|
|
8841
8759
|
this.logger.error(`Failed to connect WebSocket`, ex);
|
|
8842
|
-
|
|
8843
|
-
if (!stream.closed) {
|
|
8844
|
-
await stream.close();
|
|
8845
|
-
}
|
|
8760
|
+
abortRequest();
|
|
8846
8761
|
throw ex;
|
|
8847
8762
|
}
|
|
8848
8763
|
resetTimeout();
|
|
8849
|
-
let socketIsClosed = false;
|
|
8850
|
-
const closeSocket = () => {
|
|
8851
|
-
clearTimeout(keepAliveTimeout);
|
|
8852
|
-
if (socketIsClosed) {
|
|
8853
|
-
return;
|
|
8854
|
-
}
|
|
8855
|
-
socketIsClosed = true;
|
|
8856
|
-
rsocket.close();
|
|
8857
|
-
};
|
|
8858
8764
|
// Helps to prevent double close scenarios
|
|
8859
|
-
rsocket.onClose(() => (
|
|
8860
|
-
|
|
8861
|
-
let pendingEventsCount = syncQueueRequestSize;
|
|
8862
|
-
const disposeClosedListener = stream.registerListener({
|
|
8863
|
-
closed: () => {
|
|
8864
|
-
closeSocket();
|
|
8865
|
-
disposeClosedListener();
|
|
8866
|
-
}
|
|
8867
|
-
});
|
|
8868
|
-
const socket = await new Promise((resolve, reject) => {
|
|
8765
|
+
rsocket.onClose(() => (rsocket = null));
|
|
8766
|
+
return await new Promise((resolve, reject) => {
|
|
8869
8767
|
let connectionEstablished = false;
|
|
8870
|
-
|
|
8768
|
+
let pendingEventsCount = syncQueueRequestSize;
|
|
8769
|
+
let paused = false;
|
|
8770
|
+
let res = null;
|
|
8771
|
+
function requestMore() {
|
|
8772
|
+
const delta = syncQueueRequestSize - pendingEventsCount;
|
|
8773
|
+
if (!paused && delta > 0) {
|
|
8774
|
+
res?.request(delta);
|
|
8775
|
+
pendingEventsCount = syncQueueRequestSize;
|
|
8776
|
+
}
|
|
8777
|
+
}
|
|
8778
|
+
const events = new eventIterator.EventIterator((q) => {
|
|
8779
|
+
queue = q;
|
|
8780
|
+
q.on('highWater', () => (paused = true));
|
|
8781
|
+
q.on('lowWater', () => {
|
|
8782
|
+
paused = false;
|
|
8783
|
+
requestMore();
|
|
8784
|
+
});
|
|
8785
|
+
}, { highWaterMark: SYNC_QUEUE_REQUEST_HIGH_WATER, lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER })[Symbol.asyncIterator]();
|
|
8786
|
+
res = rsocket.requestStream({
|
|
8871
8787
|
data: toBuffer(options.data),
|
|
8872
8788
|
metadata: toBuffer({
|
|
8873
8789
|
path
|
|
@@ -8892,7 +8808,7 @@ class AbstractRemote {
|
|
|
8892
8808
|
}
|
|
8893
8809
|
// RSocket will close the RSocket stream automatically
|
|
8894
8810
|
// Close the downstream stream as well - this will close the RSocket connection and WebSocket
|
|
8895
|
-
|
|
8811
|
+
abortRequest();
|
|
8896
8812
|
// Handles cases where the connection failed e.g. auth error or connection error
|
|
8897
8813
|
if (!connectionEstablished) {
|
|
8898
8814
|
reject(e);
|
|
@@ -8902,41 +8818,40 @@ class AbstractRemote {
|
|
|
8902
8818
|
// The connection is active
|
|
8903
8819
|
if (!connectionEstablished) {
|
|
8904
8820
|
connectionEstablished = true;
|
|
8905
|
-
resolve(
|
|
8821
|
+
resolve(events);
|
|
8906
8822
|
}
|
|
8907
8823
|
const { data } = payload;
|
|
8824
|
+
if (data) {
|
|
8825
|
+
queue.push(data);
|
|
8826
|
+
}
|
|
8908
8827
|
// Less events are now pending
|
|
8909
8828
|
pendingEventsCount--;
|
|
8910
|
-
|
|
8911
|
-
|
|
8912
|
-
}
|
|
8913
|
-
stream.enqueueData(data);
|
|
8829
|
+
// Request another event (unless the downstream consumer is paused).
|
|
8830
|
+
requestMore();
|
|
8914
8831
|
},
|
|
8915
8832
|
onComplete: () => {
|
|
8916
|
-
|
|
8833
|
+
abortRequest(); // this will also emit a done event
|
|
8917
8834
|
},
|
|
8918
8835
|
onExtension: () => { }
|
|
8919
8836
|
});
|
|
8920
8837
|
});
|
|
8921
|
-
const l = stream.registerListener({
|
|
8922
|
-
lowWater: async () => {
|
|
8923
|
-
// Request to fill up the queue
|
|
8924
|
-
const required = syncQueueRequestSize - pendingEventsCount;
|
|
8925
|
-
if (required > 0) {
|
|
8926
|
-
socket.request(syncQueueRequestSize - pendingEventsCount);
|
|
8927
|
-
pendingEventsCount = syncQueueRequestSize;
|
|
8928
|
-
}
|
|
8929
|
-
},
|
|
8930
|
-
closed: () => {
|
|
8931
|
-
l();
|
|
8932
|
-
}
|
|
8933
|
-
});
|
|
8934
|
-
return stream;
|
|
8935
8838
|
}
|
|
8936
8839
|
/**
|
|
8937
|
-
*
|
|
8840
|
+
* @returns Whether the HTTP implementation on this platform can receive streamed binary responses. This is true on
|
|
8841
|
+
* all platforms except React Native (who would have guessed...), where we must not request BSON responses.
|
|
8842
|
+
*
|
|
8843
|
+
* @see https://github.com/react-native-community/fetch?tab=readme-ov-file#motivation
|
|
8844
|
+
*/
|
|
8845
|
+
get supportsStreamingBinaryResponses() {
|
|
8846
|
+
return true;
|
|
8847
|
+
}
|
|
8848
|
+
/**
|
|
8849
|
+
* Posts a `/sync/stream` request, asserts that it completes successfully and returns the streaming response as an
|
|
8850
|
+
* async iterator of byte blobs.
|
|
8851
|
+
*
|
|
8852
|
+
* To cancel the async iterator, use the abort signal from {@link SyncStreamOptions} passed to this method.
|
|
8938
8853
|
*/
|
|
8939
|
-
async
|
|
8854
|
+
async fetchStreamRaw(options) {
|
|
8940
8855
|
const { data, path, headers, abortSignal } = options;
|
|
8941
8856
|
const request = await this.buildRequest(path);
|
|
8942
8857
|
/**
|
|
@@ -8948,119 +8863,94 @@ class AbstractRemote {
|
|
|
8948
8863
|
* Aborting the active fetch request while it is being consumed seems to throw
|
|
8949
8864
|
* an unhandled exception on the window level.
|
|
8950
8865
|
*/
|
|
8951
|
-
if (abortSignal
|
|
8952
|
-
throw new AbortOperation('Abort request received before making
|
|
8866
|
+
if (abortSignal.aborted) {
|
|
8867
|
+
throw new AbortOperation('Abort request received before making fetchStreamRaw request');
|
|
8953
8868
|
}
|
|
8954
8869
|
const controller = new AbortController();
|
|
8955
|
-
let
|
|
8956
|
-
abortSignal
|
|
8957
|
-
|
|
8870
|
+
let reader = null;
|
|
8871
|
+
abortSignal.addEventListener('abort', () => {
|
|
8872
|
+
const reason = abortSignal.reason ??
|
|
8873
|
+
new AbortOperation('Cancelling network request before it resolves. Abort signal has been received.');
|
|
8874
|
+
if (reader == null) {
|
|
8958
8875
|
// Only abort via the abort controller if the request has not resolved yet
|
|
8959
|
-
controller.abort(
|
|
8960
|
-
|
|
8876
|
+
controller.abort(reason);
|
|
8877
|
+
}
|
|
8878
|
+
else {
|
|
8879
|
+
reader.cancel(reason).catch(() => {
|
|
8880
|
+
// Cancelling the reader might rethrow an exception we would have handled by throwing in next(). So we can
|
|
8881
|
+
// ignore it here.
|
|
8882
|
+
});
|
|
8961
8883
|
}
|
|
8962
8884
|
});
|
|
8963
|
-
|
|
8964
|
-
|
|
8965
|
-
|
|
8966
|
-
|
|
8967
|
-
|
|
8968
|
-
|
|
8969
|
-
|
|
8970
|
-
|
|
8971
|
-
|
|
8885
|
+
let res;
|
|
8886
|
+
let responseIsBson = false;
|
|
8887
|
+
try {
|
|
8888
|
+
const ndJson = 'application/x-ndjson';
|
|
8889
|
+
const bson = 'application/vnd.powersync.bson-stream';
|
|
8890
|
+
res = await this.fetch(request.url, {
|
|
8891
|
+
method: 'POST',
|
|
8892
|
+
headers: {
|
|
8893
|
+
...headers,
|
|
8894
|
+
...request.headers,
|
|
8895
|
+
accept: this.supportsStreamingBinaryResponses ? `${bson};q=0.9,${ndJson};q=0.8` : ndJson
|
|
8896
|
+
},
|
|
8897
|
+
body: JSON.stringify(data),
|
|
8898
|
+
signal: controller.signal,
|
|
8899
|
+
cache: 'no-store',
|
|
8900
|
+
...(this.options.fetchOptions ?? {}),
|
|
8901
|
+
...options.fetchOptions
|
|
8902
|
+
});
|
|
8903
|
+
if (!res.ok || !res.body) {
|
|
8904
|
+
const text = await res.text();
|
|
8905
|
+
this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
|
|
8906
|
+
const error = new Error(`HTTP ${res.statusText}: ${text}`);
|
|
8907
|
+
error.status = res.status;
|
|
8908
|
+
throw error;
|
|
8909
|
+
}
|
|
8910
|
+
const contentType = res.headers.get('content-type');
|
|
8911
|
+
responseIsBson = contentType == bson;
|
|
8912
|
+
}
|
|
8913
|
+
catch (ex) {
|
|
8972
8914
|
if (ex.name == 'AbortError') {
|
|
8973
8915
|
throw new AbortOperation(`Pending fetch request to ${request.url} has been aborted.`);
|
|
8974
8916
|
}
|
|
8975
8917
|
throw ex;
|
|
8976
|
-
});
|
|
8977
|
-
if (!res) {
|
|
8978
|
-
throw new Error('Fetch request was aborted');
|
|
8979
|
-
}
|
|
8980
|
-
requestResolved = true;
|
|
8981
|
-
if (!res.ok || !res.body) {
|
|
8982
|
-
const text = await res.text();
|
|
8983
|
-
this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
|
|
8984
|
-
const error = new Error(`HTTP ${res.statusText}: ${text}`);
|
|
8985
|
-
error.status = res.status;
|
|
8986
|
-
throw error;
|
|
8987
8918
|
}
|
|
8988
|
-
|
|
8989
|
-
|
|
8990
|
-
|
|
8991
|
-
|
|
8992
|
-
|
|
8993
|
-
const closeReader = async () => {
|
|
8994
|
-
try {
|
|
8995
|
-
readerReleased = true;
|
|
8996
|
-
await reader.cancel();
|
|
8997
|
-
}
|
|
8998
|
-
catch (ex) {
|
|
8999
|
-
// an error will throw if the reader hasn't been used yet
|
|
9000
|
-
}
|
|
9001
|
-
reader.releaseLock();
|
|
9002
|
-
};
|
|
9003
|
-
const stream = new DataStream({
|
|
9004
|
-
logger: this.logger,
|
|
9005
|
-
mapLine: mapLine,
|
|
9006
|
-
pressure: {
|
|
9007
|
-
highWaterMark: 20,
|
|
9008
|
-
lowWaterMark: 10
|
|
9009
|
-
}
|
|
9010
|
-
});
|
|
9011
|
-
abortSignal?.addEventListener('abort', () => {
|
|
9012
|
-
closeReader();
|
|
9013
|
-
stream.close();
|
|
9014
|
-
});
|
|
9015
|
-
const decoder = this.createTextDecoder();
|
|
9016
|
-
let buffer = '';
|
|
9017
|
-
const consumeStream = async () => {
|
|
9018
|
-
while (!stream.closed && !abortSignal?.aborted && !readerReleased) {
|
|
9019
|
-
const { done, value } = await reader.read();
|
|
9020
|
-
if (done) {
|
|
9021
|
-
const remaining = buffer.trim();
|
|
9022
|
-
if (remaining.length != 0) {
|
|
9023
|
-
stream.enqueueData(remaining);
|
|
9024
|
-
}
|
|
9025
|
-
stream.close();
|
|
9026
|
-
await closeReader();
|
|
9027
|
-
return;
|
|
8919
|
+
reader = res.body.getReader();
|
|
8920
|
+
const stream = {
|
|
8921
|
+
next: async () => {
|
|
8922
|
+
if (controller.signal.aborted) {
|
|
8923
|
+
return doneResult;
|
|
9028
8924
|
}
|
|
9029
|
-
|
|
9030
|
-
|
|
9031
|
-
const lines = buffer.split('\n');
|
|
9032
|
-
for (var i = 0; i < lines.length - 1; i++) {
|
|
9033
|
-
var l = lines[i].trim();
|
|
9034
|
-
if (l.length > 0) {
|
|
9035
|
-
stream.enqueueData(l);
|
|
9036
|
-
}
|
|
8925
|
+
try {
|
|
8926
|
+
return await reader.read();
|
|
9037
8927
|
}
|
|
9038
|
-
|
|
9039
|
-
|
|
9040
|
-
|
|
9041
|
-
|
|
9042
|
-
|
|
9043
|
-
|
|
9044
|
-
|
|
9045
|
-
dispose();
|
|
9046
|
-
},
|
|
9047
|
-
closed: () => {
|
|
9048
|
-
resolve();
|
|
9049
|
-
dispose();
|
|
9050
|
-
}
|
|
9051
|
-
});
|
|
9052
|
-
});
|
|
8928
|
+
catch (ex) {
|
|
8929
|
+
if (controller.signal.aborted) {
|
|
8930
|
+
// .read() completes with an error if we cancel the reader, which we do to disconnect. So this is just
|
|
8931
|
+
// things working as intended, we can return a done event and consider the exception handled.
|
|
8932
|
+
return doneResult;
|
|
8933
|
+
}
|
|
8934
|
+
throw ex;
|
|
9053
8935
|
}
|
|
9054
8936
|
}
|
|
9055
8937
|
};
|
|
9056
|
-
|
|
9057
|
-
|
|
9058
|
-
|
|
9059
|
-
|
|
9060
|
-
|
|
9061
|
-
|
|
9062
|
-
|
|
9063
|
-
|
|
8938
|
+
return { isBson: responseIsBson, stream };
|
|
8939
|
+
}
|
|
8940
|
+
/**
|
|
8941
|
+
* Posts a `/sync/stream` request.
|
|
8942
|
+
*
|
|
8943
|
+
* Depending on the `Content-Type` of the response, this returns strings for sync lines or encoded BSON documents as
|
|
8944
|
+
* {@link Uint8Array}s.
|
|
8945
|
+
*/
|
|
8946
|
+
async fetchStream(options) {
|
|
8947
|
+
const { isBson, stream } = await this.fetchStreamRaw(options);
|
|
8948
|
+
if (isBson) {
|
|
8949
|
+
return extractBsonObjects(stream);
|
|
8950
|
+
}
|
|
8951
|
+
else {
|
|
8952
|
+
return extractJsonLines(stream, this.createTextDecoder());
|
|
8953
|
+
}
|
|
9064
8954
|
}
|
|
9065
8955
|
}
|
|
9066
8956
|
|
|
@@ -9089,31 +8979,8 @@ function coreStatusToJs(status) {
|
|
|
9089
8979
|
priorityStatusEntries: status.priority_status.map(priorityToJs)
|
|
9090
8980
|
};
|
|
9091
8981
|
}
|
|
9092
|
-
|
|
9093
|
-
|
|
9094
|
-
return line.data != null;
|
|
9095
|
-
}
|
|
9096
|
-
function isStreamingKeepalive(line) {
|
|
9097
|
-
return line.token_expires_in != null;
|
|
9098
|
-
}
|
|
9099
|
-
function isStreamingSyncCheckpoint(line) {
|
|
9100
|
-
return line.checkpoint != null;
|
|
9101
|
-
}
|
|
9102
|
-
function isStreamingSyncCheckpointComplete(line) {
|
|
9103
|
-
return line.checkpoint_complete != null;
|
|
9104
|
-
}
|
|
9105
|
-
function isStreamingSyncCheckpointPartiallyComplete(line) {
|
|
9106
|
-
return line.partial_checkpoint_complete != null;
|
|
9107
|
-
}
|
|
9108
|
-
function isStreamingSyncCheckpointDiff(line) {
|
|
9109
|
-
return line.checkpoint_diff != null;
|
|
9110
|
-
}
|
|
9111
|
-
function isContinueCheckpointRequest(request) {
|
|
9112
|
-
return (Array.isArray(request.buckets) &&
|
|
9113
|
-
typeof request.checkpoint_token == 'string');
|
|
9114
|
-
}
|
|
9115
|
-
function isSyncNewCheckpointRequest(request) {
|
|
9116
|
-
return typeof request.request_checkpoint == 'object';
|
|
8982
|
+
function isInterruptingInstruction(instruction) {
|
|
8983
|
+
return 'EstablishSyncStream' in instruction || 'CloseSyncStream' in instruction;
|
|
9117
8984
|
}
|
|
9118
8985
|
|
|
9119
8986
|
exports.LockType = void 0;
|
|
@@ -9128,35 +8995,21 @@ exports.SyncStreamConnectionMethod = void 0;
|
|
|
9128
8995
|
})(exports.SyncStreamConnectionMethod || (exports.SyncStreamConnectionMethod = {}));
|
|
9129
8996
|
exports.SyncClientImplementation = void 0;
|
|
9130
8997
|
(function (SyncClientImplementation) {
|
|
9131
|
-
/**
|
|
9132
|
-
* Decodes and handles sync lines received from the sync service in JavaScript.
|
|
9133
|
-
*
|
|
9134
|
-
* This is the default option.
|
|
9135
|
-
*
|
|
9136
|
-
* @deprecated We recommend the {@link RUST} client implementation for all apps. If you have issues with
|
|
9137
|
-
* the Rust client, please file an issue or reach out to us. The JavaScript client will be removed in a future
|
|
9138
|
-
* version of the PowerSync SDK.
|
|
9139
|
-
*/
|
|
9140
|
-
SyncClientImplementation["JAVASCRIPT"] = "js";
|
|
9141
8998
|
/**
|
|
9142
8999
|
* This implementation offloads the sync line decoding and handling into the PowerSync
|
|
9143
9000
|
* core extension.
|
|
9144
9001
|
*
|
|
9145
|
-
* This
|
|
9146
|
-
* recommended client implementation for all apps.
|
|
9002
|
+
* This is the only option, as an older JavaScript client implementation has been removed from the SDK.
|
|
9147
9003
|
*
|
|
9148
9004
|
* ## Compatibility warning
|
|
9149
9005
|
*
|
|
9150
9006
|
* The Rust sync client stores sync data in a format that is slightly different than the one used
|
|
9151
|
-
* by the old
|
|
9152
|
-
*
|
|
9153
|
-
* Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
|
|
9154
|
-
* the version introducing {@link RUST} as an option) also supports the new format, so you can switch
|
|
9155
|
-
* back to {@link JAVASCRIPT} later.
|
|
9007
|
+
* by the old JavaScript client. When adopting the {@link RUST} client on existing databases, the PowerSync SDK will
|
|
9008
|
+
* migrate the format automatically.
|
|
9156
9009
|
*
|
|
9157
|
-
*
|
|
9158
|
-
*
|
|
9159
|
-
*
|
|
9010
|
+
* SDK versions supporting both the JavaScript and the Rust client support both formats with the JavaScript client
|
|
9011
|
+
* implementaiton. However, downgrading to an SDK version that only supports the JavaScript client would not be
|
|
9012
|
+
* possible anymore. Problematic SDK versions have been released before 2025-06-09.
|
|
9160
9013
|
*/
|
|
9161
9014
|
SyncClientImplementation["RUST"] = "rust";
|
|
9162
9015
|
})(exports.SyncClientImplementation || (exports.SyncClientImplementation = {}));
|
|
@@ -9179,13 +9032,7 @@ const DEFAULT_STREAM_CONNECTION_OPTIONS = {
|
|
|
9179
9032
|
serializedSchema: undefined,
|
|
9180
9033
|
includeDefaultStreams: true
|
|
9181
9034
|
};
|
|
9182
|
-
// The priority we assume when we receive checkpoint lines where no priority is set.
|
|
9183
|
-
// This is the default priority used by the sync service, but can be set to an arbitrary
|
|
9184
|
-
// value since sync services without priorities also won't send partial sync completion
|
|
9185
|
-
// messages.
|
|
9186
|
-
const FALLBACK_PRIORITY = 3;
|
|
9187
9035
|
class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
9188
|
-
_lastSyncedAt;
|
|
9189
9036
|
options;
|
|
9190
9037
|
abortController;
|
|
9191
9038
|
// In rare cases, mostly for tests, uploads can be triggered without being properly connected.
|
|
@@ -9195,6 +9042,7 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
|
9195
9042
|
streamingSyncPromise;
|
|
9196
9043
|
logger;
|
|
9197
9044
|
activeStreams;
|
|
9045
|
+
connectionMayHaveChanged = false;
|
|
9198
9046
|
isUploadingCrud = false;
|
|
9199
9047
|
notifyCompletedUploads;
|
|
9200
9048
|
handleActiveStreamsChange;
|
|
@@ -9274,9 +9122,6 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
|
9274
9122
|
this.crudUpdateListener = undefined;
|
|
9275
9123
|
this.uploadAbortController?.abort();
|
|
9276
9124
|
}
|
|
9277
|
-
async hasCompletedSync() {
|
|
9278
|
-
return this.options.adapter.hasCompletedSync();
|
|
9279
|
-
}
|
|
9280
9125
|
async getWriteCheckpoint() {
|
|
9281
9126
|
const clientId = await this.options.adapter.getClientId();
|
|
9282
9127
|
let path = `/write-checkpoint2.json?client_id=${clientId}`;
|
|
@@ -9358,7 +9203,7 @@ The next upload iteration will be delayed.`);
|
|
|
9358
9203
|
});
|
|
9359
9204
|
}
|
|
9360
9205
|
}
|
|
9361
|
-
this.uploadAbortController =
|
|
9206
|
+
this.uploadAbortController = undefined;
|
|
9362
9207
|
}
|
|
9363
9208
|
});
|
|
9364
9209
|
}
|
|
@@ -9474,6 +9319,11 @@ The next upload iteration will be delayed.`);
|
|
|
9474
9319
|
shouldDelayRetry = false;
|
|
9475
9320
|
// A disconnect was requested, we should not delay since there is no explicit retry
|
|
9476
9321
|
}
|
|
9322
|
+
else if (this.connectionMayHaveChanged && ex.message?.indexOf('No iteration is active') >= 0) {
|
|
9323
|
+
this.connectionMayHaveChanged = false;
|
|
9324
|
+
this.logger.info('Sync error after changed connection, retrying immediately');
|
|
9325
|
+
shouldDelayRetry = false;
|
|
9326
|
+
}
|
|
9477
9327
|
else {
|
|
9478
9328
|
this.logger.error(ex);
|
|
9479
9329
|
}
|
|
@@ -9504,17 +9354,14 @@ The next upload iteration will be delayed.`);
|
|
|
9504
9354
|
// Mark as disconnected if here
|
|
9505
9355
|
this.updateSyncStatus({ connected: false, connecting: false });
|
|
9506
9356
|
}
|
|
9507
|
-
|
|
9508
|
-
|
|
9509
|
-
|
|
9510
|
-
|
|
9511
|
-
|
|
9512
|
-
|
|
9513
|
-
|
|
9514
|
-
|
|
9515
|
-
localDescriptions.set(entry.bucket, null);
|
|
9516
|
-
}
|
|
9517
|
-
return [req, localDescriptions];
|
|
9357
|
+
markConnectionMayHaveChanged() {
|
|
9358
|
+
// By setting this field, we'll immediately retry if the next sync event causes an error triggered by us not having
|
|
9359
|
+
// an active sync iteration on the connection in use.
|
|
9360
|
+
this.connectionMayHaveChanged = true;
|
|
9361
|
+
// This triggers a `powersync_control` invocation if a sync iteration is currently active. This is a cheap call to
|
|
9362
|
+
// make when no subscriptions have actually changed, we're mainly interested in this immediately throwing if no
|
|
9363
|
+
// iteration is active. That allows us to reconnect ASAP, instead of having to wait for the next sync line.
|
|
9364
|
+
this.handleActiveStreamsChange?.();
|
|
9518
9365
|
}
|
|
9519
9366
|
/**
|
|
9520
9367
|
* Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
|
|
@@ -9555,344 +9402,98 @@ The next upload iteration will be delayed.`);
|
|
|
9555
9402
|
if (invalidMetadata.length > 0) {
|
|
9556
9403
|
throw new Error(`Invalid appMetadata provided. Only string values are allowed. Invalid values: ${invalidMetadata.map(([key, value]) => `${key}: ${value}`).join(', ')}`);
|
|
9557
9404
|
}
|
|
9558
|
-
|
|
9559
|
-
this.
|
|
9560
|
-
if (clientImplementation == exports.SyncClientImplementation.JAVASCRIPT) {
|
|
9561
|
-
await this.legacyStreamingSyncIteration(signal, resolvedOptions);
|
|
9562
|
-
return null;
|
|
9563
|
-
}
|
|
9564
|
-
else {
|
|
9565
|
-
await this.requireKeyFormat(true);
|
|
9566
|
-
return await this.rustSyncIteration(signal, resolvedOptions);
|
|
9567
|
-
}
|
|
9405
|
+
await this.requireKeyFormat(true);
|
|
9406
|
+
return await this.rustSyncIteration(signal, resolvedOptions);
|
|
9568
9407
|
}
|
|
9569
9408
|
});
|
|
9570
9409
|
}
|
|
9571
|
-
|
|
9572
|
-
const
|
|
9573
|
-
|
|
9574
|
-
|
|
9575
|
-
|
|
9576
|
-
|
|
9577
|
-
this.logger.error('Sync streams require `clientImplementation: SyncClientImplementation.RUST` when connecting.');
|
|
9578
|
-
}
|
|
9579
|
-
this.logger.debug('Streaming sync iteration started');
|
|
9580
|
-
this.options.adapter.startSession();
|
|
9581
|
-
let [req, bucketMap] = await this.collectLocalBucketState();
|
|
9582
|
-
let targetCheckpoint = null;
|
|
9583
|
-
// A checkpoint that has been validated but not applied (e.g. due to pending local writes)
|
|
9584
|
-
let pendingValidatedCheckpoint = null;
|
|
9585
|
-
const clientId = await this.options.adapter.getClientId();
|
|
9586
|
-
const usingFixedKeyFormat = await this.requireKeyFormat(false);
|
|
9587
|
-
this.logger.debug('Requesting stream from server');
|
|
9588
|
-
const syncOptions = {
|
|
9589
|
-
path: '/sync/stream',
|
|
9590
|
-
abortSignal: signal,
|
|
9591
|
-
data: {
|
|
9592
|
-
buckets: req,
|
|
9593
|
-
include_checksum: true,
|
|
9594
|
-
raw_data: true,
|
|
9595
|
-
parameters: resolvedOptions.params,
|
|
9596
|
-
app_metadata: resolvedOptions.appMetadata,
|
|
9597
|
-
client_id: clientId
|
|
9410
|
+
receiveSyncLines(data) {
|
|
9411
|
+
const { options, connection } = data;
|
|
9412
|
+
const remote = this.options.remote;
|
|
9413
|
+
const openInner = async () => {
|
|
9414
|
+
if (connection.connectionMethod == exports.SyncStreamConnectionMethod.HTTP) {
|
|
9415
|
+
return await remote.fetchStream(options);
|
|
9598
9416
|
}
|
|
9599
|
-
|
|
9600
|
-
|
|
9601
|
-
|
|
9602
|
-
|
|
9603
|
-
|
|
9604
|
-
return JSON.parse(line);
|
|
9605
|
-
}
|
|
9606
|
-
else {
|
|
9607
|
-
// Directly enqueued by us
|
|
9608
|
-
return line;
|
|
9609
|
-
}
|
|
9610
|
-
});
|
|
9611
|
-
}
|
|
9612
|
-
else {
|
|
9613
|
-
const bson = await this.options.remote.getBSON();
|
|
9614
|
-
stream = await this.options.remote.socketStreamRaw({
|
|
9615
|
-
...syncOptions,
|
|
9616
|
-
...{ fetchStrategy: resolvedOptions.fetchStrategy }
|
|
9617
|
-
}, (payload) => {
|
|
9618
|
-
if (payload instanceof Uint8Array) {
|
|
9619
|
-
return bson.deserialize(payload);
|
|
9620
|
-
}
|
|
9621
|
-
else {
|
|
9622
|
-
// Directly enqueued by us
|
|
9623
|
-
return payload;
|
|
9624
|
-
}
|
|
9625
|
-
}, bson);
|
|
9626
|
-
}
|
|
9627
|
-
this.logger.debug('Stream established. Processing events');
|
|
9628
|
-
this.notifyCompletedUploads = () => {
|
|
9629
|
-
if (!stream.closed) {
|
|
9630
|
-
stream.enqueueData({ crud_upload_completed: null });
|
|
9417
|
+
else {
|
|
9418
|
+
return await this.options.remote.socketStreamRaw({
|
|
9419
|
+
...options,
|
|
9420
|
+
...{ fetchStrategy: connection.fetchStrategy }
|
|
9421
|
+
});
|
|
9631
9422
|
}
|
|
9632
9423
|
};
|
|
9633
|
-
|
|
9634
|
-
|
|
9635
|
-
|
|
9636
|
-
|
|
9637
|
-
|
|
9638
|
-
|
|
9639
|
-
if ('crud_upload_completed' in line) {
|
|
9640
|
-
if (pendingValidatedCheckpoint != null) {
|
|
9641
|
-
const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
|
|
9642
|
-
if (applied) {
|
|
9643
|
-
pendingValidatedCheckpoint = null;
|
|
9644
|
-
}
|
|
9645
|
-
else if (endIteration) {
|
|
9646
|
-
break;
|
|
9647
|
-
}
|
|
9424
|
+
let inner;
|
|
9425
|
+
let done = false;
|
|
9426
|
+
return {
|
|
9427
|
+
async next() {
|
|
9428
|
+
if (done) {
|
|
9429
|
+
return doneResult;
|
|
9648
9430
|
}
|
|
9649
|
-
|
|
9650
|
-
|
|
9651
|
-
|
|
9652
|
-
|
|
9653
|
-
|
|
9654
|
-
|
|
9655
|
-
this.updateSyncStatus({
|
|
9656
|
-
connected: true
|
|
9657
|
-
});
|
|
9658
|
-
}
|
|
9659
|
-
if (isStreamingSyncCheckpoint(line)) {
|
|
9660
|
-
targetCheckpoint = line.checkpoint;
|
|
9661
|
-
// New checkpoint - existing validated checkpoint is no longer valid
|
|
9662
|
-
pendingValidatedCheckpoint = null;
|
|
9663
|
-
const bucketsToDelete = new Set(bucketMap.keys());
|
|
9664
|
-
const newBuckets = new Map();
|
|
9665
|
-
for (const checksum of line.checkpoint.buckets) {
|
|
9666
|
-
newBuckets.set(checksum.bucket, {
|
|
9667
|
-
name: checksum.bucket,
|
|
9668
|
-
priority: checksum.priority ?? FALLBACK_PRIORITY
|
|
9431
|
+
else if (inner == null) {
|
|
9432
|
+
inner = await openInner();
|
|
9433
|
+
// We're connected here, so we can tell the core extension about it.
|
|
9434
|
+
return valueResult({
|
|
9435
|
+
command: exports.PowerSyncControlCommand.CONNECTION_STATE,
|
|
9436
|
+
payload: 'established'
|
|
9669
9437
|
});
|
|
9670
|
-
bucketsToDelete.delete(checksum.bucket);
|
|
9671
|
-
}
|
|
9672
|
-
if (bucketsToDelete.size > 0) {
|
|
9673
|
-
this.logger.debug('Removing buckets', [...bucketsToDelete]);
|
|
9674
|
-
}
|
|
9675
|
-
bucketMap = newBuckets;
|
|
9676
|
-
await this.options.adapter.removeBuckets([...bucketsToDelete]);
|
|
9677
|
-
await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
|
|
9678
|
-
await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
|
|
9679
|
-
}
|
|
9680
|
-
else if (isStreamingSyncCheckpointComplete(line)) {
|
|
9681
|
-
const result = await this.applyCheckpoint(targetCheckpoint);
|
|
9682
|
-
if (result.endIteration) {
|
|
9683
|
-
return;
|
|
9684
|
-
}
|
|
9685
|
-
else if (!result.applied) {
|
|
9686
|
-
// "Could not apply checkpoint due to local data". We need to retry after
|
|
9687
|
-
// finishing uploads.
|
|
9688
|
-
pendingValidatedCheckpoint = targetCheckpoint;
|
|
9689
9438
|
}
|
|
9690
9439
|
else {
|
|
9691
|
-
|
|
9692
|
-
|
|
9693
|
-
|
|
9694
|
-
|
|
9695
|
-
}
|
|
9696
|
-
else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
|
|
9697
|
-
const priority = line.partial_checkpoint_complete.priority;
|
|
9698
|
-
this.logger.debug('Partial checkpoint complete', priority);
|
|
9699
|
-
const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint, priority);
|
|
9700
|
-
if (!result.checkpointValid) {
|
|
9701
|
-
// This means checksums failed. Start again with a new checkpoint.
|
|
9702
|
-
// TODO: better back-off
|
|
9703
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
9704
|
-
return;
|
|
9705
|
-
}
|
|
9706
|
-
else if (!result.ready) ;
|
|
9707
|
-
else {
|
|
9708
|
-
// We'll keep on downloading, but can report that this priority is synced now.
|
|
9709
|
-
this.logger.debug('partial checkpoint validation succeeded');
|
|
9710
|
-
// All states with a higher priority can be deleted since this partial sync includes them.
|
|
9711
|
-
const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
|
|
9712
|
-
priorityStates.push({
|
|
9713
|
-
priority,
|
|
9714
|
-
lastSyncedAt: new Date(),
|
|
9715
|
-
hasSynced: true
|
|
9716
|
-
});
|
|
9717
|
-
this.updateSyncStatus({
|
|
9718
|
-
connected: true,
|
|
9719
|
-
priorityStatusEntries: priorityStates
|
|
9720
|
-
});
|
|
9721
|
-
}
|
|
9722
|
-
}
|
|
9723
|
-
else if (isStreamingSyncCheckpointDiff(line)) {
|
|
9724
|
-
// TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
|
|
9725
|
-
if (targetCheckpoint == null) {
|
|
9726
|
-
throw new Error('Checkpoint diff without previous checkpoint');
|
|
9727
|
-
}
|
|
9728
|
-
// New checkpoint - existing validated checkpoint is no longer valid
|
|
9729
|
-
pendingValidatedCheckpoint = null;
|
|
9730
|
-
const diff = line.checkpoint_diff;
|
|
9731
|
-
const newBuckets = new Map();
|
|
9732
|
-
for (const checksum of targetCheckpoint.buckets) {
|
|
9733
|
-
newBuckets.set(checksum.bucket, checksum);
|
|
9734
|
-
}
|
|
9735
|
-
for (const checksum of diff.updated_buckets) {
|
|
9736
|
-
newBuckets.set(checksum.bucket, checksum);
|
|
9737
|
-
}
|
|
9738
|
-
for (const bucket of diff.removed_buckets) {
|
|
9739
|
-
newBuckets.delete(bucket);
|
|
9740
|
-
}
|
|
9741
|
-
const newCheckpoint = {
|
|
9742
|
-
last_op_id: diff.last_op_id,
|
|
9743
|
-
buckets: [...newBuckets.values()],
|
|
9744
|
-
write_checkpoint: diff.write_checkpoint
|
|
9745
|
-
};
|
|
9746
|
-
targetCheckpoint = newCheckpoint;
|
|
9747
|
-
await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
|
|
9748
|
-
bucketMap = new Map();
|
|
9749
|
-
newBuckets.forEach((checksum, name) => bucketMap.set(name, {
|
|
9750
|
-
name: checksum.bucket,
|
|
9751
|
-
priority: checksum.priority ?? FALLBACK_PRIORITY
|
|
9752
|
-
}));
|
|
9753
|
-
const bucketsToDelete = diff.removed_buckets;
|
|
9754
|
-
if (bucketsToDelete.length > 0) {
|
|
9755
|
-
this.logger.debug('Remove buckets', bucketsToDelete);
|
|
9756
|
-
}
|
|
9757
|
-
await this.options.adapter.removeBuckets(bucketsToDelete);
|
|
9758
|
-
await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
|
|
9759
|
-
}
|
|
9760
|
-
else if (isStreamingSyncData(line)) {
|
|
9761
|
-
const { data } = line;
|
|
9762
|
-
const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
|
|
9763
|
-
let updatedProgress = null;
|
|
9764
|
-
if (previousProgress) {
|
|
9765
|
-
updatedProgress = { ...previousProgress };
|
|
9766
|
-
const progressForBucket = updatedProgress[data.bucket];
|
|
9767
|
-
if (progressForBucket) {
|
|
9768
|
-
updatedProgress[data.bucket] = {
|
|
9769
|
-
...progressForBucket,
|
|
9770
|
-
since_last: progressForBucket.since_last + data.data.length
|
|
9771
|
-
};
|
|
9440
|
+
const event = await inner.next();
|
|
9441
|
+
if (event.done) {
|
|
9442
|
+
done = true;
|
|
9443
|
+
return valueResult({ command: exports.PowerSyncControlCommand.CONNECTION_STATE, payload: 'end' });
|
|
9772
9444
|
}
|
|
9773
|
-
|
|
9774
|
-
|
|
9775
|
-
|
|
9776
|
-
|
|
9777
|
-
|
|
9445
|
+
else {
|
|
9446
|
+
return valueResult({
|
|
9447
|
+
command: typeof event.value == 'string'
|
|
9448
|
+
? exports.PowerSyncControlCommand.PROCESS_TEXT_LINE
|
|
9449
|
+
: exports.PowerSyncControlCommand.PROCESS_BSON_LINE,
|
|
9450
|
+
payload: event.value
|
|
9451
|
+
});
|
|
9778
9452
|
}
|
|
9779
|
-
});
|
|
9780
|
-
await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] }, usingFixedKeyFormat);
|
|
9781
|
-
}
|
|
9782
|
-
else if (isStreamingKeepalive(line)) {
|
|
9783
|
-
const remaining_seconds = line.token_expires_in;
|
|
9784
|
-
if (remaining_seconds == 0) {
|
|
9785
|
-
// Connection would be closed automatically right after this
|
|
9786
|
-
this.logger.debug('Token expiring; reconnect');
|
|
9787
|
-
/**
|
|
9788
|
-
* For a rare case where the backend connector does not update the token
|
|
9789
|
-
* (uses the same one), this should have some delay.
|
|
9790
|
-
*/
|
|
9791
|
-
await this.delayRetry();
|
|
9792
|
-
return;
|
|
9793
|
-
}
|
|
9794
|
-
else if (remaining_seconds < 30) {
|
|
9795
|
-
this.logger.debug('Token will expire soon; reconnect');
|
|
9796
|
-
// Pre-emptively refresh the token
|
|
9797
|
-
this.options.remote.invalidateCredentials();
|
|
9798
|
-
return;
|
|
9799
9453
|
}
|
|
9800
|
-
this.triggerCrudUpload();
|
|
9801
|
-
}
|
|
9802
|
-
else {
|
|
9803
|
-
this.logger.debug('Received unknown sync line', line);
|
|
9804
9454
|
}
|
|
9805
|
-
}
|
|
9806
|
-
this.logger.debug('Stream input empty');
|
|
9807
|
-
// Connection closed. Likely due to auth issue.
|
|
9808
|
-
return;
|
|
9455
|
+
};
|
|
9809
9456
|
}
|
|
9810
9457
|
async rustSyncIteration(signal, resolvedOptions) {
|
|
9811
9458
|
const syncImplementation = this;
|
|
9812
9459
|
const adapter = this.options.adapter;
|
|
9813
9460
|
const remote = this.options.remote;
|
|
9814
|
-
let receivingLines = null;
|
|
9815
|
-
let hadSyncLine = false;
|
|
9816
9461
|
let hideDisconnectOnRestart = false;
|
|
9462
|
+
let notifyTokenRefreshed;
|
|
9817
9463
|
if (signal.aborted) {
|
|
9818
9464
|
throw new AbortOperation('Connection request has been aborted');
|
|
9819
9465
|
}
|
|
9820
|
-
|
|
9821
|
-
|
|
9822
|
-
|
|
9823
|
-
|
|
9824
|
-
|
|
9825
|
-
|
|
9826
|
-
async function connect(instr) {
|
|
9827
|
-
const syncOptions = {
|
|
9828
|
-
path: '/sync/stream',
|
|
9829
|
-
abortSignal: abortController.signal,
|
|
9830
|
-
data: instr.request
|
|
9466
|
+
function startCommand() {
|
|
9467
|
+
const options = {
|
|
9468
|
+
parameters: resolvedOptions.params,
|
|
9469
|
+
app_metadata: resolvedOptions.appMetadata,
|
|
9470
|
+
active_streams: syncImplementation.activeStreams,
|
|
9471
|
+
include_defaults: resolvedOptions.includeDefaultStreams
|
|
9831
9472
|
};
|
|
9832
|
-
if (resolvedOptions.
|
|
9833
|
-
|
|
9834
|
-
if (typeof line == 'string') {
|
|
9835
|
-
return {
|
|
9836
|
-
command: exports.PowerSyncControlCommand.PROCESS_TEXT_LINE,
|
|
9837
|
-
payload: line
|
|
9838
|
-
};
|
|
9839
|
-
}
|
|
9840
|
-
else {
|
|
9841
|
-
// Directly enqueued by us
|
|
9842
|
-
return line;
|
|
9843
|
-
}
|
|
9844
|
-
});
|
|
9845
|
-
}
|
|
9846
|
-
else {
|
|
9847
|
-
controlInvocations = await remote.socketStreamRaw({
|
|
9848
|
-
...syncOptions,
|
|
9849
|
-
fetchStrategy: resolvedOptions.fetchStrategy
|
|
9850
|
-
}, (payload) => {
|
|
9851
|
-
if (payload instanceof Uint8Array) {
|
|
9852
|
-
return {
|
|
9853
|
-
command: exports.PowerSyncControlCommand.PROCESS_BSON_LINE,
|
|
9854
|
-
payload: payload
|
|
9855
|
-
};
|
|
9856
|
-
}
|
|
9857
|
-
else {
|
|
9858
|
-
// Directly enqueued by us
|
|
9859
|
-
return payload;
|
|
9860
|
-
}
|
|
9861
|
-
});
|
|
9862
|
-
}
|
|
9863
|
-
// The rust client will set connected: true after the first sync line because that's when it gets invoked, but
|
|
9864
|
-
// we're already connected here and can report that.
|
|
9865
|
-
syncImplementation.updateSyncStatus({ connected: true });
|
|
9866
|
-
try {
|
|
9867
|
-
while (!controlInvocations.closed) {
|
|
9868
|
-
const line = await controlInvocations.read();
|
|
9869
|
-
if (line == null) {
|
|
9870
|
-
return;
|
|
9871
|
-
}
|
|
9872
|
-
await control(line.command, line.payload);
|
|
9873
|
-
if (!hadSyncLine) {
|
|
9874
|
-
syncImplementation.triggerCrudUpload();
|
|
9875
|
-
hadSyncLine = true;
|
|
9876
|
-
}
|
|
9877
|
-
}
|
|
9878
|
-
}
|
|
9879
|
-
finally {
|
|
9880
|
-
const activeInstructions = controlInvocations;
|
|
9881
|
-
// We concurrently add events to the active data stream when e.g. a CRUD upload is completed or a token is
|
|
9882
|
-
// refreshed. That would throw after closing (and we can't handle those events either way), so set this back
|
|
9883
|
-
// to null.
|
|
9884
|
-
controlInvocations = null;
|
|
9885
|
-
await activeInstructions.close();
|
|
9473
|
+
if (resolvedOptions.serializedSchema) {
|
|
9474
|
+
options.schema = resolvedOptions.serializedSchema;
|
|
9886
9475
|
}
|
|
9476
|
+
return invokePowerSyncControl(exports.PowerSyncControlCommand.START, JSON.stringify(options));
|
|
9887
9477
|
}
|
|
9888
9478
|
async function stop() {
|
|
9889
|
-
await
|
|
9479
|
+
const instructions = await invokePowerSyncControl(exports.PowerSyncControlCommand.STOP);
|
|
9480
|
+
for (const instruction of instructions) {
|
|
9481
|
+
// We don't need to handle interrupting instructions since we're unconditionally ending the sync iteration at
|
|
9482
|
+
// this point.
|
|
9483
|
+
if (isInterruptingInstruction(instruction))
|
|
9484
|
+
continue;
|
|
9485
|
+
await handleInstruction(instruction);
|
|
9486
|
+
}
|
|
9890
9487
|
}
|
|
9891
|
-
async function
|
|
9488
|
+
async function invokePowerSyncControl(op, payload) {
|
|
9892
9489
|
const rawResponse = await adapter.control(op, payload ?? null);
|
|
9893
9490
|
const logger = syncImplementation.logger;
|
|
9894
9491
|
logger.trace('powersync_control', op, payload == null || typeof payload == 'string' ? payload : '<bytes>', rawResponse);
|
|
9895
|
-
|
|
9492
|
+
if (op != exports.PowerSyncControlCommand.STOP) {
|
|
9493
|
+
// Evidently we have a working connection here, otherwise powersync_control would have failed.
|
|
9494
|
+
syncImplementation.connectionMayHaveChanged = false;
|
|
9495
|
+
}
|
|
9496
|
+
return JSON.parse(rawResponse);
|
|
9896
9497
|
}
|
|
9897
9498
|
async function handleInstruction(instruction) {
|
|
9898
9499
|
if ('LogLine' in instruction) {
|
|
@@ -9911,13 +9512,6 @@ The next upload iteration will be delayed.`);
|
|
|
9911
9512
|
else if ('UpdateSyncStatus' in instruction) {
|
|
9912
9513
|
syncImplementation.updateSyncStatus(coreStatusToJs(instruction.UpdateSyncStatus.status));
|
|
9913
9514
|
}
|
|
9914
|
-
else if ('EstablishSyncStream' in instruction) {
|
|
9915
|
-
if (receivingLines != null) {
|
|
9916
|
-
// Already connected, this shouldn't happen during a single iteration.
|
|
9917
|
-
throw 'Unexpected request to establish sync stream, already connected';
|
|
9918
|
-
}
|
|
9919
|
-
receivingLines = connect(instruction.EstablishSyncStream);
|
|
9920
|
-
}
|
|
9921
9515
|
else if ('FetchCredentials' in instruction) {
|
|
9922
9516
|
if (instruction.FetchCredentials.did_expire) {
|
|
9923
9517
|
remote.invalidateCredentials();
|
|
@@ -9926,16 +9520,12 @@ The next upload iteration will be delayed.`);
|
|
|
9926
9520
|
remote.invalidateCredentials();
|
|
9927
9521
|
// Restart iteration after the credentials have been refreshed.
|
|
9928
9522
|
remote.fetchCredentials().then((_) => {
|
|
9929
|
-
|
|
9523
|
+
notifyTokenRefreshed?.();
|
|
9930
9524
|
}, (err) => {
|
|
9931
9525
|
syncImplementation.logger.warn('Could not prefetch credentials', err);
|
|
9932
9526
|
});
|
|
9933
9527
|
}
|
|
9934
9528
|
}
|
|
9935
|
-
else if ('CloseSyncStream' in instruction) {
|
|
9936
|
-
abortController.abort();
|
|
9937
|
-
hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
|
|
9938
|
-
}
|
|
9939
9529
|
else if ('FlushFileSystem' in instruction) ;
|
|
9940
9530
|
else if ('DidCompleteSync' in instruction) {
|
|
9941
9531
|
syncImplementation.updateSyncStatus({
|
|
@@ -9945,105 +9535,83 @@ The next upload iteration will be delayed.`);
|
|
|
9945
9535
|
});
|
|
9946
9536
|
}
|
|
9947
9537
|
}
|
|
9948
|
-
async function handleInstructions(instructions) {
|
|
9949
|
-
for (const instr of instructions) {
|
|
9950
|
-
await handleInstruction(instr);
|
|
9951
|
-
}
|
|
9952
|
-
}
|
|
9953
9538
|
try {
|
|
9954
|
-
const
|
|
9955
|
-
|
|
9956
|
-
|
|
9957
|
-
|
|
9958
|
-
|
|
9959
|
-
|
|
9960
|
-
|
|
9961
|
-
|
|
9539
|
+
const defaultResult = { immediateRestart: false };
|
|
9540
|
+
// Pending sync lines received from the service, as well as local events that trigger a powersync_control
|
|
9541
|
+
// invocation (local events include refreshed tokens and completed uploads).
|
|
9542
|
+
// This is a single data stream so that we can handle all control calls from a single place.
|
|
9543
|
+
let controlInvocations = null;
|
|
9544
|
+
for (const startInstruction of await startCommand()) {
|
|
9545
|
+
if ('EstablishSyncStream' in startInstruction) {
|
|
9546
|
+
const syncOptions = {
|
|
9547
|
+
path: '/sync/stream',
|
|
9548
|
+
abortSignal: signal,
|
|
9549
|
+
data: startInstruction.EstablishSyncStream.request
|
|
9550
|
+
};
|
|
9551
|
+
controlInvocations = injectable(syncImplementation.receiveSyncLines({
|
|
9552
|
+
options: syncOptions,
|
|
9553
|
+
connection: resolvedOptions
|
|
9554
|
+
}));
|
|
9555
|
+
}
|
|
9556
|
+
else if ('CloseSyncStream' in startInstruction) {
|
|
9557
|
+
return defaultResult;
|
|
9558
|
+
}
|
|
9559
|
+
else {
|
|
9560
|
+
await handleInstruction(startInstruction);
|
|
9561
|
+
}
|
|
9962
9562
|
}
|
|
9963
|
-
|
|
9563
|
+
if (controlInvocations == null)
|
|
9564
|
+
return defaultResult;
|
|
9964
9565
|
this.notifyCompletedUploads = () => {
|
|
9965
|
-
|
|
9966
|
-
controlInvocations.enqueueData({ command: exports.PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
|
|
9967
|
-
}
|
|
9566
|
+
controlInvocations.inject({ command: exports.PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
|
|
9968
9567
|
};
|
|
9969
9568
|
this.handleActiveStreamsChange = () => {
|
|
9970
|
-
|
|
9971
|
-
|
|
9972
|
-
|
|
9973
|
-
|
|
9974
|
-
});
|
|
9975
|
-
}
|
|
9569
|
+
controlInvocations.inject({
|
|
9570
|
+
command: exports.PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
|
|
9571
|
+
payload: JSON.stringify(this.activeStreams)
|
|
9572
|
+
});
|
|
9976
9573
|
};
|
|
9977
|
-
|
|
9574
|
+
notifyTokenRefreshed = () => {
|
|
9575
|
+
controlInvocations.inject({
|
|
9576
|
+
command: exports.PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED
|
|
9577
|
+
});
|
|
9578
|
+
};
|
|
9579
|
+
let hadSyncLine = false;
|
|
9580
|
+
loop: while (true) {
|
|
9581
|
+
const { done, value } = await controlInvocations.next();
|
|
9582
|
+
if (done)
|
|
9583
|
+
break;
|
|
9584
|
+
if (!hadSyncLine) {
|
|
9585
|
+
// Trigger a local CRUD upload when the first sync line has been received, this allows uploading local changes
|
|
9586
|
+
// that have been made while offline or disconnected.
|
|
9587
|
+
if (value.command == exports.PowerSyncControlCommand.PROCESS_TEXT_LINE ||
|
|
9588
|
+
value.command == exports.PowerSyncControlCommand.PROCESS_BSON_LINE) {
|
|
9589
|
+
hadSyncLine = true;
|
|
9590
|
+
this.triggerCrudUpload?.();
|
|
9591
|
+
}
|
|
9592
|
+
}
|
|
9593
|
+
const instructions = await invokePowerSyncControl(value.command, value.payload);
|
|
9594
|
+
for (const instruction of instructions) {
|
|
9595
|
+
if ('EstablishSyncStream' in instruction) {
|
|
9596
|
+
throw new Error('Received EstablishSyncStream while already connected.');
|
|
9597
|
+
}
|
|
9598
|
+
else if ('CloseSyncStream' in instruction) {
|
|
9599
|
+
hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
|
|
9600
|
+
break loop;
|
|
9601
|
+
}
|
|
9602
|
+
else {
|
|
9603
|
+
await handleInstruction(instruction);
|
|
9604
|
+
}
|
|
9605
|
+
}
|
|
9606
|
+
}
|
|
9978
9607
|
}
|
|
9979
9608
|
finally {
|
|
9980
9609
|
this.notifyCompletedUploads = this.handleActiveStreamsChange = undefined;
|
|
9610
|
+
notifyTokenRefreshed = undefined;
|
|
9981
9611
|
await stop();
|
|
9982
9612
|
}
|
|
9983
9613
|
return { immediateRestart: hideDisconnectOnRestart };
|
|
9984
9614
|
}
|
|
9985
|
-
async updateSyncStatusForStartingCheckpoint(checkpoint) {
|
|
9986
|
-
const localProgress = await this.options.adapter.getBucketOperationProgress();
|
|
9987
|
-
const progress = {};
|
|
9988
|
-
let invalidated = false;
|
|
9989
|
-
for (const bucket of checkpoint.buckets) {
|
|
9990
|
-
const savedProgress = localProgress[bucket.bucket];
|
|
9991
|
-
const atLast = savedProgress?.atLast ?? 0;
|
|
9992
|
-
const sinceLast = savedProgress?.sinceLast ?? 0;
|
|
9993
|
-
progress[bucket.bucket] = {
|
|
9994
|
-
// The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
|
|
9995
|
-
// will use by default.
|
|
9996
|
-
priority: bucket.priority ?? 3,
|
|
9997
|
-
at_last: atLast,
|
|
9998
|
-
since_last: sinceLast,
|
|
9999
|
-
target_count: bucket.count ?? 0
|
|
10000
|
-
};
|
|
10001
|
-
if (bucket.count != null && bucket.count < atLast + sinceLast) {
|
|
10002
|
-
// Either due to a defrag / sync rule deploy or a compaction operation, the size
|
|
10003
|
-
// of the bucket shrank so much that the local ops exceed the ops in the updated
|
|
10004
|
-
// bucket. We can't prossibly report progress in this case (it would overshoot 100%).
|
|
10005
|
-
invalidated = true;
|
|
10006
|
-
}
|
|
10007
|
-
}
|
|
10008
|
-
if (invalidated) {
|
|
10009
|
-
for (const bucket in progress) {
|
|
10010
|
-
const bucketProgress = progress[bucket];
|
|
10011
|
-
bucketProgress.at_last = 0;
|
|
10012
|
-
bucketProgress.since_last = 0;
|
|
10013
|
-
}
|
|
10014
|
-
}
|
|
10015
|
-
this.updateSyncStatus({
|
|
10016
|
-
dataFlow: {
|
|
10017
|
-
downloading: true,
|
|
10018
|
-
downloadProgress: progress
|
|
10019
|
-
}
|
|
10020
|
-
});
|
|
10021
|
-
}
|
|
10022
|
-
async applyCheckpoint(checkpoint) {
|
|
10023
|
-
let result = await this.options.adapter.syncLocalDatabase(checkpoint);
|
|
10024
|
-
if (!result.checkpointValid) {
|
|
10025
|
-
this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
|
|
10026
|
-
// This means checksums failed. Start again with a new checkpoint.
|
|
10027
|
-
// TODO: better back-off
|
|
10028
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
10029
|
-
return { applied: false, endIteration: true };
|
|
10030
|
-
}
|
|
10031
|
-
else if (!result.ready) {
|
|
10032
|
-
this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`);
|
|
10033
|
-
return { applied: false, endIteration: false };
|
|
10034
|
-
}
|
|
10035
|
-
this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
|
|
10036
|
-
this.updateSyncStatus({
|
|
10037
|
-
connected: true,
|
|
10038
|
-
lastSyncedAt: new Date(),
|
|
10039
|
-
dataFlow: {
|
|
10040
|
-
downloading: false,
|
|
10041
|
-
downloadProgress: null,
|
|
10042
|
-
downloadError: undefined
|
|
10043
|
-
}
|
|
10044
|
-
});
|
|
10045
|
-
return { applied: true, endIteration: false };
|
|
10046
|
-
}
|
|
10047
9615
|
updateSyncStatus(options) {
|
|
10048
9616
|
const updatedStatus = new SyncStatus({
|
|
10049
9617
|
connected: options.connected ?? this.syncStatus.connected,
|
|
@@ -10989,7 +10557,7 @@ class AbstractPowerSyncDatabase extends BaseObserver {
|
|
|
10989
10557
|
* @returns A transaction of CRUD operations to upload, or null if there are none
|
|
10990
10558
|
*/
|
|
10991
10559
|
async getNextCrudTransaction() {
|
|
10992
|
-
const iterator = this.getCrudTransactions()[
|
|
10560
|
+
const iterator = this.getCrudTransactions()[Symbol.asyncIterator]();
|
|
10993
10561
|
return (await iterator.next()).value;
|
|
10994
10562
|
}
|
|
10995
10563
|
/**
|
|
@@ -11025,7 +10593,7 @@ class AbstractPowerSyncDatabase extends BaseObserver {
|
|
|
11025
10593
|
*/
|
|
11026
10594
|
getCrudTransactions() {
|
|
11027
10595
|
return {
|
|
11028
|
-
[
|
|
10596
|
+
[Symbol.asyncIterator]: () => {
|
|
11029
10597
|
let lastCrudItemId = -1;
|
|
11030
10598
|
const sql = `
|
|
11031
10599
|
WITH RECURSIVE crud_entries AS (
|
|
@@ -11582,14 +11150,12 @@ class SqliteBucketStorage extends BaseObserver {
|
|
|
11582
11150
|
db;
|
|
11583
11151
|
logger;
|
|
11584
11152
|
tableNames;
|
|
11585
|
-
_hasCompletedSync;
|
|
11586
11153
|
updateListener;
|
|
11587
11154
|
_clientId;
|
|
11588
11155
|
constructor(db, logger = Logger.get('SqliteBucketStorage')) {
|
|
11589
11156
|
super();
|
|
11590
11157
|
this.db = db;
|
|
11591
11158
|
this.logger = logger;
|
|
11592
|
-
this._hasCompletedSync = false;
|
|
11593
11159
|
this.tableNames = new Set();
|
|
11594
11160
|
this.updateListener = db.registerListener({
|
|
11595
11161
|
tablesUpdated: (update) => {
|
|
@@ -11601,7 +11167,6 @@ class SqliteBucketStorage extends BaseObserver {
|
|
|
11601
11167
|
});
|
|
11602
11168
|
}
|
|
11603
11169
|
async init() {
|
|
11604
|
-
this._hasCompletedSync = false;
|
|
11605
11170
|
const existingTableRows = await this.db.getAll(`SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'`);
|
|
11606
11171
|
for (const row of existingTableRows ?? []) {
|
|
11607
11172
|
this.tableNames.add(row.name);
|
|
@@ -11623,156 +11188,6 @@ class SqliteBucketStorage extends BaseObserver {
|
|
|
11623
11188
|
getMaxOpId() {
|
|
11624
11189
|
return MAX_OP_ID;
|
|
11625
11190
|
}
|
|
11626
|
-
/**
|
|
11627
|
-
* Reset any caches.
|
|
11628
|
-
*/
|
|
11629
|
-
startSession() { }
|
|
11630
|
-
async getBucketStates() {
|
|
11631
|
-
const result = await this.db.getAll("SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != '$local'");
|
|
11632
|
-
return result;
|
|
11633
|
-
}
|
|
11634
|
-
async getBucketOperationProgress() {
|
|
11635
|
-
const rows = await this.db.getAll('SELECT name, count_at_last, count_since_last FROM ps_buckets');
|
|
11636
|
-
return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
|
|
11637
|
-
}
|
|
11638
|
-
async saveSyncData(batch, fixedKeyFormat = false) {
|
|
11639
|
-
await this.writeTransaction(async (tx) => {
|
|
11640
|
-
for (const b of batch.buckets) {
|
|
11641
|
-
await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
|
|
11642
|
-
'save',
|
|
11643
|
-
JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
|
|
11644
|
-
]);
|
|
11645
|
-
this.logger.debug(`Saved batch of data for bucket: ${b.bucket}, operations: ${b.data.length}`);
|
|
11646
|
-
}
|
|
11647
|
-
});
|
|
11648
|
-
}
|
|
11649
|
-
async removeBuckets(buckets) {
|
|
11650
|
-
for (const bucket of buckets) {
|
|
11651
|
-
await this.deleteBucket(bucket);
|
|
11652
|
-
}
|
|
11653
|
-
}
|
|
11654
|
-
/**
|
|
11655
|
-
* Mark a bucket for deletion.
|
|
11656
|
-
*/
|
|
11657
|
-
async deleteBucket(bucket) {
|
|
11658
|
-
await this.writeTransaction(async (tx) => {
|
|
11659
|
-
await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
|
|
11660
|
-
});
|
|
11661
|
-
this.logger.debug(`Done deleting bucket ${bucket}`);
|
|
11662
|
-
}
|
|
11663
|
-
async hasCompletedSync() {
|
|
11664
|
-
if (this._hasCompletedSync) {
|
|
11665
|
-
return true;
|
|
11666
|
-
}
|
|
11667
|
-
const r = await this.db.get(`SELECT powersync_last_synced_at() as synced_at`);
|
|
11668
|
-
const completed = r.synced_at != null;
|
|
11669
|
-
if (completed) {
|
|
11670
|
-
this._hasCompletedSync = true;
|
|
11671
|
-
}
|
|
11672
|
-
return completed;
|
|
11673
|
-
}
|
|
11674
|
-
async syncLocalDatabase(checkpoint, priority) {
|
|
11675
|
-
const r = await this.validateChecksums(checkpoint, priority);
|
|
11676
|
-
if (!r.checkpointValid) {
|
|
11677
|
-
this.logger.error('Checksums failed for', r.checkpointFailures);
|
|
11678
|
-
for (const b of r.checkpointFailures ?? []) {
|
|
11679
|
-
await this.deleteBucket(b);
|
|
11680
|
-
}
|
|
11681
|
-
return { ready: false, checkpointValid: false, checkpointFailures: r.checkpointFailures };
|
|
11682
|
-
}
|
|
11683
|
-
if (priority == null) {
|
|
11684
|
-
this.logger.debug(`Validated checksums checkpoint ${checkpoint.last_op_id}`);
|
|
11685
|
-
}
|
|
11686
|
-
else {
|
|
11687
|
-
this.logger.debug(`Validated checksums for partial checkpoint ${checkpoint.last_op_id}, priority ${priority}`);
|
|
11688
|
-
}
|
|
11689
|
-
let buckets = checkpoint.buckets;
|
|
11690
|
-
if (priority !== undefined) {
|
|
11691
|
-
buckets = buckets.filter((b) => hasMatchingPriority(priority, b));
|
|
11692
|
-
}
|
|
11693
|
-
const bucketNames = buckets.map((b) => b.bucket);
|
|
11694
|
-
await this.writeTransaction(async (tx) => {
|
|
11695
|
-
await tx.execute(`UPDATE ps_buckets SET last_op = ? WHERE name IN (SELECT json_each.value FROM json_each(?))`, [
|
|
11696
|
-
checkpoint.last_op_id,
|
|
11697
|
-
JSON.stringify(bucketNames)
|
|
11698
|
-
]);
|
|
11699
|
-
if (priority == null && checkpoint.write_checkpoint) {
|
|
11700
|
-
await tx.execute("UPDATE ps_buckets SET last_op = ? WHERE name = '$local'", [checkpoint.write_checkpoint]);
|
|
11701
|
-
}
|
|
11702
|
-
});
|
|
11703
|
-
const valid = await this.updateObjectsFromBuckets(checkpoint, priority);
|
|
11704
|
-
if (!valid) {
|
|
11705
|
-
return { ready: false, checkpointValid: true };
|
|
11706
|
-
}
|
|
11707
|
-
return {
|
|
11708
|
-
ready: true,
|
|
11709
|
-
checkpointValid: true
|
|
11710
|
-
};
|
|
11711
|
-
}
|
|
11712
|
-
/**
|
|
11713
|
-
* Atomically update the local state to the current checkpoint.
|
|
11714
|
-
*
|
|
11715
|
-
* This includes creating new tables, dropping old tables, and copying data over from the oplog.
|
|
11716
|
-
*/
|
|
11717
|
-
async updateObjectsFromBuckets(checkpoint, priority) {
|
|
11718
|
-
let arg = '';
|
|
11719
|
-
if (priority !== undefined) {
|
|
11720
|
-
const affectedBuckets = [];
|
|
11721
|
-
for (const desc of checkpoint.buckets) {
|
|
11722
|
-
if (hasMatchingPriority(priority, desc)) {
|
|
11723
|
-
affectedBuckets.push(desc.bucket);
|
|
11724
|
-
}
|
|
11725
|
-
}
|
|
11726
|
-
arg = JSON.stringify({ priority, buckets: affectedBuckets });
|
|
11727
|
-
}
|
|
11728
|
-
return this.writeTransaction(async (tx) => {
|
|
11729
|
-
const { insertId: result } = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
|
|
11730
|
-
'sync_local',
|
|
11731
|
-
arg
|
|
11732
|
-
]);
|
|
11733
|
-
if (result == 1) {
|
|
11734
|
-
if (priority == null) {
|
|
11735
|
-
const bucketToCount = Object.fromEntries(checkpoint.buckets.map((b) => [b.bucket, b.count]));
|
|
11736
|
-
// The two parameters could be replaced with one, but: https://github.com/powersync-ja/better-sqlite3/pull/6
|
|
11737
|
-
const jsonBucketCount = JSON.stringify(bucketToCount);
|
|
11738
|
-
await tx.execute("UPDATE ps_buckets SET count_since_last = 0, count_at_last = ?->name WHERE name != '$local' AND ?->name IS NOT NULL", [jsonBucketCount, jsonBucketCount]);
|
|
11739
|
-
}
|
|
11740
|
-
return true;
|
|
11741
|
-
}
|
|
11742
|
-
else {
|
|
11743
|
-
return false;
|
|
11744
|
-
}
|
|
11745
|
-
});
|
|
11746
|
-
}
|
|
11747
|
-
async validateChecksums(checkpoint, priority) {
|
|
11748
|
-
if (priority !== undefined) {
|
|
11749
|
-
// Only validate the buckets within the priority we care about
|
|
11750
|
-
const newBuckets = checkpoint.buckets.filter((cs) => hasMatchingPriority(priority, cs));
|
|
11751
|
-
checkpoint = { ...checkpoint, buckets: newBuckets };
|
|
11752
|
-
}
|
|
11753
|
-
const rs = await this.db.execute('SELECT powersync_validate_checkpoint(?) as result', [
|
|
11754
|
-
JSON.stringify({ ...checkpoint })
|
|
11755
|
-
]);
|
|
11756
|
-
const resultItem = rs.rows?.item(0);
|
|
11757
|
-
if (!resultItem) {
|
|
11758
|
-
return {
|
|
11759
|
-
checkpointValid: false,
|
|
11760
|
-
ready: false,
|
|
11761
|
-
checkpointFailures: []
|
|
11762
|
-
};
|
|
11763
|
-
}
|
|
11764
|
-
const result = JSON.parse(resultItem['result']);
|
|
11765
|
-
if (result['valid']) {
|
|
11766
|
-
return { ready: true, checkpointValid: true };
|
|
11767
|
-
}
|
|
11768
|
-
else {
|
|
11769
|
-
return {
|
|
11770
|
-
checkpointValid: false,
|
|
11771
|
-
ready: false,
|
|
11772
|
-
checkpointFailures: result['failed_buckets']
|
|
11773
|
-
};
|
|
11774
|
-
}
|
|
11775
|
-
}
|
|
11776
11191
|
async updateLocalTarget(cb) {
|
|
11777
11192
|
const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", [MAX_OP_ID]);
|
|
11778
11193
|
if (!rs1.length) {
|
|
@@ -11863,12 +11278,6 @@ class SqliteBucketStorage extends BaseObserver {
|
|
|
11863
11278
|
async writeTransaction(callback, options) {
|
|
11864
11279
|
return this.db.writeTransaction(callback, options);
|
|
11865
11280
|
}
|
|
11866
|
-
/**
|
|
11867
|
-
* Set a target checkpoint.
|
|
11868
|
-
*/
|
|
11869
|
-
async setTargetCheckpoint(checkpoint) {
|
|
11870
|
-
// No-op for now
|
|
11871
|
-
}
|
|
11872
11281
|
async control(op, payload) {
|
|
11873
11282
|
return await this.writeTransaction(async (tx) => {
|
|
11874
11283
|
const [[raw]] = await tx.executeRaw('SELECT powersync_control(?, ?)', [op, payload]);
|
|
@@ -11892,20 +11301,6 @@ class SqliteBucketStorage extends BaseObserver {
|
|
|
11892
11301
|
}
|
|
11893
11302
|
static _subkeyMigrationKey = 'powersync_js_migrated_subkeys';
|
|
11894
11303
|
}
|
|
11895
|
-
function hasMatchingPriority(priority, bucket) {
|
|
11896
|
-
return bucket.priority != null && bucket.priority <= priority;
|
|
11897
|
-
}
|
|
11898
|
-
|
|
11899
|
-
// TODO JSON
|
|
11900
|
-
class SyncDataBatch {
|
|
11901
|
-
buckets;
|
|
11902
|
-
static fromJSON(json) {
|
|
11903
|
-
return new SyncDataBatch(json.buckets.map((bucket) => SyncDataBucket.fromRow(bucket)));
|
|
11904
|
-
}
|
|
11905
|
-
constructor(buckets) {
|
|
11906
|
-
this.buckets = buckets;
|
|
11907
|
-
}
|
|
11908
|
-
}
|
|
11909
11304
|
|
|
11910
11305
|
/**
|
|
11911
11306
|
* Thrown when an underlying database connection is closed.
|
|
@@ -11965,10 +11360,8 @@ class Schema {
|
|
|
11965
11360
|
* developer instead of automatically by PowerSync.
|
|
11966
11361
|
* Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
|
|
11967
11362
|
* using client-side table and column constraints.
|
|
11968
|
-
* Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
|
|
11969
11363
|
*
|
|
11970
11364
|
* @param tables An object of (table name, raw table definition) entries.
|
|
11971
|
-
* @experimental Note that the raw tables API is still experimental and may change in the future.
|
|
11972
11365
|
*/
|
|
11973
11366
|
withRawTables(tables) {
|
|
11974
11367
|
for (const [name, rawTableDefinition] of Object.entries(tables)) {
|
|
@@ -12194,7 +11587,6 @@ exports.DEFAULT_INDEX_OPTIONS = DEFAULT_INDEX_OPTIONS;
|
|
|
12194
11587
|
exports.DEFAULT_LOCK_TIMEOUT_MS = DEFAULT_LOCK_TIMEOUT_MS;
|
|
12195
11588
|
exports.DEFAULT_POWERSYNC_CLOSE_OPTIONS = DEFAULT_POWERSYNC_CLOSE_OPTIONS;
|
|
12196
11589
|
exports.DEFAULT_POWERSYNC_DB_OPTIONS = DEFAULT_POWERSYNC_DB_OPTIONS;
|
|
12197
|
-
exports.DEFAULT_PRESSURE_LIMITS = DEFAULT_PRESSURE_LIMITS;
|
|
12198
11590
|
exports.DEFAULT_REMOTE_LOGGER = DEFAULT_REMOTE_LOGGER;
|
|
12199
11591
|
exports.DEFAULT_REMOTE_OPTIONS = DEFAULT_REMOTE_OPTIONS;
|
|
12200
11592
|
exports.DEFAULT_RETRY_DELAY_MS = DEFAULT_RETRY_DELAY_MS;
|
|
@@ -12205,7 +11597,6 @@ exports.DEFAULT_SYNC_CLIENT_IMPLEMENTATION = DEFAULT_SYNC_CLIENT_IMPLEMENTATION;
|
|
|
12205
11597
|
exports.DEFAULT_TABLE_OPTIONS = DEFAULT_TABLE_OPTIONS;
|
|
12206
11598
|
exports.DEFAULT_WATCH_QUERY_OPTIONS = DEFAULT_WATCH_QUERY_OPTIONS;
|
|
12207
11599
|
exports.DEFAULT_WATCH_THROTTLE_MS = DEFAULT_WATCH_THROTTLE_MS;
|
|
12208
|
-
exports.DataStream = DataStream;
|
|
12209
11600
|
exports.DifferentialQueryProcessor = DifferentialQueryProcessor;
|
|
12210
11601
|
exports.EMPTY_DIFFERENTIAL = EMPTY_DIFFERENTIAL;
|
|
12211
11602
|
exports.FalsyComparator = FalsyComparator;
|
|
@@ -12220,13 +11611,9 @@ exports.MAX_OP_ID = MAX_OP_ID;
|
|
|
12220
11611
|
exports.MEMORY_TRIGGER_CLAIM_MANAGER = MEMORY_TRIGGER_CLAIM_MANAGER;
|
|
12221
11612
|
exports.Mutex = Mutex;
|
|
12222
11613
|
exports.OnChangeQueryProcessor = OnChangeQueryProcessor;
|
|
12223
|
-
exports.OpType = OpType;
|
|
12224
|
-
exports.OplogEntry = OplogEntry;
|
|
12225
11614
|
exports.Schema = Schema;
|
|
12226
11615
|
exports.Semaphore = Semaphore;
|
|
12227
11616
|
exports.SqliteBucketStorage = SqliteBucketStorage;
|
|
12228
|
-
exports.SyncDataBatch = SyncDataBatch;
|
|
12229
|
-
exports.SyncDataBucket = SyncDataBucket;
|
|
12230
11617
|
exports.SyncProgress = SyncProgress;
|
|
12231
11618
|
exports.SyncStatus = SyncStatus;
|
|
12232
11619
|
exports.SyncingService = SyncingService;
|
|
@@ -12241,18 +11628,10 @@ exports.createBaseLogger = createBaseLogger;
|
|
|
12241
11628
|
exports.createLogger = createLogger;
|
|
12242
11629
|
exports.extractTableUpdates = extractTableUpdates;
|
|
12243
11630
|
exports.isBatchedUpdateNotification = isBatchedUpdateNotification;
|
|
12244
|
-
exports.isContinueCheckpointRequest = isContinueCheckpointRequest;
|
|
12245
11631
|
exports.isDBAdapter = isDBAdapter;
|
|
12246
11632
|
exports.isPowerSyncDatabaseOptionsWithSettings = isPowerSyncDatabaseOptionsWithSettings;
|
|
12247
11633
|
exports.isSQLOpenFactory = isSQLOpenFactory;
|
|
12248
11634
|
exports.isSQLOpenOptions = isSQLOpenOptions;
|
|
12249
|
-
exports.isStreamingKeepalive = isStreamingKeepalive;
|
|
12250
|
-
exports.isStreamingSyncCheckpoint = isStreamingSyncCheckpoint;
|
|
12251
|
-
exports.isStreamingSyncCheckpointComplete = isStreamingSyncCheckpointComplete;
|
|
12252
|
-
exports.isStreamingSyncCheckpointDiff = isStreamingSyncCheckpointDiff;
|
|
12253
|
-
exports.isStreamingSyncCheckpointPartiallyComplete = isStreamingSyncCheckpointPartiallyComplete;
|
|
12254
|
-
exports.isStreamingSyncData = isStreamingSyncData;
|
|
12255
|
-
exports.isSyncNewCheckpointRequest = isSyncNewCheckpointRequest;
|
|
12256
11635
|
exports.parseQuery = parseQuery;
|
|
12257
11636
|
exports.runOnSchemaChange = runOnSchemaChange;
|
|
12258
11637
|
exports.sanitizeSQL = sanitizeSQL;
|