@powersync/common 1.51.0 → 1.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.cjs +510 -1129
- package/dist/bundle.cjs.map +1 -1
- package/dist/bundle.mjs +511 -1116
- package/dist/bundle.mjs.map +1 -1
- package/dist/bundle.node.cjs +508 -1129
- package/dist/bundle.node.cjs.map +1 -1
- package/dist/bundle.node.mjs +509 -1116
- package/dist/bundle.node.mjs.map +1 -1
- package/dist/index.d.cts +73 -433
- package/legacy/sync_protocol.d.ts +103 -0
- package/lib/client/AbstractPowerSyncDatabase.js +3 -3
- package/lib/client/AbstractPowerSyncDatabase.js.map +1 -1
- package/lib/client/ConnectionManager.js +1 -1
- package/lib/client/ConnectionManager.js.map +1 -1
- package/lib/client/sync/bucket/BucketStorageAdapter.d.ts +6 -64
- package/lib/client/sync/bucket/BucketStorageAdapter.js +4 -0
- package/lib/client/sync/bucket/BucketStorageAdapter.js.map +1 -1
- package/lib/client/sync/bucket/SqliteBucketStorage.d.ts +1 -28
- package/lib/client/sync/bucket/SqliteBucketStorage.js +0 -162
- package/lib/client/sync/bucket/SqliteBucketStorage.js.map +1 -1
- package/lib/client/sync/stream/AbstractRemote.d.ts +29 -18
- package/lib/client/sync/stream/AbstractRemote.js +155 -188
- package/lib/client/sync/stream/AbstractRemote.js.map +1 -1
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +13 -35
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +150 -448
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js.map +1 -1
- package/lib/client/sync/stream/JsonValue.d.ts +7 -0
- package/lib/client/sync/stream/JsonValue.js +2 -0
- package/lib/client/sync/stream/JsonValue.js.map +1 -0
- package/lib/client/sync/stream/core-instruction.d.ts +14 -9
- package/lib/client/sync/stream/core-instruction.js +3 -0
- package/lib/client/sync/stream/core-instruction.js.map +1 -1
- package/lib/db/DBAdapter.d.ts +9 -0
- package/lib/db/DBAdapter.js +8 -1
- package/lib/db/DBAdapter.js.map +1 -1
- package/lib/db/crud/SyncStatus.d.ts +3 -4
- package/lib/db/crud/SyncStatus.js +0 -4
- package/lib/db/crud/SyncStatus.js.map +1 -1
- package/lib/db/schema/RawTable.d.ts +0 -5
- package/lib/db/schema/Schema.d.ts +0 -2
- package/lib/db/schema/Schema.js +0 -2
- package/lib/db/schema/Schema.js.map +1 -1
- package/lib/index.d.ts +2 -6
- package/lib/index.js +1 -6
- package/lib/index.js.map +1 -1
- package/lib/utils/async.d.ts +0 -9
- package/lib/utils/async.js +0 -9
- package/lib/utils/async.js.map +1 -1
- package/lib/utils/stream_transform.d.ts +39 -0
- package/lib/utils/stream_transform.js +206 -0
- package/lib/utils/stream_transform.js.map +1 -0
- package/package.json +15 -10
- package/src/client/AbstractPowerSyncDatabase.ts +3 -3
- package/src/client/ConnectionManager.ts +1 -1
- package/src/client/sync/bucket/BucketStorageAdapter.ts +6 -71
- package/src/client/sync/bucket/SqliteBucketStorage.ts +1 -197
- package/src/client/sync/stream/AbstractRemote.ts +183 -229
- package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +181 -510
- package/src/client/sync/stream/JsonValue.ts +8 -0
- package/src/client/sync/stream/core-instruction.ts +15 -5
- package/src/db/DBAdapter.ts +20 -2
- package/src/db/crud/SyncStatus.ts +4 -5
- package/src/db/schema/RawTable.ts +0 -5
- package/src/db/schema/Schema.ts +0 -2
- package/src/index.ts +2 -6
- package/src/utils/async.ts +0 -11
- package/src/utils/stream_transform.ts +252 -0
- package/lib/client/sync/bucket/OpType.d.ts +0 -16
- package/lib/client/sync/bucket/OpType.js +0 -23
- package/lib/client/sync/bucket/OpType.js.map +0 -1
- package/lib/client/sync/bucket/OplogEntry.d.ts +0 -23
- package/lib/client/sync/bucket/OplogEntry.js +0 -36
- package/lib/client/sync/bucket/OplogEntry.js.map +0 -1
- package/lib/client/sync/bucket/SyncDataBatch.d.ts +0 -6
- package/lib/client/sync/bucket/SyncDataBatch.js +0 -12
- package/lib/client/sync/bucket/SyncDataBatch.js.map +0 -1
- package/lib/client/sync/bucket/SyncDataBucket.d.ts +0 -40
- package/lib/client/sync/bucket/SyncDataBucket.js +0 -40
- package/lib/client/sync/bucket/SyncDataBucket.js.map +0 -1
- package/lib/client/sync/stream/streaming-sync-types.d.ts +0 -143
- package/lib/client/sync/stream/streaming-sync-types.js +0 -26
- package/lib/client/sync/stream/streaming-sync-types.js.map +0 -1
- package/lib/utils/DataStream.d.ts +0 -62
- package/lib/utils/DataStream.js +0 -169
- package/lib/utils/DataStream.js.map +0 -1
- package/src/client/sync/bucket/OpType.ts +0 -23
- package/src/client/sync/bucket/OplogEntry.ts +0 -50
- package/src/client/sync/bucket/SyncDataBatch.ts +0 -11
- package/src/client/sync/bucket/SyncDataBucket.ts +0 -49
- package/src/client/sync/stream/streaming-sync-types.ts +0 -210
- package/src/utils/DataStream.ts +0 -222
package/dist/bundle.node.mjs
CHANGED
|
@@ -1817,8 +1817,15 @@ class BaseTransaction {
|
|
|
1817
1817
|
class TransactionImplementation extends DBGetUtilsDefaultMixin(BaseTransaction) {
|
|
1818
1818
|
static async runWith(ctx, fn) {
|
|
1819
1819
|
let tx = new TransactionImplementation(ctx);
|
|
1820
|
+
// For write transactions, use BEGIN IMMEDIATE to immediately obtain a write lock on the database (instead of doing
|
|
1821
|
+
// that on the first statement). If we have a genuine read-only connection, we also use BEGIN IMMEDIATE there: In
|
|
1822
|
+
// WAL mode, that ensures we pin the current state of the database (instead of the state at the first statement in
|
|
1823
|
+
// the transaction). But if we have a "fake" read-only connection implemented through `pragma query_only = true`, we
|
|
1824
|
+
// can't use this trick because it would attempt to lock the connection. So there, we use a regular `BEGIN`
|
|
1825
|
+
// statement.
|
|
1826
|
+
const useBeginImmediate = ctx.connectionType != 'queryOnly';
|
|
1820
1827
|
try {
|
|
1821
|
-
await ctx.execute('BEGIN IMMEDIATE');
|
|
1828
|
+
await ctx.execute(useBeginImmediate ? 'BEGIN IMMEDIATE' : 'BEGIN');
|
|
1822
1829
|
const result = await fn(tx);
|
|
1823
1830
|
await tx.commit();
|
|
1824
1831
|
return result;
|
|
@@ -1977,16 +1984,12 @@ class SyncStatus {
|
|
|
1977
1984
|
*
|
|
1978
1985
|
* This returns null when the database is currently being opened and we don't have reliable information about all
|
|
1979
1986
|
* included streams yet.
|
|
1980
|
-
*
|
|
1981
|
-
* @experimental Sync streams are currently in alpha.
|
|
1982
1987
|
*/
|
|
1983
1988
|
get syncStreams() {
|
|
1984
1989
|
return this.options.dataFlow?.internalStreamSubscriptions?.map((core) => new SyncStreamStatusView(this, core));
|
|
1985
1990
|
}
|
|
1986
1991
|
/**
|
|
1987
1992
|
* If the `stream` appears in {@link syncStreams}, returns the current status for that stream.
|
|
1988
|
-
*
|
|
1989
|
-
* @experimental Sync streams are currently in alpha.
|
|
1990
1993
|
*/
|
|
1991
1994
|
forStream(stream) {
|
|
1992
1995
|
const asJson = JSON.stringify(stream.parameters);
|
|
@@ -2255,15 +2258,6 @@ class ControlledExecutor {
|
|
|
2255
2258
|
}
|
|
2256
2259
|
}
|
|
2257
2260
|
|
|
2258
|
-
/**
|
|
2259
|
-
* A ponyfill for `Symbol.asyncIterator` that is compatible with the
|
|
2260
|
-
* [recommended polyfill](https://github.com/Azure/azure-sdk-for-js/blob/%40azure/core-asynciterator-polyfill_1.0.2/sdk/core/core-asynciterator-polyfill/src/index.ts#L4-L6)
|
|
2261
|
-
* we recommend for React Native.
|
|
2262
|
-
*
|
|
2263
|
-
* As long as we use this symbol (instead of `for await` and `async *`) in this package, we can be compatible with async
|
|
2264
|
-
* iterators without requiring them.
|
|
2265
|
-
*/
|
|
2266
|
-
const symbolAsyncIterator = Symbol.asyncIterator ?? Symbol.for('Symbol.asyncIterator');
|
|
2267
2261
|
/**
|
|
2268
2262
|
* Throttle a function to be called at most once every "wait" milliseconds,
|
|
2269
2263
|
* on the trailing edge.
|
|
@@ -2578,7 +2572,7 @@ class SyncStreamSubscriptionHandle {
|
|
|
2578
2572
|
constructor(subscription) {
|
|
2579
2573
|
this.subscription = subscription;
|
|
2580
2574
|
subscription.refcount++;
|
|
2581
|
-
_finalizer?.register(this, subscription);
|
|
2575
|
+
_finalizer?.register(this, subscription, this);
|
|
2582
2576
|
}
|
|
2583
2577
|
get name() {
|
|
2584
2578
|
return this.subscription.name;
|
|
@@ -3167,6 +3161,10 @@ var PowerSyncControlCommand;
|
|
|
3167
3161
|
PowerSyncControlCommand["NOTIFY_TOKEN_REFRESHED"] = "refreshed_token";
|
|
3168
3162
|
PowerSyncControlCommand["NOTIFY_CRUD_UPLOAD_COMPLETED"] = "completed_upload";
|
|
3169
3163
|
PowerSyncControlCommand["UPDATE_SUBSCRIPTIONS"] = "update_subscriptions";
|
|
3164
|
+
/**
|
|
3165
|
+
* An `established` or `end` event for response streams.
|
|
3166
|
+
*/
|
|
3167
|
+
PowerSyncControlCommand["CONNECTION_STATE"] = "connection";
|
|
3170
3168
|
})(PowerSyncControlCommand || (PowerSyncControlCommand = {}));
|
|
3171
3169
|
|
|
3172
3170
|
/**
|
|
@@ -3348,103 +3346,6 @@ class AbortOperation extends Error {
|
|
|
3348
3346
|
}
|
|
3349
3347
|
}
|
|
3350
3348
|
|
|
3351
|
-
var OpTypeEnum;
|
|
3352
|
-
(function (OpTypeEnum) {
|
|
3353
|
-
OpTypeEnum[OpTypeEnum["CLEAR"] = 1] = "CLEAR";
|
|
3354
|
-
OpTypeEnum[OpTypeEnum["MOVE"] = 2] = "MOVE";
|
|
3355
|
-
OpTypeEnum[OpTypeEnum["PUT"] = 3] = "PUT";
|
|
3356
|
-
OpTypeEnum[OpTypeEnum["REMOVE"] = 4] = "REMOVE";
|
|
3357
|
-
})(OpTypeEnum || (OpTypeEnum = {}));
|
|
3358
|
-
/**
|
|
3359
|
-
* Used internally for sync buckets.
|
|
3360
|
-
*/
|
|
3361
|
-
class OpType {
|
|
3362
|
-
value;
|
|
3363
|
-
static fromJSON(jsonValue) {
|
|
3364
|
-
return new OpType(OpTypeEnum[jsonValue]);
|
|
3365
|
-
}
|
|
3366
|
-
constructor(value) {
|
|
3367
|
-
this.value = value;
|
|
3368
|
-
}
|
|
3369
|
-
toJSON() {
|
|
3370
|
-
return Object.entries(OpTypeEnum).find(([, value]) => value === this.value)[0];
|
|
3371
|
-
}
|
|
3372
|
-
}
|
|
3373
|
-
|
|
3374
|
-
class OplogEntry {
|
|
3375
|
-
op_id;
|
|
3376
|
-
op;
|
|
3377
|
-
checksum;
|
|
3378
|
-
subkey;
|
|
3379
|
-
object_type;
|
|
3380
|
-
object_id;
|
|
3381
|
-
data;
|
|
3382
|
-
static fromRow(row) {
|
|
3383
|
-
return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, row.subkey, row.object_type, row.object_id, row.data);
|
|
3384
|
-
}
|
|
3385
|
-
constructor(op_id, op, checksum, subkey, object_type, object_id, data) {
|
|
3386
|
-
this.op_id = op_id;
|
|
3387
|
-
this.op = op;
|
|
3388
|
-
this.checksum = checksum;
|
|
3389
|
-
this.subkey = subkey;
|
|
3390
|
-
this.object_type = object_type;
|
|
3391
|
-
this.object_id = object_id;
|
|
3392
|
-
this.data = data;
|
|
3393
|
-
}
|
|
3394
|
-
toJSON(fixedKeyEncoding = false) {
|
|
3395
|
-
return {
|
|
3396
|
-
op_id: this.op_id,
|
|
3397
|
-
op: this.op.toJSON(),
|
|
3398
|
-
object_type: this.object_type,
|
|
3399
|
-
object_id: this.object_id,
|
|
3400
|
-
checksum: this.checksum,
|
|
3401
|
-
data: this.data,
|
|
3402
|
-
// Older versions of the JS SDK used to always JSON.stringify here. That has always been wrong,
|
|
3403
|
-
// but we need to migrate gradually to not break existing databases.
|
|
3404
|
-
subkey: fixedKeyEncoding ? this.subkey : JSON.stringify(this.subkey)
|
|
3405
|
-
};
|
|
3406
|
-
}
|
|
3407
|
-
}
|
|
3408
|
-
|
|
3409
|
-
class SyncDataBucket {
|
|
3410
|
-
bucket;
|
|
3411
|
-
data;
|
|
3412
|
-
has_more;
|
|
3413
|
-
after;
|
|
3414
|
-
next_after;
|
|
3415
|
-
static fromRow(row) {
|
|
3416
|
-
return new SyncDataBucket(row.bucket, row.data.map((entry) => OplogEntry.fromRow(entry)), row.has_more ?? false, row.after, row.next_after);
|
|
3417
|
-
}
|
|
3418
|
-
constructor(bucket, data,
|
|
3419
|
-
/**
|
|
3420
|
-
* True if the response does not contain all the data for this bucket, and another request must be made.
|
|
3421
|
-
*/
|
|
3422
|
-
has_more,
|
|
3423
|
-
/**
|
|
3424
|
-
* The `after` specified in the request.
|
|
3425
|
-
*/
|
|
3426
|
-
after,
|
|
3427
|
-
/**
|
|
3428
|
-
* Use this for the next request.
|
|
3429
|
-
*/
|
|
3430
|
-
next_after) {
|
|
3431
|
-
this.bucket = bucket;
|
|
3432
|
-
this.data = data;
|
|
3433
|
-
this.has_more = has_more;
|
|
3434
|
-
this.after = after;
|
|
3435
|
-
this.next_after = next_after;
|
|
3436
|
-
}
|
|
3437
|
-
toJSON(fixedKeyEncoding = false) {
|
|
3438
|
-
return {
|
|
3439
|
-
bucket: this.bucket,
|
|
3440
|
-
has_more: this.has_more,
|
|
3441
|
-
after: this.after,
|
|
3442
|
-
next_after: this.next_after,
|
|
3443
|
-
data: this.data.map((entry) => entry.toJSON(fixedKeyEncoding))
|
|
3444
|
-
};
|
|
3445
|
-
}
|
|
3446
|
-
}
|
|
3447
|
-
|
|
3448
3349
|
var dist = {};
|
|
3449
3350
|
|
|
3450
3351
|
var Codecs = {};
|
|
@@ -8229,177 +8130,10 @@ function requireDist () {
|
|
|
8229
8130
|
|
|
8230
8131
|
var distExports = requireDist();
|
|
8231
8132
|
|
|
8232
|
-
var version = "1.
|
|
8133
|
+
var version = "1.53.0";
|
|
8233
8134
|
var PACKAGE = {
|
|
8234
8135
|
version: version};
|
|
8235
8136
|
|
|
8236
|
-
const DEFAULT_PRESSURE_LIMITS = {
|
|
8237
|
-
highWater: 10,
|
|
8238
|
-
lowWater: 0
|
|
8239
|
-
};
|
|
8240
|
-
/**
|
|
8241
|
-
* A very basic implementation of a data stream with backpressure support which does not use
|
|
8242
|
-
* native JS streams or async iterators.
|
|
8243
|
-
* This is handy for environments such as React Native which need polyfills for the above.
|
|
8244
|
-
*/
|
|
8245
|
-
class DataStream extends BaseObserver {
|
|
8246
|
-
options;
|
|
8247
|
-
dataQueue;
|
|
8248
|
-
isClosed;
|
|
8249
|
-
processingPromise;
|
|
8250
|
-
notifyDataAdded;
|
|
8251
|
-
logger;
|
|
8252
|
-
mapLine;
|
|
8253
|
-
constructor(options) {
|
|
8254
|
-
super();
|
|
8255
|
-
this.options = options;
|
|
8256
|
-
this.processingPromise = null;
|
|
8257
|
-
this.isClosed = false;
|
|
8258
|
-
this.dataQueue = [];
|
|
8259
|
-
this.mapLine = options?.mapLine ?? ((line) => line);
|
|
8260
|
-
this.logger = options?.logger ?? Logger.get('DataStream');
|
|
8261
|
-
if (options?.closeOnError) {
|
|
8262
|
-
const l = this.registerListener({
|
|
8263
|
-
error: (ex) => {
|
|
8264
|
-
l?.();
|
|
8265
|
-
this.close();
|
|
8266
|
-
}
|
|
8267
|
-
});
|
|
8268
|
-
}
|
|
8269
|
-
}
|
|
8270
|
-
get highWatermark() {
|
|
8271
|
-
return this.options?.pressure?.highWaterMark ?? DEFAULT_PRESSURE_LIMITS.highWater;
|
|
8272
|
-
}
|
|
8273
|
-
get lowWatermark() {
|
|
8274
|
-
return this.options?.pressure?.lowWaterMark ?? DEFAULT_PRESSURE_LIMITS.lowWater;
|
|
8275
|
-
}
|
|
8276
|
-
get closed() {
|
|
8277
|
-
return this.isClosed;
|
|
8278
|
-
}
|
|
8279
|
-
async close() {
|
|
8280
|
-
this.isClosed = true;
|
|
8281
|
-
await this.processingPromise;
|
|
8282
|
-
this.iterateListeners((l) => l.closed?.());
|
|
8283
|
-
// Discard any data in the queue
|
|
8284
|
-
this.dataQueue = [];
|
|
8285
|
-
this.listeners.clear();
|
|
8286
|
-
}
|
|
8287
|
-
/**
|
|
8288
|
-
* Enqueues data for the consumers to read
|
|
8289
|
-
*/
|
|
8290
|
-
enqueueData(data) {
|
|
8291
|
-
if (this.isClosed) {
|
|
8292
|
-
throw new Error('Cannot enqueue data into closed stream.');
|
|
8293
|
-
}
|
|
8294
|
-
this.dataQueue.push(data);
|
|
8295
|
-
this.notifyDataAdded?.();
|
|
8296
|
-
this.processQueue();
|
|
8297
|
-
}
|
|
8298
|
-
/**
|
|
8299
|
-
* Reads data once from the data stream
|
|
8300
|
-
* @returns a Data payload or Null if the stream closed.
|
|
8301
|
-
*/
|
|
8302
|
-
async read() {
|
|
8303
|
-
if (this.closed) {
|
|
8304
|
-
return null;
|
|
8305
|
-
}
|
|
8306
|
-
// Wait for any pending processing to complete first.
|
|
8307
|
-
// This ensures we register our listener before calling processQueue(),
|
|
8308
|
-
// avoiding a race where processQueue() sees no reader and returns early.
|
|
8309
|
-
if (this.processingPromise) {
|
|
8310
|
-
await this.processingPromise;
|
|
8311
|
-
}
|
|
8312
|
-
// Re-check after await - stream may have closed while we were waiting
|
|
8313
|
-
if (this.closed) {
|
|
8314
|
-
return null;
|
|
8315
|
-
}
|
|
8316
|
-
return new Promise((resolve, reject) => {
|
|
8317
|
-
const l = this.registerListener({
|
|
8318
|
-
data: async (data) => {
|
|
8319
|
-
resolve(data);
|
|
8320
|
-
// Remove the listener
|
|
8321
|
-
l?.();
|
|
8322
|
-
},
|
|
8323
|
-
closed: () => {
|
|
8324
|
-
resolve(null);
|
|
8325
|
-
l?.();
|
|
8326
|
-
},
|
|
8327
|
-
error: (ex) => {
|
|
8328
|
-
reject(ex);
|
|
8329
|
-
l?.();
|
|
8330
|
-
}
|
|
8331
|
-
});
|
|
8332
|
-
this.processQueue();
|
|
8333
|
-
});
|
|
8334
|
-
}
|
|
8335
|
-
/**
|
|
8336
|
-
* Executes a callback for each data item in the stream
|
|
8337
|
-
*/
|
|
8338
|
-
forEach(callback) {
|
|
8339
|
-
if (this.dataQueue.length <= this.lowWatermark) {
|
|
8340
|
-
this.iterateAsyncErrored(async (l) => l.lowWater?.());
|
|
8341
|
-
}
|
|
8342
|
-
return this.registerListener({
|
|
8343
|
-
data: callback
|
|
8344
|
-
});
|
|
8345
|
-
}
|
|
8346
|
-
processQueue() {
|
|
8347
|
-
if (this.processingPromise) {
|
|
8348
|
-
return;
|
|
8349
|
-
}
|
|
8350
|
-
const promise = (this.processingPromise = this._processQueue());
|
|
8351
|
-
promise.finally(() => {
|
|
8352
|
-
this.processingPromise = null;
|
|
8353
|
-
});
|
|
8354
|
-
return promise;
|
|
8355
|
-
}
|
|
8356
|
-
hasDataReader() {
|
|
8357
|
-
return Array.from(this.listeners.values()).some((l) => !!l.data);
|
|
8358
|
-
}
|
|
8359
|
-
async _processQueue() {
|
|
8360
|
-
/**
|
|
8361
|
-
* Allow listeners to mutate the queue before processing.
|
|
8362
|
-
* This allows for operations such as dropping or compressing data
|
|
8363
|
-
* on high water or requesting more data on low water.
|
|
8364
|
-
*/
|
|
8365
|
-
if (this.dataQueue.length >= this.highWatermark) {
|
|
8366
|
-
await this.iterateAsyncErrored(async (l) => l.highWater?.());
|
|
8367
|
-
}
|
|
8368
|
-
if (this.isClosed || !this.hasDataReader()) {
|
|
8369
|
-
return;
|
|
8370
|
-
}
|
|
8371
|
-
if (this.dataQueue.length) {
|
|
8372
|
-
const data = this.dataQueue.shift();
|
|
8373
|
-
const mapped = this.mapLine(data);
|
|
8374
|
-
await this.iterateAsyncErrored(async (l) => l.data?.(mapped));
|
|
8375
|
-
}
|
|
8376
|
-
if (this.dataQueue.length <= this.lowWatermark) {
|
|
8377
|
-
const dataAdded = new Promise((resolve) => {
|
|
8378
|
-
this.notifyDataAdded = resolve;
|
|
8379
|
-
});
|
|
8380
|
-
await Promise.race([this.iterateAsyncErrored(async (l) => l.lowWater?.()), dataAdded]);
|
|
8381
|
-
this.notifyDataAdded = null;
|
|
8382
|
-
}
|
|
8383
|
-
if (this.dataQueue.length > 0) {
|
|
8384
|
-
setTimeout(() => this.processQueue());
|
|
8385
|
-
}
|
|
8386
|
-
}
|
|
8387
|
-
async iterateAsyncErrored(cb) {
|
|
8388
|
-
// Important: We need to copy the listeners, as calling a listener could result in adding another
|
|
8389
|
-
// listener, resulting in infinite loops.
|
|
8390
|
-
const listeners = Array.from(this.listeners.values());
|
|
8391
|
-
for (let i of listeners) {
|
|
8392
|
-
try {
|
|
8393
|
-
await cb(i);
|
|
8394
|
-
}
|
|
8395
|
-
catch (ex) {
|
|
8396
|
-
this.logger.error(ex);
|
|
8397
|
-
this.iterateListeners((l) => l.error?.(ex));
|
|
8398
|
-
}
|
|
8399
|
-
}
|
|
8400
|
-
}
|
|
8401
|
-
}
|
|
8402
|
-
|
|
8403
8137
|
var WebsocketDuplexConnection = {};
|
|
8404
8138
|
|
|
8405
8139
|
var hasRequiredWebsocketDuplexConnection;
|
|
@@ -8562,8 +8296,199 @@ class WebsocketClientTransport {
|
|
|
8562
8296
|
}
|
|
8563
8297
|
}
|
|
8564
8298
|
|
|
8299
|
+
const doneResult = { done: true, value: undefined };
|
|
8300
|
+
function valueResult(value) {
|
|
8301
|
+
return { done: false, value };
|
|
8302
|
+
}
|
|
8303
|
+
/**
|
|
8304
|
+
* Expands a source async iterator by allowing to inject events asynchronously.
|
|
8305
|
+
*
|
|
8306
|
+
* The resulting iterator will emit all events from its source. Additionally though, events can be injected. These
|
|
8307
|
+
* events are dropped once the main iterator completes, but are otherwise forwarded.
|
|
8308
|
+
*
|
|
8309
|
+
* The iterator completes when its source completes, and it supports backpressure by only calling `next()` on the source
|
|
8310
|
+
* in response to a `next()` call from downstream if no pending injected events can be dispatched.
|
|
8311
|
+
*/
|
|
8312
|
+
function injectable(source) {
|
|
8313
|
+
let sourceIsDone = false;
|
|
8314
|
+
let waiter = undefined; // An active, waiting next() call.
|
|
8315
|
+
// A pending upstream event that couldn't be dispatched because inject() has been called before it was resolved.
|
|
8316
|
+
let pendingSourceEvent = null;
|
|
8317
|
+
let pendingInjectedEvents = [];
|
|
8318
|
+
const consumeWaiter = () => {
|
|
8319
|
+
const pending = waiter;
|
|
8320
|
+
waiter = undefined;
|
|
8321
|
+
return pending;
|
|
8322
|
+
};
|
|
8323
|
+
const fetchFromSource = () => {
|
|
8324
|
+
const resolveWaiter = (propagate) => {
|
|
8325
|
+
const active = consumeWaiter();
|
|
8326
|
+
if (active) {
|
|
8327
|
+
propagate(active);
|
|
8328
|
+
}
|
|
8329
|
+
else {
|
|
8330
|
+
pendingSourceEvent = propagate;
|
|
8331
|
+
}
|
|
8332
|
+
};
|
|
8333
|
+
const nextFromSource = source.next();
|
|
8334
|
+
nextFromSource.then((value) => {
|
|
8335
|
+
sourceIsDone = value.done == true;
|
|
8336
|
+
resolveWaiter((w) => w.resolve(value));
|
|
8337
|
+
}, (error) => {
|
|
8338
|
+
resolveWaiter((w) => w.reject(error));
|
|
8339
|
+
});
|
|
8340
|
+
};
|
|
8341
|
+
return {
|
|
8342
|
+
next: () => {
|
|
8343
|
+
return new Promise((resolve, reject) => {
|
|
8344
|
+
// First priority: Dispatch ready upstream events.
|
|
8345
|
+
if (sourceIsDone) {
|
|
8346
|
+
return resolve(doneResult);
|
|
8347
|
+
}
|
|
8348
|
+
if (pendingSourceEvent) {
|
|
8349
|
+
pendingSourceEvent({ resolve, reject });
|
|
8350
|
+
pendingSourceEvent = null;
|
|
8351
|
+
return;
|
|
8352
|
+
}
|
|
8353
|
+
// Second priority: Dispatch injected events
|
|
8354
|
+
if (pendingInjectedEvents.length) {
|
|
8355
|
+
return resolve(valueResult(pendingInjectedEvents.shift()));
|
|
8356
|
+
}
|
|
8357
|
+
// Nothing pending? Fetch from source
|
|
8358
|
+
waiter = { resolve, reject };
|
|
8359
|
+
return fetchFromSource();
|
|
8360
|
+
});
|
|
8361
|
+
},
|
|
8362
|
+
inject: (event) => {
|
|
8363
|
+
const pending = consumeWaiter();
|
|
8364
|
+
if (pending != null) {
|
|
8365
|
+
pending.resolve(valueResult(event));
|
|
8366
|
+
}
|
|
8367
|
+
else {
|
|
8368
|
+
pendingInjectedEvents.push(event);
|
|
8369
|
+
}
|
|
8370
|
+
}
|
|
8371
|
+
};
|
|
8372
|
+
}
|
|
8373
|
+
/**
|
|
8374
|
+
* Splits a byte stream at line endings, emitting each line as a string.
|
|
8375
|
+
*/
|
|
8376
|
+
function extractJsonLines(source, decoder) {
|
|
8377
|
+
let buffer = '';
|
|
8378
|
+
const pendingLines = [];
|
|
8379
|
+
let isFinalEvent = false;
|
|
8380
|
+
return {
|
|
8381
|
+
next: async () => {
|
|
8382
|
+
while (true) {
|
|
8383
|
+
if (isFinalEvent) {
|
|
8384
|
+
return doneResult;
|
|
8385
|
+
}
|
|
8386
|
+
{
|
|
8387
|
+
const first = pendingLines.shift();
|
|
8388
|
+
if (first) {
|
|
8389
|
+
return { done: false, value: first };
|
|
8390
|
+
}
|
|
8391
|
+
}
|
|
8392
|
+
const { done, value } = await source.next();
|
|
8393
|
+
if (done) {
|
|
8394
|
+
const remaining = buffer.trim();
|
|
8395
|
+
if (remaining.length != 0) {
|
|
8396
|
+
isFinalEvent = true;
|
|
8397
|
+
return { done: false, value: remaining };
|
|
8398
|
+
}
|
|
8399
|
+
return doneResult;
|
|
8400
|
+
}
|
|
8401
|
+
const data = decoder.decode(value, { stream: true });
|
|
8402
|
+
buffer += data;
|
|
8403
|
+
const lines = buffer.split('\n');
|
|
8404
|
+
for (let i = 0; i < lines.length - 1; i++) {
|
|
8405
|
+
const l = lines[i].trim();
|
|
8406
|
+
if (l.length > 0) {
|
|
8407
|
+
pendingLines.push(l);
|
|
8408
|
+
}
|
|
8409
|
+
}
|
|
8410
|
+
buffer = lines[lines.length - 1];
|
|
8411
|
+
}
|
|
8412
|
+
}
|
|
8413
|
+
};
|
|
8414
|
+
}
|
|
8415
|
+
/**
|
|
8416
|
+
* Splits a concatenated stream of BSON objects by emitting individual objects.
|
|
8417
|
+
*/
|
|
8418
|
+
function extractBsonObjects(source) {
|
|
8419
|
+
// Fully read but not emitted yet.
|
|
8420
|
+
const completedObjects = [];
|
|
8421
|
+
// Whether source has returned { done: true }. We do the same once completed objects have been emitted.
|
|
8422
|
+
let isDone = false;
|
|
8423
|
+
const lengthBuffer = new DataView(new ArrayBuffer(4));
|
|
8424
|
+
let objectBody = null;
|
|
8425
|
+
// If we're parsing the length field, a number between 1 and 4 (inclusive) describing remaining bytes in the header.
|
|
8426
|
+
// If we're consuming a document, the bytes remaining.
|
|
8427
|
+
let remainingLength = 4;
|
|
8428
|
+
return {
|
|
8429
|
+
async next() {
|
|
8430
|
+
while (true) {
|
|
8431
|
+
// Before fetching new data from upstream, return completed objects.
|
|
8432
|
+
if (completedObjects.length) {
|
|
8433
|
+
return valueResult(completedObjects.shift());
|
|
8434
|
+
}
|
|
8435
|
+
if (isDone) {
|
|
8436
|
+
return doneResult;
|
|
8437
|
+
}
|
|
8438
|
+
const upstreamEvent = await source.next();
|
|
8439
|
+
if (upstreamEvent.done) {
|
|
8440
|
+
isDone = true;
|
|
8441
|
+
if (objectBody || remainingLength != 4) {
|
|
8442
|
+
throw new Error('illegal end of stream in BSON object');
|
|
8443
|
+
}
|
|
8444
|
+
return doneResult;
|
|
8445
|
+
}
|
|
8446
|
+
const chunk = upstreamEvent.value;
|
|
8447
|
+
for (let i = 0; i < chunk.length;) {
|
|
8448
|
+
const availableInData = chunk.length - i;
|
|
8449
|
+
if (objectBody) {
|
|
8450
|
+
// We're in the middle of reading a BSON document.
|
|
8451
|
+
const bytesToRead = Math.min(availableInData, remainingLength);
|
|
8452
|
+
const copySource = new Uint8Array(chunk.buffer, chunk.byteOffset + i, bytesToRead);
|
|
8453
|
+
objectBody.set(copySource, objectBody.length - remainingLength);
|
|
8454
|
+
i += bytesToRead;
|
|
8455
|
+
remainingLength -= bytesToRead;
|
|
8456
|
+
if (remainingLength == 0) {
|
|
8457
|
+
completedObjects.push(objectBody);
|
|
8458
|
+
// Prepare to read another document, starting with its length
|
|
8459
|
+
objectBody = null;
|
|
8460
|
+
remainingLength = 4;
|
|
8461
|
+
}
|
|
8462
|
+
}
|
|
8463
|
+
else {
|
|
8464
|
+
// Copy up to 4 bytes into lengthBuffer, depending on how many we still need.
|
|
8465
|
+
const bytesToRead = Math.min(availableInData, remainingLength);
|
|
8466
|
+
for (let j = 0; j < bytesToRead; j++) {
|
|
8467
|
+
lengthBuffer.setUint8(4 - remainingLength + j, chunk[i + j]);
|
|
8468
|
+
}
|
|
8469
|
+
i += bytesToRead;
|
|
8470
|
+
remainingLength -= bytesToRead;
|
|
8471
|
+
if (remainingLength == 0) {
|
|
8472
|
+
// Transition from reading length header to reading document. Subtracting 4 because the length of the
|
|
8473
|
+
// header is included in length.
|
|
8474
|
+
const length = lengthBuffer.getInt32(0, true /* little endian */);
|
|
8475
|
+
remainingLength = length - 4;
|
|
8476
|
+
if (remainingLength < 1) {
|
|
8477
|
+
throw new Error(`invalid length for bson: ${length}`);
|
|
8478
|
+
}
|
|
8479
|
+
objectBody = new Uint8Array(length);
|
|
8480
|
+
new DataView(objectBody.buffer).setInt32(0, length, true);
|
|
8481
|
+
}
|
|
8482
|
+
}
|
|
8483
|
+
}
|
|
8484
|
+
}
|
|
8485
|
+
}
|
|
8486
|
+
};
|
|
8487
|
+
}
|
|
8488
|
+
|
|
8565
8489
|
const POWERSYNC_TRAILING_SLASH_MATCH = /\/+$/;
|
|
8566
8490
|
const POWERSYNC_JS_VERSION = PACKAGE.version;
|
|
8491
|
+
const SYNC_QUEUE_REQUEST_HIGH_WATER = 10;
|
|
8567
8492
|
const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
|
|
8568
8493
|
// Keep alive message is sent every period
|
|
8569
8494
|
const KEEP_ALIVE_MS = 20_000;
|
|
@@ -8743,73 +8668,67 @@ class AbstractRemote {
|
|
|
8743
8668
|
return new WebSocket(url);
|
|
8744
8669
|
}
|
|
8745
8670
|
/**
|
|
8746
|
-
* Returns a data stream of sync line data.
|
|
8671
|
+
* Returns a data stream of sync line data, fetched via RSocket-over-WebSocket.
|
|
8747
8672
|
*
|
|
8748
|
-
*
|
|
8749
|
-
* @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
|
|
8750
|
-
* (required for compatibility with older sync services).
|
|
8673
|
+
* The only mechanism to abort the returned stream is to use the abort signal in {@link SocketSyncStreamOptions}.
|
|
8751
8674
|
*/
|
|
8752
|
-
async socketStreamRaw(options
|
|
8675
|
+
async socketStreamRaw(options) {
|
|
8753
8676
|
const { path, fetchStrategy = FetchStrategy.Buffered } = options;
|
|
8754
|
-
const mimeType =
|
|
8677
|
+
const mimeType = 'application/json';
|
|
8755
8678
|
function toBuffer(js) {
|
|
8756
|
-
|
|
8757
|
-
if (bson != null) {
|
|
8758
|
-
contents = bson.serialize(js);
|
|
8759
|
-
}
|
|
8760
|
-
else {
|
|
8761
|
-
contents = JSON.stringify(js);
|
|
8762
|
-
}
|
|
8763
|
-
return Buffer.from(contents);
|
|
8679
|
+
return Buffer.from(JSON.stringify(js));
|
|
8764
8680
|
}
|
|
8765
8681
|
const syncQueueRequestSize = fetchStrategy == FetchStrategy.Buffered ? 10 : 1;
|
|
8766
8682
|
const request = await this.buildRequest(path);
|
|
8683
|
+
const url = this.options.socketUrlTransformer(request.url);
|
|
8767
8684
|
// Add the user agent in the setup payload - we can't set custom
|
|
8768
8685
|
// headers with websockets on web. The browser userAgent is however added
|
|
8769
8686
|
// automatically as a header.
|
|
8770
8687
|
const userAgent = this.getUserAgent();
|
|
8771
|
-
|
|
8772
|
-
|
|
8773
|
-
|
|
8774
|
-
|
|
8775
|
-
|
|
8776
|
-
|
|
8777
|
-
|
|
8688
|
+
// While we're connecting (a process that can't be aborted in RSocket), the WebSocket instance to close if we wanted
|
|
8689
|
+
// to abort the connection.
|
|
8690
|
+
let pendingSocket = null;
|
|
8691
|
+
let keepAliveTimeout;
|
|
8692
|
+
let rsocket = null;
|
|
8693
|
+
let queue = null;
|
|
8694
|
+
let didClose = false;
|
|
8695
|
+
const abortRequest = () => {
|
|
8696
|
+
if (didClose) {
|
|
8697
|
+
return;
|
|
8698
|
+
}
|
|
8699
|
+
didClose = true;
|
|
8700
|
+
clearTimeout(keepAliveTimeout);
|
|
8701
|
+
if (pendingSocket) {
|
|
8702
|
+
pendingSocket.close();
|
|
8703
|
+
}
|
|
8704
|
+
if (rsocket) {
|
|
8705
|
+
rsocket.close();
|
|
8706
|
+
}
|
|
8707
|
+
if (queue) {
|
|
8708
|
+
queue.stop();
|
|
8709
|
+
}
|
|
8710
|
+
};
|
|
8778
8711
|
// Handle upstream abort
|
|
8779
|
-
if (options.abortSignal
|
|
8712
|
+
if (options.abortSignal.aborted) {
|
|
8780
8713
|
throw new AbortOperation('Connection request aborted');
|
|
8781
8714
|
}
|
|
8782
8715
|
else {
|
|
8783
|
-
options.abortSignal
|
|
8784
|
-
stream.close();
|
|
8785
|
-
}, { once: true });
|
|
8716
|
+
options.abortSignal.addEventListener('abort', abortRequest);
|
|
8786
8717
|
}
|
|
8787
|
-
let keepAliveTimeout;
|
|
8788
8718
|
const resetTimeout = () => {
|
|
8789
8719
|
clearTimeout(keepAliveTimeout);
|
|
8790
8720
|
keepAliveTimeout = setTimeout(() => {
|
|
8791
8721
|
this.logger.error(`No data received on WebSocket in ${SOCKET_TIMEOUT_MS}ms, closing connection.`);
|
|
8792
|
-
|
|
8722
|
+
abortRequest();
|
|
8793
8723
|
}, SOCKET_TIMEOUT_MS);
|
|
8794
8724
|
};
|
|
8795
8725
|
resetTimeout();
|
|
8796
|
-
// Typescript complains about this being `never` if it's not assigned here.
|
|
8797
|
-
// This is assigned in `wsCreator`.
|
|
8798
|
-
let disposeSocketConnectionTimeout = () => { };
|
|
8799
|
-
const url = this.options.socketUrlTransformer(request.url);
|
|
8800
8726
|
const connector = new distExports.RSocketConnector({
|
|
8801
8727
|
transport: new WebsocketClientTransport({
|
|
8802
8728
|
url,
|
|
8803
8729
|
wsCreator: (url) => {
|
|
8804
|
-
const socket = this.createSocket(url);
|
|
8805
|
-
|
|
8806
|
-
closed: () => {
|
|
8807
|
-
// Allow closing the underlying WebSocket if the stream was closed before the
|
|
8808
|
-
// RSocket connect completed. This should effectively abort the request.
|
|
8809
|
-
socket.close();
|
|
8810
|
-
}
|
|
8811
|
-
});
|
|
8812
|
-
socket.addEventListener('message', (event) => {
|
|
8730
|
+
const socket = (pendingSocket = this.createSocket(url));
|
|
8731
|
+
socket.addEventListener('message', () => {
|
|
8813
8732
|
resetTimeout();
|
|
8814
8733
|
});
|
|
8815
8734
|
return socket;
|
|
@@ -8829,43 +8748,40 @@ class AbstractRemote {
|
|
|
8829
8748
|
}
|
|
8830
8749
|
}
|
|
8831
8750
|
});
|
|
8832
|
-
let rsocket;
|
|
8833
8751
|
try {
|
|
8834
8752
|
rsocket = await connector.connect();
|
|
8835
8753
|
// The connection is established, we no longer need to monitor the initial timeout
|
|
8836
|
-
|
|
8754
|
+
pendingSocket = null;
|
|
8837
8755
|
}
|
|
8838
8756
|
catch (ex) {
|
|
8839
8757
|
this.logger.error(`Failed to connect WebSocket`, ex);
|
|
8840
|
-
|
|
8841
|
-
if (!stream.closed) {
|
|
8842
|
-
await stream.close();
|
|
8843
|
-
}
|
|
8758
|
+
abortRequest();
|
|
8844
8759
|
throw ex;
|
|
8845
8760
|
}
|
|
8846
8761
|
resetTimeout();
|
|
8847
|
-
let socketIsClosed = false;
|
|
8848
|
-
const closeSocket = () => {
|
|
8849
|
-
clearTimeout(keepAliveTimeout);
|
|
8850
|
-
if (socketIsClosed) {
|
|
8851
|
-
return;
|
|
8852
|
-
}
|
|
8853
|
-
socketIsClosed = true;
|
|
8854
|
-
rsocket.close();
|
|
8855
|
-
};
|
|
8856
8762
|
// Helps to prevent double close scenarios
|
|
8857
|
-
rsocket.onClose(() => (
|
|
8858
|
-
|
|
8859
|
-
let pendingEventsCount = syncQueueRequestSize;
|
|
8860
|
-
const disposeClosedListener = stream.registerListener({
|
|
8861
|
-
closed: () => {
|
|
8862
|
-
closeSocket();
|
|
8863
|
-
disposeClosedListener();
|
|
8864
|
-
}
|
|
8865
|
-
});
|
|
8866
|
-
const socket = await new Promise((resolve, reject) => {
|
|
8763
|
+
rsocket.onClose(() => (rsocket = null));
|
|
8764
|
+
return await new Promise((resolve, reject) => {
|
|
8867
8765
|
let connectionEstablished = false;
|
|
8868
|
-
|
|
8766
|
+
let pendingEventsCount = syncQueueRequestSize;
|
|
8767
|
+
let paused = false;
|
|
8768
|
+
let res = null;
|
|
8769
|
+
function requestMore() {
|
|
8770
|
+
const delta = syncQueueRequestSize - pendingEventsCount;
|
|
8771
|
+
if (!paused && delta > 0) {
|
|
8772
|
+
res?.request(delta);
|
|
8773
|
+
pendingEventsCount = syncQueueRequestSize;
|
|
8774
|
+
}
|
|
8775
|
+
}
|
|
8776
|
+
const events = new EventIterator((q) => {
|
|
8777
|
+
queue = q;
|
|
8778
|
+
q.on('highWater', () => (paused = true));
|
|
8779
|
+
q.on('lowWater', () => {
|
|
8780
|
+
paused = false;
|
|
8781
|
+
requestMore();
|
|
8782
|
+
});
|
|
8783
|
+
}, { highWaterMark: SYNC_QUEUE_REQUEST_HIGH_WATER, lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER })[Symbol.asyncIterator]();
|
|
8784
|
+
res = rsocket.requestStream({
|
|
8869
8785
|
data: toBuffer(options.data),
|
|
8870
8786
|
metadata: toBuffer({
|
|
8871
8787
|
path
|
|
@@ -8890,7 +8806,7 @@ class AbstractRemote {
|
|
|
8890
8806
|
}
|
|
8891
8807
|
// RSocket will close the RSocket stream automatically
|
|
8892
8808
|
// Close the downstream stream as well - this will close the RSocket connection and WebSocket
|
|
8893
|
-
|
|
8809
|
+
abortRequest();
|
|
8894
8810
|
// Handles cases where the connection failed e.g. auth error or connection error
|
|
8895
8811
|
if (!connectionEstablished) {
|
|
8896
8812
|
reject(e);
|
|
@@ -8900,41 +8816,40 @@ class AbstractRemote {
|
|
|
8900
8816
|
// The connection is active
|
|
8901
8817
|
if (!connectionEstablished) {
|
|
8902
8818
|
connectionEstablished = true;
|
|
8903
|
-
resolve(
|
|
8819
|
+
resolve(events);
|
|
8904
8820
|
}
|
|
8905
8821
|
const { data } = payload;
|
|
8822
|
+
if (data) {
|
|
8823
|
+
queue.push(data);
|
|
8824
|
+
}
|
|
8906
8825
|
// Less events are now pending
|
|
8907
8826
|
pendingEventsCount--;
|
|
8908
|
-
|
|
8909
|
-
|
|
8910
|
-
}
|
|
8911
|
-
stream.enqueueData(data);
|
|
8827
|
+
// Request another event (unless the downstream consumer is paused).
|
|
8828
|
+
requestMore();
|
|
8912
8829
|
},
|
|
8913
8830
|
onComplete: () => {
|
|
8914
|
-
|
|
8831
|
+
abortRequest(); // this will also emit a done event
|
|
8915
8832
|
},
|
|
8916
8833
|
onExtension: () => { }
|
|
8917
8834
|
});
|
|
8918
8835
|
});
|
|
8919
|
-
const l = stream.registerListener({
|
|
8920
|
-
lowWater: async () => {
|
|
8921
|
-
// Request to fill up the queue
|
|
8922
|
-
const required = syncQueueRequestSize - pendingEventsCount;
|
|
8923
|
-
if (required > 0) {
|
|
8924
|
-
socket.request(syncQueueRequestSize - pendingEventsCount);
|
|
8925
|
-
pendingEventsCount = syncQueueRequestSize;
|
|
8926
|
-
}
|
|
8927
|
-
},
|
|
8928
|
-
closed: () => {
|
|
8929
|
-
l();
|
|
8930
|
-
}
|
|
8931
|
-
});
|
|
8932
|
-
return stream;
|
|
8933
8836
|
}
|
|
8934
8837
|
/**
|
|
8935
|
-
*
|
|
8838
|
+
* @returns Whether the HTTP implementation on this platform can receive streamed binary responses. This is true on
|
|
8839
|
+
* all platforms except React Native (who would have guessed...), where we must not request BSON responses.
|
|
8840
|
+
*
|
|
8841
|
+
* @see https://github.com/react-native-community/fetch?tab=readme-ov-file#motivation
|
|
8842
|
+
*/
|
|
8843
|
+
get supportsStreamingBinaryResponses() {
|
|
8844
|
+
return true;
|
|
8845
|
+
}
|
|
8846
|
+
/**
|
|
8847
|
+
* Posts a `/sync/stream` request, asserts that it completes successfully and returns the streaming response as an
|
|
8848
|
+
* async iterator of byte blobs.
|
|
8849
|
+
*
|
|
8850
|
+
* To cancel the async iterator, use the abort signal from {@link SyncStreamOptions} passed to this method.
|
|
8936
8851
|
*/
|
|
8937
|
-
async
|
|
8852
|
+
async fetchStreamRaw(options) {
|
|
8938
8853
|
const { data, path, headers, abortSignal } = options;
|
|
8939
8854
|
const request = await this.buildRequest(path);
|
|
8940
8855
|
/**
|
|
@@ -8946,119 +8861,94 @@ class AbstractRemote {
|
|
|
8946
8861
|
* Aborting the active fetch request while it is being consumed seems to throw
|
|
8947
8862
|
* an unhandled exception on the window level.
|
|
8948
8863
|
*/
|
|
8949
|
-
if (abortSignal
|
|
8950
|
-
throw new AbortOperation('Abort request received before making
|
|
8864
|
+
if (abortSignal.aborted) {
|
|
8865
|
+
throw new AbortOperation('Abort request received before making fetchStreamRaw request');
|
|
8951
8866
|
}
|
|
8952
8867
|
const controller = new AbortController();
|
|
8953
|
-
let
|
|
8954
|
-
abortSignal
|
|
8955
|
-
|
|
8868
|
+
let reader = null;
|
|
8869
|
+
abortSignal.addEventListener('abort', () => {
|
|
8870
|
+
const reason = abortSignal.reason ??
|
|
8871
|
+
new AbortOperation('Cancelling network request before it resolves. Abort signal has been received.');
|
|
8872
|
+
if (reader == null) {
|
|
8956
8873
|
// Only abort via the abort controller if the request has not resolved yet
|
|
8957
|
-
controller.abort(
|
|
8958
|
-
|
|
8874
|
+
controller.abort(reason);
|
|
8875
|
+
}
|
|
8876
|
+
else {
|
|
8877
|
+
reader.cancel(reason).catch(() => {
|
|
8878
|
+
// Cancelling the reader might rethrow an exception we would have handled by throwing in next(). So we can
|
|
8879
|
+
// ignore it here.
|
|
8880
|
+
});
|
|
8959
8881
|
}
|
|
8960
8882
|
});
|
|
8961
|
-
|
|
8962
|
-
|
|
8963
|
-
|
|
8964
|
-
|
|
8965
|
-
|
|
8966
|
-
|
|
8967
|
-
|
|
8968
|
-
|
|
8969
|
-
|
|
8883
|
+
let res;
|
|
8884
|
+
let responseIsBson = false;
|
|
8885
|
+
try {
|
|
8886
|
+
const ndJson = 'application/x-ndjson';
|
|
8887
|
+
const bson = 'application/vnd.powersync.bson-stream';
|
|
8888
|
+
res = await this.fetch(request.url, {
|
|
8889
|
+
method: 'POST',
|
|
8890
|
+
headers: {
|
|
8891
|
+
...headers,
|
|
8892
|
+
...request.headers,
|
|
8893
|
+
accept: this.supportsStreamingBinaryResponses ? `${bson};q=0.9,${ndJson};q=0.8` : ndJson
|
|
8894
|
+
},
|
|
8895
|
+
body: JSON.stringify(data),
|
|
8896
|
+
signal: controller.signal,
|
|
8897
|
+
cache: 'no-store',
|
|
8898
|
+
...(this.options.fetchOptions ?? {}),
|
|
8899
|
+
...options.fetchOptions
|
|
8900
|
+
});
|
|
8901
|
+
if (!res.ok || !res.body) {
|
|
8902
|
+
const text = await res.text();
|
|
8903
|
+
this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
|
|
8904
|
+
const error = new Error(`HTTP ${res.statusText}: ${text}`);
|
|
8905
|
+
error.status = res.status;
|
|
8906
|
+
throw error;
|
|
8907
|
+
}
|
|
8908
|
+
const contentType = res.headers.get('content-type');
|
|
8909
|
+
responseIsBson = contentType == bson;
|
|
8910
|
+
}
|
|
8911
|
+
catch (ex) {
|
|
8970
8912
|
if (ex.name == 'AbortError') {
|
|
8971
8913
|
throw new AbortOperation(`Pending fetch request to ${request.url} has been aborted.`);
|
|
8972
8914
|
}
|
|
8973
8915
|
throw ex;
|
|
8974
|
-
});
|
|
8975
|
-
if (!res) {
|
|
8976
|
-
throw new Error('Fetch request was aborted');
|
|
8977
|
-
}
|
|
8978
|
-
requestResolved = true;
|
|
8979
|
-
if (!res.ok || !res.body) {
|
|
8980
|
-
const text = await res.text();
|
|
8981
|
-
this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
|
|
8982
|
-
const error = new Error(`HTTP ${res.statusText}: ${text}`);
|
|
8983
|
-
error.status = res.status;
|
|
8984
|
-
throw error;
|
|
8985
8916
|
}
|
|
8986
|
-
|
|
8987
|
-
|
|
8988
|
-
|
|
8989
|
-
|
|
8990
|
-
|
|
8991
|
-
const closeReader = async () => {
|
|
8992
|
-
try {
|
|
8993
|
-
readerReleased = true;
|
|
8994
|
-
await reader.cancel();
|
|
8995
|
-
}
|
|
8996
|
-
catch (ex) {
|
|
8997
|
-
// an error will throw if the reader hasn't been used yet
|
|
8998
|
-
}
|
|
8999
|
-
reader.releaseLock();
|
|
9000
|
-
};
|
|
9001
|
-
const stream = new DataStream({
|
|
9002
|
-
logger: this.logger,
|
|
9003
|
-
mapLine: mapLine,
|
|
9004
|
-
pressure: {
|
|
9005
|
-
highWaterMark: 20,
|
|
9006
|
-
lowWaterMark: 10
|
|
9007
|
-
}
|
|
9008
|
-
});
|
|
9009
|
-
abortSignal?.addEventListener('abort', () => {
|
|
9010
|
-
closeReader();
|
|
9011
|
-
stream.close();
|
|
9012
|
-
});
|
|
9013
|
-
const decoder = this.createTextDecoder();
|
|
9014
|
-
let buffer = '';
|
|
9015
|
-
const consumeStream = async () => {
|
|
9016
|
-
while (!stream.closed && !abortSignal?.aborted && !readerReleased) {
|
|
9017
|
-
const { done, value } = await reader.read();
|
|
9018
|
-
if (done) {
|
|
9019
|
-
const remaining = buffer.trim();
|
|
9020
|
-
if (remaining.length != 0) {
|
|
9021
|
-
stream.enqueueData(remaining);
|
|
9022
|
-
}
|
|
9023
|
-
stream.close();
|
|
9024
|
-
await closeReader();
|
|
9025
|
-
return;
|
|
8917
|
+
reader = res.body.getReader();
|
|
8918
|
+
const stream = {
|
|
8919
|
+
next: async () => {
|
|
8920
|
+
if (controller.signal.aborted) {
|
|
8921
|
+
return doneResult;
|
|
9026
8922
|
}
|
|
9027
|
-
|
|
9028
|
-
|
|
9029
|
-
const lines = buffer.split('\n');
|
|
9030
|
-
for (var i = 0; i < lines.length - 1; i++) {
|
|
9031
|
-
var l = lines[i].trim();
|
|
9032
|
-
if (l.length > 0) {
|
|
9033
|
-
stream.enqueueData(l);
|
|
9034
|
-
}
|
|
8923
|
+
try {
|
|
8924
|
+
return await reader.read();
|
|
9035
8925
|
}
|
|
9036
|
-
|
|
9037
|
-
|
|
9038
|
-
|
|
9039
|
-
|
|
9040
|
-
|
|
9041
|
-
|
|
9042
|
-
|
|
9043
|
-
dispose();
|
|
9044
|
-
},
|
|
9045
|
-
closed: () => {
|
|
9046
|
-
resolve();
|
|
9047
|
-
dispose();
|
|
9048
|
-
}
|
|
9049
|
-
});
|
|
9050
|
-
});
|
|
8926
|
+
catch (ex) {
|
|
8927
|
+
if (controller.signal.aborted) {
|
|
8928
|
+
// .read() completes with an error if we cancel the reader, which we do to disconnect. So this is just
|
|
8929
|
+
// things working as intended, we can return a done event and consider the exception handled.
|
|
8930
|
+
return doneResult;
|
|
8931
|
+
}
|
|
8932
|
+
throw ex;
|
|
9051
8933
|
}
|
|
9052
8934
|
}
|
|
9053
8935
|
};
|
|
9054
|
-
|
|
9055
|
-
|
|
9056
|
-
|
|
9057
|
-
|
|
9058
|
-
|
|
9059
|
-
|
|
9060
|
-
|
|
9061
|
-
|
|
8936
|
+
return { isBson: responseIsBson, stream };
|
|
8937
|
+
}
|
|
8938
|
+
/**
|
|
8939
|
+
* Posts a `/sync/stream` request.
|
|
8940
|
+
*
|
|
8941
|
+
* Depending on the `Content-Type` of the response, this returns strings for sync lines or encoded BSON documents as
|
|
8942
|
+
* {@link Uint8Array}s.
|
|
8943
|
+
*/
|
|
8944
|
+
async fetchStream(options) {
|
|
8945
|
+
const { isBson, stream } = await this.fetchStreamRaw(options);
|
|
8946
|
+
if (isBson) {
|
|
8947
|
+
return extractBsonObjects(stream);
|
|
8948
|
+
}
|
|
8949
|
+
else {
|
|
8950
|
+
return extractJsonLines(stream, this.createTextDecoder());
|
|
8951
|
+
}
|
|
9062
8952
|
}
|
|
9063
8953
|
}
|
|
9064
8954
|
|
|
@@ -9087,31 +8977,8 @@ function coreStatusToJs(status) {
|
|
|
9087
8977
|
priorityStatusEntries: status.priority_status.map(priorityToJs)
|
|
9088
8978
|
};
|
|
9089
8979
|
}
|
|
9090
|
-
|
|
9091
|
-
|
|
9092
|
-
return line.data != null;
|
|
9093
|
-
}
|
|
9094
|
-
function isStreamingKeepalive(line) {
|
|
9095
|
-
return line.token_expires_in != null;
|
|
9096
|
-
}
|
|
9097
|
-
function isStreamingSyncCheckpoint(line) {
|
|
9098
|
-
return line.checkpoint != null;
|
|
9099
|
-
}
|
|
9100
|
-
function isStreamingSyncCheckpointComplete(line) {
|
|
9101
|
-
return line.checkpoint_complete != null;
|
|
9102
|
-
}
|
|
9103
|
-
function isStreamingSyncCheckpointPartiallyComplete(line) {
|
|
9104
|
-
return line.partial_checkpoint_complete != null;
|
|
9105
|
-
}
|
|
9106
|
-
function isStreamingSyncCheckpointDiff(line) {
|
|
9107
|
-
return line.checkpoint_diff != null;
|
|
9108
|
-
}
|
|
9109
|
-
function isContinueCheckpointRequest(request) {
|
|
9110
|
-
return (Array.isArray(request.buckets) &&
|
|
9111
|
-
typeof request.checkpoint_token == 'string');
|
|
9112
|
-
}
|
|
9113
|
-
function isSyncNewCheckpointRequest(request) {
|
|
9114
|
-
return typeof request.request_checkpoint == 'object';
|
|
8980
|
+
function isInterruptingInstruction(instruction) {
|
|
8981
|
+
return 'EstablishSyncStream' in instruction || 'CloseSyncStream' in instruction;
|
|
9115
8982
|
}
|
|
9116
8983
|
|
|
9117
8984
|
var LockType;
|
|
@@ -9126,35 +8993,21 @@ var SyncStreamConnectionMethod;
|
|
|
9126
8993
|
})(SyncStreamConnectionMethod || (SyncStreamConnectionMethod = {}));
|
|
9127
8994
|
var SyncClientImplementation;
|
|
9128
8995
|
(function (SyncClientImplementation) {
|
|
9129
|
-
/**
|
|
9130
|
-
* Decodes and handles sync lines received from the sync service in JavaScript.
|
|
9131
|
-
*
|
|
9132
|
-
* This is the default option.
|
|
9133
|
-
*
|
|
9134
|
-
* @deprecated We recommend the {@link RUST} client implementation for all apps. If you have issues with
|
|
9135
|
-
* the Rust client, please file an issue or reach out to us. The JavaScript client will be removed in a future
|
|
9136
|
-
* version of the PowerSync SDK.
|
|
9137
|
-
*/
|
|
9138
|
-
SyncClientImplementation["JAVASCRIPT"] = "js";
|
|
9139
8996
|
/**
|
|
9140
8997
|
* This implementation offloads the sync line decoding and handling into the PowerSync
|
|
9141
8998
|
* core extension.
|
|
9142
8999
|
*
|
|
9143
|
-
* This
|
|
9144
|
-
* recommended client implementation for all apps.
|
|
9000
|
+
* This is the only option, as an older JavaScript client implementation has been removed from the SDK.
|
|
9145
9001
|
*
|
|
9146
9002
|
* ## Compatibility warning
|
|
9147
9003
|
*
|
|
9148
9004
|
* The Rust sync client stores sync data in a format that is slightly different than the one used
|
|
9149
|
-
* by the old
|
|
9150
|
-
*
|
|
9151
|
-
* Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
|
|
9152
|
-
* the version introducing {@link RUST} as an option) also supports the new format, so you can switch
|
|
9153
|
-
* back to {@link JAVASCRIPT} later.
|
|
9005
|
+
* by the old JavaScript client. When adopting the {@link RUST} client on existing databases, the PowerSync SDK will
|
|
9006
|
+
* migrate the format automatically.
|
|
9154
9007
|
*
|
|
9155
|
-
*
|
|
9156
|
-
*
|
|
9157
|
-
*
|
|
9008
|
+
* SDK versions supporting both the JavaScript and the Rust client support both formats with the JavaScript client
|
|
9009
|
+
* implementaiton. However, downgrading to an SDK version that only supports the JavaScript client would not be
|
|
9010
|
+
* possible anymore. Problematic SDK versions have been released before 2025-06-09.
|
|
9158
9011
|
*/
|
|
9159
9012
|
SyncClientImplementation["RUST"] = "rust";
|
|
9160
9013
|
})(SyncClientImplementation || (SyncClientImplementation = {}));
|
|
@@ -9177,13 +9030,7 @@ const DEFAULT_STREAM_CONNECTION_OPTIONS = {
|
|
|
9177
9030
|
serializedSchema: undefined,
|
|
9178
9031
|
includeDefaultStreams: true
|
|
9179
9032
|
};
|
|
9180
|
-
// The priority we assume when we receive checkpoint lines where no priority is set.
|
|
9181
|
-
// This is the default priority used by the sync service, but can be set to an arbitrary
|
|
9182
|
-
// value since sync services without priorities also won't send partial sync completion
|
|
9183
|
-
// messages.
|
|
9184
|
-
const FALLBACK_PRIORITY = 3;
|
|
9185
9033
|
class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
9186
|
-
_lastSyncedAt;
|
|
9187
9034
|
options;
|
|
9188
9035
|
abortController;
|
|
9189
9036
|
// In rare cases, mostly for tests, uploads can be triggered without being properly connected.
|
|
@@ -9193,6 +9040,7 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
|
9193
9040
|
streamingSyncPromise;
|
|
9194
9041
|
logger;
|
|
9195
9042
|
activeStreams;
|
|
9043
|
+
connectionMayHaveChanged = false;
|
|
9196
9044
|
isUploadingCrud = false;
|
|
9197
9045
|
notifyCompletedUploads;
|
|
9198
9046
|
handleActiveStreamsChange;
|
|
@@ -9272,9 +9120,6 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
|
9272
9120
|
this.crudUpdateListener = undefined;
|
|
9273
9121
|
this.uploadAbortController?.abort();
|
|
9274
9122
|
}
|
|
9275
|
-
async hasCompletedSync() {
|
|
9276
|
-
return this.options.adapter.hasCompletedSync();
|
|
9277
|
-
}
|
|
9278
9123
|
async getWriteCheckpoint() {
|
|
9279
9124
|
const clientId = await this.options.adapter.getClientId();
|
|
9280
9125
|
let path = `/write-checkpoint2.json?client_id=${clientId}`;
|
|
@@ -9356,7 +9201,7 @@ The next upload iteration will be delayed.`);
|
|
|
9356
9201
|
});
|
|
9357
9202
|
}
|
|
9358
9203
|
}
|
|
9359
|
-
this.uploadAbortController =
|
|
9204
|
+
this.uploadAbortController = undefined;
|
|
9360
9205
|
}
|
|
9361
9206
|
});
|
|
9362
9207
|
}
|
|
@@ -9472,6 +9317,11 @@ The next upload iteration will be delayed.`);
|
|
|
9472
9317
|
shouldDelayRetry = false;
|
|
9473
9318
|
// A disconnect was requested, we should not delay since there is no explicit retry
|
|
9474
9319
|
}
|
|
9320
|
+
else if (this.connectionMayHaveChanged && ex.message?.indexOf('No iteration is active') >= 0) {
|
|
9321
|
+
this.connectionMayHaveChanged = false;
|
|
9322
|
+
this.logger.info('Sync error after changed connection, retrying immediately');
|
|
9323
|
+
shouldDelayRetry = false;
|
|
9324
|
+
}
|
|
9475
9325
|
else {
|
|
9476
9326
|
this.logger.error(ex);
|
|
9477
9327
|
}
|
|
@@ -9502,17 +9352,14 @@ The next upload iteration will be delayed.`);
|
|
|
9502
9352
|
// Mark as disconnected if here
|
|
9503
9353
|
this.updateSyncStatus({ connected: false, connecting: false });
|
|
9504
9354
|
}
|
|
9505
|
-
|
|
9506
|
-
|
|
9507
|
-
|
|
9508
|
-
|
|
9509
|
-
|
|
9510
|
-
|
|
9511
|
-
|
|
9512
|
-
|
|
9513
|
-
localDescriptions.set(entry.bucket, null);
|
|
9514
|
-
}
|
|
9515
|
-
return [req, localDescriptions];
|
|
9355
|
+
markConnectionMayHaveChanged() {
|
|
9356
|
+
// By setting this field, we'll immediately retry if the next sync event causes an error triggered by us not having
|
|
9357
|
+
// an active sync iteration on the connection in use.
|
|
9358
|
+
this.connectionMayHaveChanged = true;
|
|
9359
|
+
// This triggers a `powersync_control` invocation if a sync iteration is currently active. This is a cheap call to
|
|
9360
|
+
// make when no subscriptions have actually changed, we're mainly interested in this immediately throwing if no
|
|
9361
|
+
// iteration is active. That allows us to reconnect ASAP, instead of having to wait for the next sync line.
|
|
9362
|
+
this.handleActiveStreamsChange?.();
|
|
9516
9363
|
}
|
|
9517
9364
|
/**
|
|
9518
9365
|
* Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
|
|
@@ -9553,344 +9400,98 @@ The next upload iteration will be delayed.`);
|
|
|
9553
9400
|
if (invalidMetadata.length > 0) {
|
|
9554
9401
|
throw new Error(`Invalid appMetadata provided. Only string values are allowed. Invalid values: ${invalidMetadata.map(([key, value]) => `${key}: ${value}`).join(', ')}`);
|
|
9555
9402
|
}
|
|
9556
|
-
|
|
9557
|
-
this.
|
|
9558
|
-
if (clientImplementation == SyncClientImplementation.JAVASCRIPT) {
|
|
9559
|
-
await this.legacyStreamingSyncIteration(signal, resolvedOptions);
|
|
9560
|
-
return null;
|
|
9561
|
-
}
|
|
9562
|
-
else {
|
|
9563
|
-
await this.requireKeyFormat(true);
|
|
9564
|
-
return await this.rustSyncIteration(signal, resolvedOptions);
|
|
9565
|
-
}
|
|
9403
|
+
await this.requireKeyFormat(true);
|
|
9404
|
+
return await this.rustSyncIteration(signal, resolvedOptions);
|
|
9566
9405
|
}
|
|
9567
9406
|
});
|
|
9568
9407
|
}
|
|
9569
|
-
|
|
9570
|
-
const
|
|
9571
|
-
|
|
9572
|
-
|
|
9573
|
-
|
|
9574
|
-
|
|
9575
|
-
this.logger.error('Sync streams require `clientImplementation: SyncClientImplementation.RUST` when connecting.');
|
|
9576
|
-
}
|
|
9577
|
-
this.logger.debug('Streaming sync iteration started');
|
|
9578
|
-
this.options.adapter.startSession();
|
|
9579
|
-
let [req, bucketMap] = await this.collectLocalBucketState();
|
|
9580
|
-
let targetCheckpoint = null;
|
|
9581
|
-
// A checkpoint that has been validated but not applied (e.g. due to pending local writes)
|
|
9582
|
-
let pendingValidatedCheckpoint = null;
|
|
9583
|
-
const clientId = await this.options.adapter.getClientId();
|
|
9584
|
-
const usingFixedKeyFormat = await this.requireKeyFormat(false);
|
|
9585
|
-
this.logger.debug('Requesting stream from server');
|
|
9586
|
-
const syncOptions = {
|
|
9587
|
-
path: '/sync/stream',
|
|
9588
|
-
abortSignal: signal,
|
|
9589
|
-
data: {
|
|
9590
|
-
buckets: req,
|
|
9591
|
-
include_checksum: true,
|
|
9592
|
-
raw_data: true,
|
|
9593
|
-
parameters: resolvedOptions.params,
|
|
9594
|
-
app_metadata: resolvedOptions.appMetadata,
|
|
9595
|
-
client_id: clientId
|
|
9408
|
+
receiveSyncLines(data) {
|
|
9409
|
+
const { options, connection } = data;
|
|
9410
|
+
const remote = this.options.remote;
|
|
9411
|
+
const openInner = async () => {
|
|
9412
|
+
if (connection.connectionMethod == SyncStreamConnectionMethod.HTTP) {
|
|
9413
|
+
return await remote.fetchStream(options);
|
|
9596
9414
|
}
|
|
9597
|
-
|
|
9598
|
-
|
|
9599
|
-
|
|
9600
|
-
|
|
9601
|
-
|
|
9602
|
-
return JSON.parse(line);
|
|
9603
|
-
}
|
|
9604
|
-
else {
|
|
9605
|
-
// Directly enqueued by us
|
|
9606
|
-
return line;
|
|
9607
|
-
}
|
|
9608
|
-
});
|
|
9609
|
-
}
|
|
9610
|
-
else {
|
|
9611
|
-
const bson = await this.options.remote.getBSON();
|
|
9612
|
-
stream = await this.options.remote.socketStreamRaw({
|
|
9613
|
-
...syncOptions,
|
|
9614
|
-
...{ fetchStrategy: resolvedOptions.fetchStrategy }
|
|
9615
|
-
}, (payload) => {
|
|
9616
|
-
if (payload instanceof Uint8Array) {
|
|
9617
|
-
return bson.deserialize(payload);
|
|
9618
|
-
}
|
|
9619
|
-
else {
|
|
9620
|
-
// Directly enqueued by us
|
|
9621
|
-
return payload;
|
|
9622
|
-
}
|
|
9623
|
-
}, bson);
|
|
9624
|
-
}
|
|
9625
|
-
this.logger.debug('Stream established. Processing events');
|
|
9626
|
-
this.notifyCompletedUploads = () => {
|
|
9627
|
-
if (!stream.closed) {
|
|
9628
|
-
stream.enqueueData({ crud_upload_completed: null });
|
|
9415
|
+
else {
|
|
9416
|
+
return await this.options.remote.socketStreamRaw({
|
|
9417
|
+
...options,
|
|
9418
|
+
...{ fetchStrategy: connection.fetchStrategy }
|
|
9419
|
+
});
|
|
9629
9420
|
}
|
|
9630
9421
|
};
|
|
9631
|
-
|
|
9632
|
-
|
|
9633
|
-
|
|
9634
|
-
|
|
9635
|
-
|
|
9636
|
-
|
|
9637
|
-
if ('crud_upload_completed' in line) {
|
|
9638
|
-
if (pendingValidatedCheckpoint != null) {
|
|
9639
|
-
const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
|
|
9640
|
-
if (applied) {
|
|
9641
|
-
pendingValidatedCheckpoint = null;
|
|
9642
|
-
}
|
|
9643
|
-
else if (endIteration) {
|
|
9644
|
-
break;
|
|
9645
|
-
}
|
|
9422
|
+
let inner;
|
|
9423
|
+
let done = false;
|
|
9424
|
+
return {
|
|
9425
|
+
async next() {
|
|
9426
|
+
if (done) {
|
|
9427
|
+
return doneResult;
|
|
9646
9428
|
}
|
|
9647
|
-
|
|
9648
|
-
|
|
9649
|
-
|
|
9650
|
-
|
|
9651
|
-
|
|
9652
|
-
|
|
9653
|
-
this.updateSyncStatus({
|
|
9654
|
-
connected: true
|
|
9655
|
-
});
|
|
9656
|
-
}
|
|
9657
|
-
if (isStreamingSyncCheckpoint(line)) {
|
|
9658
|
-
targetCheckpoint = line.checkpoint;
|
|
9659
|
-
// New checkpoint - existing validated checkpoint is no longer valid
|
|
9660
|
-
pendingValidatedCheckpoint = null;
|
|
9661
|
-
const bucketsToDelete = new Set(bucketMap.keys());
|
|
9662
|
-
const newBuckets = new Map();
|
|
9663
|
-
for (const checksum of line.checkpoint.buckets) {
|
|
9664
|
-
newBuckets.set(checksum.bucket, {
|
|
9665
|
-
name: checksum.bucket,
|
|
9666
|
-
priority: checksum.priority ?? FALLBACK_PRIORITY
|
|
9429
|
+
else if (inner == null) {
|
|
9430
|
+
inner = await openInner();
|
|
9431
|
+
// We're connected here, so we can tell the core extension about it.
|
|
9432
|
+
return valueResult({
|
|
9433
|
+
command: PowerSyncControlCommand.CONNECTION_STATE,
|
|
9434
|
+
payload: 'established'
|
|
9667
9435
|
});
|
|
9668
|
-
bucketsToDelete.delete(checksum.bucket);
|
|
9669
|
-
}
|
|
9670
|
-
if (bucketsToDelete.size > 0) {
|
|
9671
|
-
this.logger.debug('Removing buckets', [...bucketsToDelete]);
|
|
9672
|
-
}
|
|
9673
|
-
bucketMap = newBuckets;
|
|
9674
|
-
await this.options.adapter.removeBuckets([...bucketsToDelete]);
|
|
9675
|
-
await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
|
|
9676
|
-
await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
|
|
9677
|
-
}
|
|
9678
|
-
else if (isStreamingSyncCheckpointComplete(line)) {
|
|
9679
|
-
const result = await this.applyCheckpoint(targetCheckpoint);
|
|
9680
|
-
if (result.endIteration) {
|
|
9681
|
-
return;
|
|
9682
|
-
}
|
|
9683
|
-
else if (!result.applied) {
|
|
9684
|
-
// "Could not apply checkpoint due to local data". We need to retry after
|
|
9685
|
-
// finishing uploads.
|
|
9686
|
-
pendingValidatedCheckpoint = targetCheckpoint;
|
|
9687
9436
|
}
|
|
9688
9437
|
else {
|
|
9689
|
-
|
|
9690
|
-
|
|
9691
|
-
|
|
9692
|
-
|
|
9693
|
-
}
|
|
9694
|
-
else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
|
|
9695
|
-
const priority = line.partial_checkpoint_complete.priority;
|
|
9696
|
-
this.logger.debug('Partial checkpoint complete', priority);
|
|
9697
|
-
const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint, priority);
|
|
9698
|
-
if (!result.checkpointValid) {
|
|
9699
|
-
// This means checksums failed. Start again with a new checkpoint.
|
|
9700
|
-
// TODO: better back-off
|
|
9701
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
9702
|
-
return;
|
|
9703
|
-
}
|
|
9704
|
-
else if (!result.ready) ;
|
|
9705
|
-
else {
|
|
9706
|
-
// We'll keep on downloading, but can report that this priority is synced now.
|
|
9707
|
-
this.logger.debug('partial checkpoint validation succeeded');
|
|
9708
|
-
// All states with a higher priority can be deleted since this partial sync includes them.
|
|
9709
|
-
const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
|
|
9710
|
-
priorityStates.push({
|
|
9711
|
-
priority,
|
|
9712
|
-
lastSyncedAt: new Date(),
|
|
9713
|
-
hasSynced: true
|
|
9714
|
-
});
|
|
9715
|
-
this.updateSyncStatus({
|
|
9716
|
-
connected: true,
|
|
9717
|
-
priorityStatusEntries: priorityStates
|
|
9718
|
-
});
|
|
9719
|
-
}
|
|
9720
|
-
}
|
|
9721
|
-
else if (isStreamingSyncCheckpointDiff(line)) {
|
|
9722
|
-
// TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
|
|
9723
|
-
if (targetCheckpoint == null) {
|
|
9724
|
-
throw new Error('Checkpoint diff without previous checkpoint');
|
|
9725
|
-
}
|
|
9726
|
-
// New checkpoint - existing validated checkpoint is no longer valid
|
|
9727
|
-
pendingValidatedCheckpoint = null;
|
|
9728
|
-
const diff = line.checkpoint_diff;
|
|
9729
|
-
const newBuckets = new Map();
|
|
9730
|
-
for (const checksum of targetCheckpoint.buckets) {
|
|
9731
|
-
newBuckets.set(checksum.bucket, checksum);
|
|
9732
|
-
}
|
|
9733
|
-
for (const checksum of diff.updated_buckets) {
|
|
9734
|
-
newBuckets.set(checksum.bucket, checksum);
|
|
9735
|
-
}
|
|
9736
|
-
for (const bucket of diff.removed_buckets) {
|
|
9737
|
-
newBuckets.delete(bucket);
|
|
9738
|
-
}
|
|
9739
|
-
const newCheckpoint = {
|
|
9740
|
-
last_op_id: diff.last_op_id,
|
|
9741
|
-
buckets: [...newBuckets.values()],
|
|
9742
|
-
write_checkpoint: diff.write_checkpoint
|
|
9743
|
-
};
|
|
9744
|
-
targetCheckpoint = newCheckpoint;
|
|
9745
|
-
await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
|
|
9746
|
-
bucketMap = new Map();
|
|
9747
|
-
newBuckets.forEach((checksum, name) => bucketMap.set(name, {
|
|
9748
|
-
name: checksum.bucket,
|
|
9749
|
-
priority: checksum.priority ?? FALLBACK_PRIORITY
|
|
9750
|
-
}));
|
|
9751
|
-
const bucketsToDelete = diff.removed_buckets;
|
|
9752
|
-
if (bucketsToDelete.length > 0) {
|
|
9753
|
-
this.logger.debug('Remove buckets', bucketsToDelete);
|
|
9754
|
-
}
|
|
9755
|
-
await this.options.adapter.removeBuckets(bucketsToDelete);
|
|
9756
|
-
await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
|
|
9757
|
-
}
|
|
9758
|
-
else if (isStreamingSyncData(line)) {
|
|
9759
|
-
const { data } = line;
|
|
9760
|
-
const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
|
|
9761
|
-
let updatedProgress = null;
|
|
9762
|
-
if (previousProgress) {
|
|
9763
|
-
updatedProgress = { ...previousProgress };
|
|
9764
|
-
const progressForBucket = updatedProgress[data.bucket];
|
|
9765
|
-
if (progressForBucket) {
|
|
9766
|
-
updatedProgress[data.bucket] = {
|
|
9767
|
-
...progressForBucket,
|
|
9768
|
-
since_last: progressForBucket.since_last + data.data.length
|
|
9769
|
-
};
|
|
9438
|
+
const event = await inner.next();
|
|
9439
|
+
if (event.done) {
|
|
9440
|
+
done = true;
|
|
9441
|
+
return valueResult({ command: PowerSyncControlCommand.CONNECTION_STATE, payload: 'end' });
|
|
9770
9442
|
}
|
|
9771
|
-
|
|
9772
|
-
|
|
9773
|
-
|
|
9774
|
-
|
|
9775
|
-
|
|
9443
|
+
else {
|
|
9444
|
+
return valueResult({
|
|
9445
|
+
command: typeof event.value == 'string'
|
|
9446
|
+
? PowerSyncControlCommand.PROCESS_TEXT_LINE
|
|
9447
|
+
: PowerSyncControlCommand.PROCESS_BSON_LINE,
|
|
9448
|
+
payload: event.value
|
|
9449
|
+
});
|
|
9776
9450
|
}
|
|
9777
|
-
});
|
|
9778
|
-
await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] }, usingFixedKeyFormat);
|
|
9779
|
-
}
|
|
9780
|
-
else if (isStreamingKeepalive(line)) {
|
|
9781
|
-
const remaining_seconds = line.token_expires_in;
|
|
9782
|
-
if (remaining_seconds == 0) {
|
|
9783
|
-
// Connection would be closed automatically right after this
|
|
9784
|
-
this.logger.debug('Token expiring; reconnect');
|
|
9785
|
-
/**
|
|
9786
|
-
* For a rare case where the backend connector does not update the token
|
|
9787
|
-
* (uses the same one), this should have some delay.
|
|
9788
|
-
*/
|
|
9789
|
-
await this.delayRetry();
|
|
9790
|
-
return;
|
|
9791
|
-
}
|
|
9792
|
-
else if (remaining_seconds < 30) {
|
|
9793
|
-
this.logger.debug('Token will expire soon; reconnect');
|
|
9794
|
-
// Pre-emptively refresh the token
|
|
9795
|
-
this.options.remote.invalidateCredentials();
|
|
9796
|
-
return;
|
|
9797
9451
|
}
|
|
9798
|
-
this.triggerCrudUpload();
|
|
9799
|
-
}
|
|
9800
|
-
else {
|
|
9801
|
-
this.logger.debug('Received unknown sync line', line);
|
|
9802
9452
|
}
|
|
9803
|
-
}
|
|
9804
|
-
this.logger.debug('Stream input empty');
|
|
9805
|
-
// Connection closed. Likely due to auth issue.
|
|
9806
|
-
return;
|
|
9453
|
+
};
|
|
9807
9454
|
}
|
|
9808
9455
|
async rustSyncIteration(signal, resolvedOptions) {
|
|
9809
9456
|
const syncImplementation = this;
|
|
9810
9457
|
const adapter = this.options.adapter;
|
|
9811
9458
|
const remote = this.options.remote;
|
|
9812
|
-
let receivingLines = null;
|
|
9813
|
-
let hadSyncLine = false;
|
|
9814
9459
|
let hideDisconnectOnRestart = false;
|
|
9460
|
+
let notifyTokenRefreshed;
|
|
9815
9461
|
if (signal.aborted) {
|
|
9816
9462
|
throw new AbortOperation('Connection request has been aborted');
|
|
9817
9463
|
}
|
|
9818
|
-
|
|
9819
|
-
|
|
9820
|
-
|
|
9821
|
-
|
|
9822
|
-
|
|
9823
|
-
|
|
9824
|
-
async function connect(instr) {
|
|
9825
|
-
const syncOptions = {
|
|
9826
|
-
path: '/sync/stream',
|
|
9827
|
-
abortSignal: abortController.signal,
|
|
9828
|
-
data: instr.request
|
|
9464
|
+
function startCommand() {
|
|
9465
|
+
const options = {
|
|
9466
|
+
parameters: resolvedOptions.params,
|
|
9467
|
+
app_metadata: resolvedOptions.appMetadata,
|
|
9468
|
+
active_streams: syncImplementation.activeStreams,
|
|
9469
|
+
include_defaults: resolvedOptions.includeDefaultStreams
|
|
9829
9470
|
};
|
|
9830
|
-
if (resolvedOptions.
|
|
9831
|
-
|
|
9832
|
-
if (typeof line == 'string') {
|
|
9833
|
-
return {
|
|
9834
|
-
command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
|
|
9835
|
-
payload: line
|
|
9836
|
-
};
|
|
9837
|
-
}
|
|
9838
|
-
else {
|
|
9839
|
-
// Directly enqueued by us
|
|
9840
|
-
return line;
|
|
9841
|
-
}
|
|
9842
|
-
});
|
|
9843
|
-
}
|
|
9844
|
-
else {
|
|
9845
|
-
controlInvocations = await remote.socketStreamRaw({
|
|
9846
|
-
...syncOptions,
|
|
9847
|
-
fetchStrategy: resolvedOptions.fetchStrategy
|
|
9848
|
-
}, (payload) => {
|
|
9849
|
-
if (payload instanceof Uint8Array) {
|
|
9850
|
-
return {
|
|
9851
|
-
command: PowerSyncControlCommand.PROCESS_BSON_LINE,
|
|
9852
|
-
payload: payload
|
|
9853
|
-
};
|
|
9854
|
-
}
|
|
9855
|
-
else {
|
|
9856
|
-
// Directly enqueued by us
|
|
9857
|
-
return payload;
|
|
9858
|
-
}
|
|
9859
|
-
});
|
|
9860
|
-
}
|
|
9861
|
-
// The rust client will set connected: true after the first sync line because that's when it gets invoked, but
|
|
9862
|
-
// we're already connected here and can report that.
|
|
9863
|
-
syncImplementation.updateSyncStatus({ connected: true });
|
|
9864
|
-
try {
|
|
9865
|
-
while (!controlInvocations.closed) {
|
|
9866
|
-
const line = await controlInvocations.read();
|
|
9867
|
-
if (line == null) {
|
|
9868
|
-
return;
|
|
9869
|
-
}
|
|
9870
|
-
await control(line.command, line.payload);
|
|
9871
|
-
if (!hadSyncLine) {
|
|
9872
|
-
syncImplementation.triggerCrudUpload();
|
|
9873
|
-
hadSyncLine = true;
|
|
9874
|
-
}
|
|
9875
|
-
}
|
|
9876
|
-
}
|
|
9877
|
-
finally {
|
|
9878
|
-
const activeInstructions = controlInvocations;
|
|
9879
|
-
// We concurrently add events to the active data stream when e.g. a CRUD upload is completed or a token is
|
|
9880
|
-
// refreshed. That would throw after closing (and we can't handle those events either way), so set this back
|
|
9881
|
-
// to null.
|
|
9882
|
-
controlInvocations = null;
|
|
9883
|
-
await activeInstructions.close();
|
|
9471
|
+
if (resolvedOptions.serializedSchema) {
|
|
9472
|
+
options.schema = resolvedOptions.serializedSchema;
|
|
9884
9473
|
}
|
|
9474
|
+
return invokePowerSyncControl(PowerSyncControlCommand.START, JSON.stringify(options));
|
|
9885
9475
|
}
|
|
9886
9476
|
async function stop() {
|
|
9887
|
-
await
|
|
9477
|
+
const instructions = await invokePowerSyncControl(PowerSyncControlCommand.STOP);
|
|
9478
|
+
for (const instruction of instructions) {
|
|
9479
|
+
// We don't need to handle interrupting instructions since we're unconditionally ending the sync iteration at
|
|
9480
|
+
// this point.
|
|
9481
|
+
if (isInterruptingInstruction(instruction))
|
|
9482
|
+
continue;
|
|
9483
|
+
await handleInstruction(instruction);
|
|
9484
|
+
}
|
|
9888
9485
|
}
|
|
9889
|
-
async function
|
|
9486
|
+
async function invokePowerSyncControl(op, payload) {
|
|
9890
9487
|
const rawResponse = await adapter.control(op, payload ?? null);
|
|
9891
9488
|
const logger = syncImplementation.logger;
|
|
9892
9489
|
logger.trace('powersync_control', op, payload == null || typeof payload == 'string' ? payload : '<bytes>', rawResponse);
|
|
9893
|
-
|
|
9490
|
+
if (op != PowerSyncControlCommand.STOP) {
|
|
9491
|
+
// Evidently we have a working connection here, otherwise powersync_control would have failed.
|
|
9492
|
+
syncImplementation.connectionMayHaveChanged = false;
|
|
9493
|
+
}
|
|
9494
|
+
return JSON.parse(rawResponse);
|
|
9894
9495
|
}
|
|
9895
9496
|
async function handleInstruction(instruction) {
|
|
9896
9497
|
if ('LogLine' in instruction) {
|
|
@@ -9909,13 +9510,6 @@ The next upload iteration will be delayed.`);
|
|
|
9909
9510
|
else if ('UpdateSyncStatus' in instruction) {
|
|
9910
9511
|
syncImplementation.updateSyncStatus(coreStatusToJs(instruction.UpdateSyncStatus.status));
|
|
9911
9512
|
}
|
|
9912
|
-
else if ('EstablishSyncStream' in instruction) {
|
|
9913
|
-
if (receivingLines != null) {
|
|
9914
|
-
// Already connected, this shouldn't happen during a single iteration.
|
|
9915
|
-
throw 'Unexpected request to establish sync stream, already connected';
|
|
9916
|
-
}
|
|
9917
|
-
receivingLines = connect(instruction.EstablishSyncStream);
|
|
9918
|
-
}
|
|
9919
9513
|
else if ('FetchCredentials' in instruction) {
|
|
9920
9514
|
if (instruction.FetchCredentials.did_expire) {
|
|
9921
9515
|
remote.invalidateCredentials();
|
|
@@ -9924,16 +9518,12 @@ The next upload iteration will be delayed.`);
|
|
|
9924
9518
|
remote.invalidateCredentials();
|
|
9925
9519
|
// Restart iteration after the credentials have been refreshed.
|
|
9926
9520
|
remote.fetchCredentials().then((_) => {
|
|
9927
|
-
|
|
9521
|
+
notifyTokenRefreshed?.();
|
|
9928
9522
|
}, (err) => {
|
|
9929
9523
|
syncImplementation.logger.warn('Could not prefetch credentials', err);
|
|
9930
9524
|
});
|
|
9931
9525
|
}
|
|
9932
9526
|
}
|
|
9933
|
-
else if ('CloseSyncStream' in instruction) {
|
|
9934
|
-
abortController.abort();
|
|
9935
|
-
hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
|
|
9936
|
-
}
|
|
9937
9527
|
else if ('FlushFileSystem' in instruction) ;
|
|
9938
9528
|
else if ('DidCompleteSync' in instruction) {
|
|
9939
9529
|
syncImplementation.updateSyncStatus({
|
|
@@ -9943,105 +9533,83 @@ The next upload iteration will be delayed.`);
|
|
|
9943
9533
|
});
|
|
9944
9534
|
}
|
|
9945
9535
|
}
|
|
9946
|
-
async function handleInstructions(instructions) {
|
|
9947
|
-
for (const instr of instructions) {
|
|
9948
|
-
await handleInstruction(instr);
|
|
9949
|
-
}
|
|
9950
|
-
}
|
|
9951
9536
|
try {
|
|
9952
|
-
const
|
|
9953
|
-
|
|
9954
|
-
|
|
9955
|
-
|
|
9956
|
-
|
|
9957
|
-
|
|
9958
|
-
|
|
9959
|
-
|
|
9537
|
+
const defaultResult = { immediateRestart: false };
|
|
9538
|
+
// Pending sync lines received from the service, as well as local events that trigger a powersync_control
|
|
9539
|
+
// invocation (local events include refreshed tokens and completed uploads).
|
|
9540
|
+
// This is a single data stream so that we can handle all control calls from a single place.
|
|
9541
|
+
let controlInvocations = null;
|
|
9542
|
+
for (const startInstruction of await startCommand()) {
|
|
9543
|
+
if ('EstablishSyncStream' in startInstruction) {
|
|
9544
|
+
const syncOptions = {
|
|
9545
|
+
path: '/sync/stream',
|
|
9546
|
+
abortSignal: signal,
|
|
9547
|
+
data: startInstruction.EstablishSyncStream.request
|
|
9548
|
+
};
|
|
9549
|
+
controlInvocations = injectable(syncImplementation.receiveSyncLines({
|
|
9550
|
+
options: syncOptions,
|
|
9551
|
+
connection: resolvedOptions
|
|
9552
|
+
}));
|
|
9553
|
+
}
|
|
9554
|
+
else if ('CloseSyncStream' in startInstruction) {
|
|
9555
|
+
return defaultResult;
|
|
9556
|
+
}
|
|
9557
|
+
else {
|
|
9558
|
+
await handleInstruction(startInstruction);
|
|
9559
|
+
}
|
|
9960
9560
|
}
|
|
9961
|
-
|
|
9561
|
+
if (controlInvocations == null)
|
|
9562
|
+
return defaultResult;
|
|
9962
9563
|
this.notifyCompletedUploads = () => {
|
|
9963
|
-
|
|
9964
|
-
controlInvocations.enqueueData({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
|
|
9965
|
-
}
|
|
9564
|
+
controlInvocations.inject({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
|
|
9966
9565
|
};
|
|
9967
9566
|
this.handleActiveStreamsChange = () => {
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
|
|
9972
|
-
});
|
|
9973
|
-
}
|
|
9567
|
+
controlInvocations.inject({
|
|
9568
|
+
command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
|
|
9569
|
+
payload: JSON.stringify(this.activeStreams)
|
|
9570
|
+
});
|
|
9974
9571
|
};
|
|
9975
|
-
|
|
9572
|
+
notifyTokenRefreshed = () => {
|
|
9573
|
+
controlInvocations.inject({
|
|
9574
|
+
command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED
|
|
9575
|
+
});
|
|
9576
|
+
};
|
|
9577
|
+
let hadSyncLine = false;
|
|
9578
|
+
loop: while (true) {
|
|
9579
|
+
const { done, value } = await controlInvocations.next();
|
|
9580
|
+
if (done)
|
|
9581
|
+
break;
|
|
9582
|
+
if (!hadSyncLine) {
|
|
9583
|
+
// Trigger a local CRUD upload when the first sync line has been received, this allows uploading local changes
|
|
9584
|
+
// that have been made while offline or disconnected.
|
|
9585
|
+
if (value.command == PowerSyncControlCommand.PROCESS_TEXT_LINE ||
|
|
9586
|
+
value.command == PowerSyncControlCommand.PROCESS_BSON_LINE) {
|
|
9587
|
+
hadSyncLine = true;
|
|
9588
|
+
this.triggerCrudUpload?.();
|
|
9589
|
+
}
|
|
9590
|
+
}
|
|
9591
|
+
const instructions = await invokePowerSyncControl(value.command, value.payload);
|
|
9592
|
+
for (const instruction of instructions) {
|
|
9593
|
+
if ('EstablishSyncStream' in instruction) {
|
|
9594
|
+
throw new Error('Received EstablishSyncStream while already connected.');
|
|
9595
|
+
}
|
|
9596
|
+
else if ('CloseSyncStream' in instruction) {
|
|
9597
|
+
hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
|
|
9598
|
+
break loop;
|
|
9599
|
+
}
|
|
9600
|
+
else {
|
|
9601
|
+
await handleInstruction(instruction);
|
|
9602
|
+
}
|
|
9603
|
+
}
|
|
9604
|
+
}
|
|
9976
9605
|
}
|
|
9977
9606
|
finally {
|
|
9978
9607
|
this.notifyCompletedUploads = this.handleActiveStreamsChange = undefined;
|
|
9608
|
+
notifyTokenRefreshed = undefined;
|
|
9979
9609
|
await stop();
|
|
9980
9610
|
}
|
|
9981
9611
|
return { immediateRestart: hideDisconnectOnRestart };
|
|
9982
9612
|
}
|
|
9983
|
-
async updateSyncStatusForStartingCheckpoint(checkpoint) {
|
|
9984
|
-
const localProgress = await this.options.adapter.getBucketOperationProgress();
|
|
9985
|
-
const progress = {};
|
|
9986
|
-
let invalidated = false;
|
|
9987
|
-
for (const bucket of checkpoint.buckets) {
|
|
9988
|
-
const savedProgress = localProgress[bucket.bucket];
|
|
9989
|
-
const atLast = savedProgress?.atLast ?? 0;
|
|
9990
|
-
const sinceLast = savedProgress?.sinceLast ?? 0;
|
|
9991
|
-
progress[bucket.bucket] = {
|
|
9992
|
-
// The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
|
|
9993
|
-
// will use by default.
|
|
9994
|
-
priority: bucket.priority ?? 3,
|
|
9995
|
-
at_last: atLast,
|
|
9996
|
-
since_last: sinceLast,
|
|
9997
|
-
target_count: bucket.count ?? 0
|
|
9998
|
-
};
|
|
9999
|
-
if (bucket.count != null && bucket.count < atLast + sinceLast) {
|
|
10000
|
-
// Either due to a defrag / sync rule deploy or a compaction operation, the size
|
|
10001
|
-
// of the bucket shrank so much that the local ops exceed the ops in the updated
|
|
10002
|
-
// bucket. We can't prossibly report progress in this case (it would overshoot 100%).
|
|
10003
|
-
invalidated = true;
|
|
10004
|
-
}
|
|
10005
|
-
}
|
|
10006
|
-
if (invalidated) {
|
|
10007
|
-
for (const bucket in progress) {
|
|
10008
|
-
const bucketProgress = progress[bucket];
|
|
10009
|
-
bucketProgress.at_last = 0;
|
|
10010
|
-
bucketProgress.since_last = 0;
|
|
10011
|
-
}
|
|
10012
|
-
}
|
|
10013
|
-
this.updateSyncStatus({
|
|
10014
|
-
dataFlow: {
|
|
10015
|
-
downloading: true,
|
|
10016
|
-
downloadProgress: progress
|
|
10017
|
-
}
|
|
10018
|
-
});
|
|
10019
|
-
}
|
|
10020
|
-
async applyCheckpoint(checkpoint) {
|
|
10021
|
-
let result = await this.options.adapter.syncLocalDatabase(checkpoint);
|
|
10022
|
-
if (!result.checkpointValid) {
|
|
10023
|
-
this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
|
|
10024
|
-
// This means checksums failed. Start again with a new checkpoint.
|
|
10025
|
-
// TODO: better back-off
|
|
10026
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
10027
|
-
return { applied: false, endIteration: true };
|
|
10028
|
-
}
|
|
10029
|
-
else if (!result.ready) {
|
|
10030
|
-
this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`);
|
|
10031
|
-
return { applied: false, endIteration: false };
|
|
10032
|
-
}
|
|
10033
|
-
this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
|
|
10034
|
-
this.updateSyncStatus({
|
|
10035
|
-
connected: true,
|
|
10036
|
-
lastSyncedAt: new Date(),
|
|
10037
|
-
dataFlow: {
|
|
10038
|
-
downloading: false,
|
|
10039
|
-
downloadProgress: null,
|
|
10040
|
-
downloadError: undefined
|
|
10041
|
-
}
|
|
10042
|
-
});
|
|
10043
|
-
return { applied: true, endIteration: false };
|
|
10044
|
-
}
|
|
10045
9613
|
updateSyncStatus(options) {
|
|
10046
9614
|
const updatedStatus = new SyncStatus({
|
|
10047
9615
|
connected: options.connected ?? this.syncStatus.connected,
|
|
@@ -10987,7 +10555,7 @@ class AbstractPowerSyncDatabase extends BaseObserver {
|
|
|
10987
10555
|
* @returns A transaction of CRUD operations to upload, or null if there are none
|
|
10988
10556
|
*/
|
|
10989
10557
|
async getNextCrudTransaction() {
|
|
10990
|
-
const iterator = this.getCrudTransactions()[
|
|
10558
|
+
const iterator = this.getCrudTransactions()[Symbol.asyncIterator]();
|
|
10991
10559
|
return (await iterator.next()).value;
|
|
10992
10560
|
}
|
|
10993
10561
|
/**
|
|
@@ -11023,7 +10591,7 @@ class AbstractPowerSyncDatabase extends BaseObserver {
|
|
|
11023
10591
|
*/
|
|
11024
10592
|
getCrudTransactions() {
|
|
11025
10593
|
return {
|
|
11026
|
-
[
|
|
10594
|
+
[Symbol.asyncIterator]: () => {
|
|
11027
10595
|
let lastCrudItemId = -1;
|
|
11028
10596
|
const sql = `
|
|
11029
10597
|
WITH RECURSIVE crud_entries AS (
|
|
@@ -11580,14 +11148,12 @@ class SqliteBucketStorage extends BaseObserver {
|
|
|
11580
11148
|
db;
|
|
11581
11149
|
logger;
|
|
11582
11150
|
tableNames;
|
|
11583
|
-
_hasCompletedSync;
|
|
11584
11151
|
updateListener;
|
|
11585
11152
|
_clientId;
|
|
11586
11153
|
constructor(db, logger = Logger.get('SqliteBucketStorage')) {
|
|
11587
11154
|
super();
|
|
11588
11155
|
this.db = db;
|
|
11589
11156
|
this.logger = logger;
|
|
11590
|
-
this._hasCompletedSync = false;
|
|
11591
11157
|
this.tableNames = new Set();
|
|
11592
11158
|
this.updateListener = db.registerListener({
|
|
11593
11159
|
tablesUpdated: (update) => {
|
|
@@ -11599,7 +11165,6 @@ class SqliteBucketStorage extends BaseObserver {
|
|
|
11599
11165
|
});
|
|
11600
11166
|
}
|
|
11601
11167
|
async init() {
|
|
11602
|
-
this._hasCompletedSync = false;
|
|
11603
11168
|
const existingTableRows = await this.db.getAll(`SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'`);
|
|
11604
11169
|
for (const row of existingTableRows ?? []) {
|
|
11605
11170
|
this.tableNames.add(row.name);
|
|
@@ -11621,156 +11186,6 @@ class SqliteBucketStorage extends BaseObserver {
|
|
|
11621
11186
|
getMaxOpId() {
|
|
11622
11187
|
return MAX_OP_ID;
|
|
11623
11188
|
}
|
|
11624
|
-
/**
|
|
11625
|
-
* Reset any caches.
|
|
11626
|
-
*/
|
|
11627
|
-
startSession() { }
|
|
11628
|
-
async getBucketStates() {
|
|
11629
|
-
const result = await this.db.getAll("SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != '$local'");
|
|
11630
|
-
return result;
|
|
11631
|
-
}
|
|
11632
|
-
async getBucketOperationProgress() {
|
|
11633
|
-
const rows = await this.db.getAll('SELECT name, count_at_last, count_since_last FROM ps_buckets');
|
|
11634
|
-
return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
|
|
11635
|
-
}
|
|
11636
|
-
async saveSyncData(batch, fixedKeyFormat = false) {
|
|
11637
|
-
await this.writeTransaction(async (tx) => {
|
|
11638
|
-
for (const b of batch.buckets) {
|
|
11639
|
-
await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
|
|
11640
|
-
'save',
|
|
11641
|
-
JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
|
|
11642
|
-
]);
|
|
11643
|
-
this.logger.debug(`Saved batch of data for bucket: ${b.bucket}, operations: ${b.data.length}`);
|
|
11644
|
-
}
|
|
11645
|
-
});
|
|
11646
|
-
}
|
|
11647
|
-
async removeBuckets(buckets) {
|
|
11648
|
-
for (const bucket of buckets) {
|
|
11649
|
-
await this.deleteBucket(bucket);
|
|
11650
|
-
}
|
|
11651
|
-
}
|
|
11652
|
-
/**
|
|
11653
|
-
* Mark a bucket for deletion.
|
|
11654
|
-
*/
|
|
11655
|
-
async deleteBucket(bucket) {
|
|
11656
|
-
await this.writeTransaction(async (tx) => {
|
|
11657
|
-
await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
|
|
11658
|
-
});
|
|
11659
|
-
this.logger.debug(`Done deleting bucket ${bucket}`);
|
|
11660
|
-
}
|
|
11661
|
-
async hasCompletedSync() {
|
|
11662
|
-
if (this._hasCompletedSync) {
|
|
11663
|
-
return true;
|
|
11664
|
-
}
|
|
11665
|
-
const r = await this.db.get(`SELECT powersync_last_synced_at() as synced_at`);
|
|
11666
|
-
const completed = r.synced_at != null;
|
|
11667
|
-
if (completed) {
|
|
11668
|
-
this._hasCompletedSync = true;
|
|
11669
|
-
}
|
|
11670
|
-
return completed;
|
|
11671
|
-
}
|
|
11672
|
-
async syncLocalDatabase(checkpoint, priority) {
|
|
11673
|
-
const r = await this.validateChecksums(checkpoint, priority);
|
|
11674
|
-
if (!r.checkpointValid) {
|
|
11675
|
-
this.logger.error('Checksums failed for', r.checkpointFailures);
|
|
11676
|
-
for (const b of r.checkpointFailures ?? []) {
|
|
11677
|
-
await this.deleteBucket(b);
|
|
11678
|
-
}
|
|
11679
|
-
return { ready: false, checkpointValid: false, checkpointFailures: r.checkpointFailures };
|
|
11680
|
-
}
|
|
11681
|
-
if (priority == null) {
|
|
11682
|
-
this.logger.debug(`Validated checksums checkpoint ${checkpoint.last_op_id}`);
|
|
11683
|
-
}
|
|
11684
|
-
else {
|
|
11685
|
-
this.logger.debug(`Validated checksums for partial checkpoint ${checkpoint.last_op_id}, priority ${priority}`);
|
|
11686
|
-
}
|
|
11687
|
-
let buckets = checkpoint.buckets;
|
|
11688
|
-
if (priority !== undefined) {
|
|
11689
|
-
buckets = buckets.filter((b) => hasMatchingPriority(priority, b));
|
|
11690
|
-
}
|
|
11691
|
-
const bucketNames = buckets.map((b) => b.bucket);
|
|
11692
|
-
await this.writeTransaction(async (tx) => {
|
|
11693
|
-
await tx.execute(`UPDATE ps_buckets SET last_op = ? WHERE name IN (SELECT json_each.value FROM json_each(?))`, [
|
|
11694
|
-
checkpoint.last_op_id,
|
|
11695
|
-
JSON.stringify(bucketNames)
|
|
11696
|
-
]);
|
|
11697
|
-
if (priority == null && checkpoint.write_checkpoint) {
|
|
11698
|
-
await tx.execute("UPDATE ps_buckets SET last_op = ? WHERE name = '$local'", [checkpoint.write_checkpoint]);
|
|
11699
|
-
}
|
|
11700
|
-
});
|
|
11701
|
-
const valid = await this.updateObjectsFromBuckets(checkpoint, priority);
|
|
11702
|
-
if (!valid) {
|
|
11703
|
-
return { ready: false, checkpointValid: true };
|
|
11704
|
-
}
|
|
11705
|
-
return {
|
|
11706
|
-
ready: true,
|
|
11707
|
-
checkpointValid: true
|
|
11708
|
-
};
|
|
11709
|
-
}
|
|
11710
|
-
/**
|
|
11711
|
-
* Atomically update the local state to the current checkpoint.
|
|
11712
|
-
*
|
|
11713
|
-
* This includes creating new tables, dropping old tables, and copying data over from the oplog.
|
|
11714
|
-
*/
|
|
11715
|
-
async updateObjectsFromBuckets(checkpoint, priority) {
|
|
11716
|
-
let arg = '';
|
|
11717
|
-
if (priority !== undefined) {
|
|
11718
|
-
const affectedBuckets = [];
|
|
11719
|
-
for (const desc of checkpoint.buckets) {
|
|
11720
|
-
if (hasMatchingPriority(priority, desc)) {
|
|
11721
|
-
affectedBuckets.push(desc.bucket);
|
|
11722
|
-
}
|
|
11723
|
-
}
|
|
11724
|
-
arg = JSON.stringify({ priority, buckets: affectedBuckets });
|
|
11725
|
-
}
|
|
11726
|
-
return this.writeTransaction(async (tx) => {
|
|
11727
|
-
const { insertId: result } = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
|
|
11728
|
-
'sync_local',
|
|
11729
|
-
arg
|
|
11730
|
-
]);
|
|
11731
|
-
if (result == 1) {
|
|
11732
|
-
if (priority == null) {
|
|
11733
|
-
const bucketToCount = Object.fromEntries(checkpoint.buckets.map((b) => [b.bucket, b.count]));
|
|
11734
|
-
// The two parameters could be replaced with one, but: https://github.com/powersync-ja/better-sqlite3/pull/6
|
|
11735
|
-
const jsonBucketCount = JSON.stringify(bucketToCount);
|
|
11736
|
-
await tx.execute("UPDATE ps_buckets SET count_since_last = 0, count_at_last = ?->name WHERE name != '$local' AND ?->name IS NOT NULL", [jsonBucketCount, jsonBucketCount]);
|
|
11737
|
-
}
|
|
11738
|
-
return true;
|
|
11739
|
-
}
|
|
11740
|
-
else {
|
|
11741
|
-
return false;
|
|
11742
|
-
}
|
|
11743
|
-
});
|
|
11744
|
-
}
|
|
11745
|
-
async validateChecksums(checkpoint, priority) {
|
|
11746
|
-
if (priority !== undefined) {
|
|
11747
|
-
// Only validate the buckets within the priority we care about
|
|
11748
|
-
const newBuckets = checkpoint.buckets.filter((cs) => hasMatchingPriority(priority, cs));
|
|
11749
|
-
checkpoint = { ...checkpoint, buckets: newBuckets };
|
|
11750
|
-
}
|
|
11751
|
-
const rs = await this.db.execute('SELECT powersync_validate_checkpoint(?) as result', [
|
|
11752
|
-
JSON.stringify({ ...checkpoint })
|
|
11753
|
-
]);
|
|
11754
|
-
const resultItem = rs.rows?.item(0);
|
|
11755
|
-
if (!resultItem) {
|
|
11756
|
-
return {
|
|
11757
|
-
checkpointValid: false,
|
|
11758
|
-
ready: false,
|
|
11759
|
-
checkpointFailures: []
|
|
11760
|
-
};
|
|
11761
|
-
}
|
|
11762
|
-
const result = JSON.parse(resultItem['result']);
|
|
11763
|
-
if (result['valid']) {
|
|
11764
|
-
return { ready: true, checkpointValid: true };
|
|
11765
|
-
}
|
|
11766
|
-
else {
|
|
11767
|
-
return {
|
|
11768
|
-
checkpointValid: false,
|
|
11769
|
-
ready: false,
|
|
11770
|
-
checkpointFailures: result['failed_buckets']
|
|
11771
|
-
};
|
|
11772
|
-
}
|
|
11773
|
-
}
|
|
11774
11189
|
async updateLocalTarget(cb) {
|
|
11775
11190
|
const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", [MAX_OP_ID]);
|
|
11776
11191
|
if (!rs1.length) {
|
|
@@ -11861,12 +11276,6 @@ class SqliteBucketStorage extends BaseObserver {
|
|
|
11861
11276
|
async writeTransaction(callback, options) {
|
|
11862
11277
|
return this.db.writeTransaction(callback, options);
|
|
11863
11278
|
}
|
|
11864
|
-
/**
|
|
11865
|
-
* Set a target checkpoint.
|
|
11866
|
-
*/
|
|
11867
|
-
async setTargetCheckpoint(checkpoint) {
|
|
11868
|
-
// No-op for now
|
|
11869
|
-
}
|
|
11870
11279
|
async control(op, payload) {
|
|
11871
11280
|
return await this.writeTransaction(async (tx) => {
|
|
11872
11281
|
const [[raw]] = await tx.executeRaw('SELECT powersync_control(?, ?)', [op, payload]);
|
|
@@ -11890,20 +11299,6 @@ class SqliteBucketStorage extends BaseObserver {
|
|
|
11890
11299
|
}
|
|
11891
11300
|
static _subkeyMigrationKey = 'powersync_js_migrated_subkeys';
|
|
11892
11301
|
}
|
|
11893
|
-
function hasMatchingPriority(priority, bucket) {
|
|
11894
|
-
return bucket.priority != null && bucket.priority <= priority;
|
|
11895
|
-
}
|
|
11896
|
-
|
|
11897
|
-
// TODO JSON
|
|
11898
|
-
class SyncDataBatch {
|
|
11899
|
-
buckets;
|
|
11900
|
-
static fromJSON(json) {
|
|
11901
|
-
return new SyncDataBatch(json.buckets.map((bucket) => SyncDataBucket.fromRow(bucket)));
|
|
11902
|
-
}
|
|
11903
|
-
constructor(buckets) {
|
|
11904
|
-
this.buckets = buckets;
|
|
11905
|
-
}
|
|
11906
|
-
}
|
|
11907
11302
|
|
|
11908
11303
|
/**
|
|
11909
11304
|
* Thrown when an underlying database connection is closed.
|
|
@@ -11963,10 +11358,8 @@ class Schema {
|
|
|
11963
11358
|
* developer instead of automatically by PowerSync.
|
|
11964
11359
|
* Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
|
|
11965
11360
|
* using client-side table and column constraints.
|
|
11966
|
-
* Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
|
|
11967
11361
|
*
|
|
11968
11362
|
* @param tables An object of (table name, raw table definition) entries.
|
|
11969
|
-
* @experimental Note that the raw tables API is still experimental and may change in the future.
|
|
11970
11363
|
*/
|
|
11971
11364
|
withRawTables(tables) {
|
|
11972
11365
|
for (const [name, rawTableDefinition] of Object.entries(tables)) {
|
|
@@ -12163,5 +11556,5 @@ const parseQuery = (query, parameters) => {
|
|
|
12163
11556
|
return { sqlStatement, parameters: parameters };
|
|
12164
11557
|
};
|
|
12165
11558
|
|
|
12166
|
-
export { ATTACHMENT_TABLE, AbortOperation, AbstractPowerSyncDatabase, AbstractPowerSyncDatabaseOpenFactory, AbstractQueryProcessor, AbstractRemote, AbstractStreamingSyncImplementation, ArrayComparator, AttachmentContext, AttachmentQueue, AttachmentService, AttachmentState, AttachmentTable, BaseObserver, Column, ColumnType, ConnectionClosedError, ConnectionManager, ControlledExecutor, CrudBatch, CrudEntry, CrudTransaction, DBAdapterDefaultMixin, DBGetUtilsDefaultMixin, DEFAULT_CRUD_BATCH_LIMIT, DEFAULT_CRUD_UPLOAD_THROTTLE_MS, DEFAULT_INDEX_COLUMN_OPTIONS, DEFAULT_INDEX_OPTIONS, DEFAULT_LOCK_TIMEOUT_MS, DEFAULT_POWERSYNC_CLOSE_OPTIONS, DEFAULT_POWERSYNC_DB_OPTIONS,
|
|
11559
|
+
export { ATTACHMENT_TABLE, AbortOperation, AbstractPowerSyncDatabase, AbstractPowerSyncDatabaseOpenFactory, AbstractQueryProcessor, AbstractRemote, AbstractStreamingSyncImplementation, ArrayComparator, AttachmentContext, AttachmentQueue, AttachmentService, AttachmentState, AttachmentTable, BaseObserver, Column, ColumnType, ConnectionClosedError, ConnectionManager, ControlledExecutor, CrudBatch, CrudEntry, CrudTransaction, DBAdapterDefaultMixin, DBGetUtilsDefaultMixin, DEFAULT_CRUD_BATCH_LIMIT, DEFAULT_CRUD_UPLOAD_THROTTLE_MS, DEFAULT_INDEX_COLUMN_OPTIONS, DEFAULT_INDEX_OPTIONS, DEFAULT_LOCK_TIMEOUT_MS, DEFAULT_POWERSYNC_CLOSE_OPTIONS, DEFAULT_POWERSYNC_DB_OPTIONS, DEFAULT_REMOTE_LOGGER, DEFAULT_REMOTE_OPTIONS, DEFAULT_RETRY_DELAY_MS, DEFAULT_ROW_COMPARATOR, DEFAULT_STREAMING_SYNC_OPTIONS, DEFAULT_STREAM_CONNECTION_OPTIONS, DEFAULT_SYNC_CLIENT_IMPLEMENTATION, DEFAULT_TABLE_OPTIONS, DEFAULT_WATCH_QUERY_OPTIONS, DEFAULT_WATCH_THROTTLE_MS, DiffTriggerOperation, DifferentialQueryProcessor, EMPTY_DIFFERENTIAL, EncodingType, FalsyComparator, FetchImplementationProvider, FetchStrategy, GetAllQuery, Index, IndexedColumn, InvalidSQLCharacters, LockType, LogLevel, MAX_AMOUNT_OF_COLUMNS, MAX_OP_ID, MEMORY_TRIGGER_CLAIM_MANAGER, Mutex, OnChangeQueryProcessor, PSInternalTable, PowerSyncControlCommand, RowUpdateType, Schema, Semaphore, SqliteBucketStorage, SyncClientImplementation, SyncProgress, SyncStatus, SyncStreamConnectionMethod, SyncingService, Table, TableV2, TriggerManagerImpl, UpdateType, UploadQueueStats, WatchedQueryListenerEvent, attachmentFromSql, column, compilableQueryWatch, createBaseLogger, createLogger, extractTableUpdates, isBatchedUpdateNotification, isDBAdapter, isPowerSyncDatabaseOptionsWithSettings, isSQLOpenFactory, isSQLOpenOptions, parseQuery, runOnSchemaChange, sanitizeSQL, sanitizeUUID, timeoutSignal };
|
|
12167
11560
|
//# sourceMappingURL=bundle.node.mjs.map
|