@powersync/common 1.51.0 → 1.53.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/dist/bundle.cjs +510 -1129
  2. package/dist/bundle.cjs.map +1 -1
  3. package/dist/bundle.mjs +511 -1116
  4. package/dist/bundle.mjs.map +1 -1
  5. package/dist/bundle.node.cjs +508 -1129
  6. package/dist/bundle.node.cjs.map +1 -1
  7. package/dist/bundle.node.mjs +509 -1116
  8. package/dist/bundle.node.mjs.map +1 -1
  9. package/dist/index.d.cts +73 -433
  10. package/legacy/sync_protocol.d.ts +103 -0
  11. package/lib/client/AbstractPowerSyncDatabase.js +3 -3
  12. package/lib/client/AbstractPowerSyncDatabase.js.map +1 -1
  13. package/lib/client/ConnectionManager.js +1 -1
  14. package/lib/client/ConnectionManager.js.map +1 -1
  15. package/lib/client/sync/bucket/BucketStorageAdapter.d.ts +6 -64
  16. package/lib/client/sync/bucket/BucketStorageAdapter.js +4 -0
  17. package/lib/client/sync/bucket/BucketStorageAdapter.js.map +1 -1
  18. package/lib/client/sync/bucket/SqliteBucketStorage.d.ts +1 -28
  19. package/lib/client/sync/bucket/SqliteBucketStorage.js +0 -162
  20. package/lib/client/sync/bucket/SqliteBucketStorage.js.map +1 -1
  21. package/lib/client/sync/stream/AbstractRemote.d.ts +29 -18
  22. package/lib/client/sync/stream/AbstractRemote.js +155 -188
  23. package/lib/client/sync/stream/AbstractRemote.js.map +1 -1
  24. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +13 -35
  25. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +150 -448
  26. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js.map +1 -1
  27. package/lib/client/sync/stream/JsonValue.d.ts +7 -0
  28. package/lib/client/sync/stream/JsonValue.js +2 -0
  29. package/lib/client/sync/stream/JsonValue.js.map +1 -0
  30. package/lib/client/sync/stream/core-instruction.d.ts +14 -9
  31. package/lib/client/sync/stream/core-instruction.js +3 -0
  32. package/lib/client/sync/stream/core-instruction.js.map +1 -1
  33. package/lib/db/DBAdapter.d.ts +9 -0
  34. package/lib/db/DBAdapter.js +8 -1
  35. package/lib/db/DBAdapter.js.map +1 -1
  36. package/lib/db/crud/SyncStatus.d.ts +3 -4
  37. package/lib/db/crud/SyncStatus.js +0 -4
  38. package/lib/db/crud/SyncStatus.js.map +1 -1
  39. package/lib/db/schema/RawTable.d.ts +0 -5
  40. package/lib/db/schema/Schema.d.ts +0 -2
  41. package/lib/db/schema/Schema.js +0 -2
  42. package/lib/db/schema/Schema.js.map +1 -1
  43. package/lib/index.d.ts +2 -6
  44. package/lib/index.js +1 -6
  45. package/lib/index.js.map +1 -1
  46. package/lib/utils/async.d.ts +0 -9
  47. package/lib/utils/async.js +0 -9
  48. package/lib/utils/async.js.map +1 -1
  49. package/lib/utils/stream_transform.d.ts +39 -0
  50. package/lib/utils/stream_transform.js +206 -0
  51. package/lib/utils/stream_transform.js.map +1 -0
  52. package/package.json +15 -10
  53. package/src/client/AbstractPowerSyncDatabase.ts +3 -3
  54. package/src/client/ConnectionManager.ts +1 -1
  55. package/src/client/sync/bucket/BucketStorageAdapter.ts +6 -71
  56. package/src/client/sync/bucket/SqliteBucketStorage.ts +1 -197
  57. package/src/client/sync/stream/AbstractRemote.ts +183 -229
  58. package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +181 -510
  59. package/src/client/sync/stream/JsonValue.ts +8 -0
  60. package/src/client/sync/stream/core-instruction.ts +15 -5
  61. package/src/db/DBAdapter.ts +20 -2
  62. package/src/db/crud/SyncStatus.ts +4 -5
  63. package/src/db/schema/RawTable.ts +0 -5
  64. package/src/db/schema/Schema.ts +0 -2
  65. package/src/index.ts +2 -6
  66. package/src/utils/async.ts +0 -11
  67. package/src/utils/stream_transform.ts +252 -0
  68. package/lib/client/sync/bucket/OpType.d.ts +0 -16
  69. package/lib/client/sync/bucket/OpType.js +0 -23
  70. package/lib/client/sync/bucket/OpType.js.map +0 -1
  71. package/lib/client/sync/bucket/OplogEntry.d.ts +0 -23
  72. package/lib/client/sync/bucket/OplogEntry.js +0 -36
  73. package/lib/client/sync/bucket/OplogEntry.js.map +0 -1
  74. package/lib/client/sync/bucket/SyncDataBatch.d.ts +0 -6
  75. package/lib/client/sync/bucket/SyncDataBatch.js +0 -12
  76. package/lib/client/sync/bucket/SyncDataBatch.js.map +0 -1
  77. package/lib/client/sync/bucket/SyncDataBucket.d.ts +0 -40
  78. package/lib/client/sync/bucket/SyncDataBucket.js +0 -40
  79. package/lib/client/sync/bucket/SyncDataBucket.js.map +0 -1
  80. package/lib/client/sync/stream/streaming-sync-types.d.ts +0 -143
  81. package/lib/client/sync/stream/streaming-sync-types.js +0 -26
  82. package/lib/client/sync/stream/streaming-sync-types.js.map +0 -1
  83. package/lib/utils/DataStream.d.ts +0 -62
  84. package/lib/utils/DataStream.js +0 -169
  85. package/lib/utils/DataStream.js.map +0 -1
  86. package/src/client/sync/bucket/OpType.ts +0 -23
  87. package/src/client/sync/bucket/OplogEntry.ts +0 -50
  88. package/src/client/sync/bucket/SyncDataBatch.ts +0 -11
  89. package/src/client/sync/bucket/SyncDataBucket.ts +0 -49
  90. package/src/client/sync/stream/streaming-sync-types.ts +0 -210
  91. package/src/utils/DataStream.ts +0 -222
@@ -1817,8 +1817,15 @@ class BaseTransaction {
1817
1817
  class TransactionImplementation extends DBGetUtilsDefaultMixin(BaseTransaction) {
1818
1818
  static async runWith(ctx, fn) {
1819
1819
  let tx = new TransactionImplementation(ctx);
1820
+ // For write transactions, use BEGIN IMMEDIATE to immediately obtain a write lock on the database (instead of doing
1821
+ // that on the first statement). If we have a genuine read-only connection, we also use BEGIN IMMEDIATE there: In
1822
+ // WAL mode, that ensures we pin the current state of the database (instead of the state at the first statement in
1823
+ // the transaction). But if we have a "fake" read-only connection implemented through `pragma query_only = true`, we
1824
+ // can't use this trick because it would attempt to lock the connection. So there, we use a regular `BEGIN`
1825
+ // statement.
1826
+ const useBeginImmediate = ctx.connectionType != 'queryOnly';
1820
1827
  try {
1821
- await ctx.execute('BEGIN IMMEDIATE');
1828
+ await ctx.execute(useBeginImmediate ? 'BEGIN IMMEDIATE' : 'BEGIN');
1822
1829
  const result = await fn(tx);
1823
1830
  await tx.commit();
1824
1831
  return result;
@@ -1977,16 +1984,12 @@ class SyncStatus {
1977
1984
  *
1978
1985
  * This returns null when the database is currently being opened and we don't have reliable information about all
1979
1986
  * included streams yet.
1980
- *
1981
- * @experimental Sync streams are currently in alpha.
1982
1987
  */
1983
1988
  get syncStreams() {
1984
1989
  return this.options.dataFlow?.internalStreamSubscriptions?.map((core) => new SyncStreamStatusView(this, core));
1985
1990
  }
1986
1991
  /**
1987
1992
  * If the `stream` appears in {@link syncStreams}, returns the current status for that stream.
1988
- *
1989
- * @experimental Sync streams are currently in alpha.
1990
1993
  */
1991
1994
  forStream(stream) {
1992
1995
  const asJson = JSON.stringify(stream.parameters);
@@ -2255,15 +2258,6 @@ class ControlledExecutor {
2255
2258
  }
2256
2259
  }
2257
2260
 
2258
- /**
2259
- * A ponyfill for `Symbol.asyncIterator` that is compatible with the
2260
- * [recommended polyfill](https://github.com/Azure/azure-sdk-for-js/blob/%40azure/core-asynciterator-polyfill_1.0.2/sdk/core/core-asynciterator-polyfill/src/index.ts#L4-L6)
2261
- * we recommend for React Native.
2262
- *
2263
- * As long as we use this symbol (instead of `for await` and `async *`) in this package, we can be compatible with async
2264
- * iterators without requiring them.
2265
- */
2266
- const symbolAsyncIterator = Symbol.asyncIterator ?? Symbol.for('Symbol.asyncIterator');
2267
2261
  /**
2268
2262
  * Throttle a function to be called at most once every "wait" milliseconds,
2269
2263
  * on the trailing edge.
@@ -2578,7 +2572,7 @@ class SyncStreamSubscriptionHandle {
2578
2572
  constructor(subscription) {
2579
2573
  this.subscription = subscription;
2580
2574
  subscription.refcount++;
2581
- _finalizer?.register(this, subscription);
2575
+ _finalizer?.register(this, subscription, this);
2582
2576
  }
2583
2577
  get name() {
2584
2578
  return this.subscription.name;
@@ -3167,6 +3161,10 @@ var PowerSyncControlCommand;
3167
3161
  PowerSyncControlCommand["NOTIFY_TOKEN_REFRESHED"] = "refreshed_token";
3168
3162
  PowerSyncControlCommand["NOTIFY_CRUD_UPLOAD_COMPLETED"] = "completed_upload";
3169
3163
  PowerSyncControlCommand["UPDATE_SUBSCRIPTIONS"] = "update_subscriptions";
3164
+ /**
3165
+ * An `established` or `end` event for response streams.
3166
+ */
3167
+ PowerSyncControlCommand["CONNECTION_STATE"] = "connection";
3170
3168
  })(PowerSyncControlCommand || (PowerSyncControlCommand = {}));
3171
3169
 
3172
3170
  /**
@@ -3348,103 +3346,6 @@ class AbortOperation extends Error {
3348
3346
  }
3349
3347
  }
3350
3348
 
3351
- var OpTypeEnum;
3352
- (function (OpTypeEnum) {
3353
- OpTypeEnum[OpTypeEnum["CLEAR"] = 1] = "CLEAR";
3354
- OpTypeEnum[OpTypeEnum["MOVE"] = 2] = "MOVE";
3355
- OpTypeEnum[OpTypeEnum["PUT"] = 3] = "PUT";
3356
- OpTypeEnum[OpTypeEnum["REMOVE"] = 4] = "REMOVE";
3357
- })(OpTypeEnum || (OpTypeEnum = {}));
3358
- /**
3359
- * Used internally for sync buckets.
3360
- */
3361
- class OpType {
3362
- value;
3363
- static fromJSON(jsonValue) {
3364
- return new OpType(OpTypeEnum[jsonValue]);
3365
- }
3366
- constructor(value) {
3367
- this.value = value;
3368
- }
3369
- toJSON() {
3370
- return Object.entries(OpTypeEnum).find(([, value]) => value === this.value)[0];
3371
- }
3372
- }
3373
-
3374
- class OplogEntry {
3375
- op_id;
3376
- op;
3377
- checksum;
3378
- subkey;
3379
- object_type;
3380
- object_id;
3381
- data;
3382
- static fromRow(row) {
3383
- return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, row.subkey, row.object_type, row.object_id, row.data);
3384
- }
3385
- constructor(op_id, op, checksum, subkey, object_type, object_id, data) {
3386
- this.op_id = op_id;
3387
- this.op = op;
3388
- this.checksum = checksum;
3389
- this.subkey = subkey;
3390
- this.object_type = object_type;
3391
- this.object_id = object_id;
3392
- this.data = data;
3393
- }
3394
- toJSON(fixedKeyEncoding = false) {
3395
- return {
3396
- op_id: this.op_id,
3397
- op: this.op.toJSON(),
3398
- object_type: this.object_type,
3399
- object_id: this.object_id,
3400
- checksum: this.checksum,
3401
- data: this.data,
3402
- // Older versions of the JS SDK used to always JSON.stringify here. That has always been wrong,
3403
- // but we need to migrate gradually to not break existing databases.
3404
- subkey: fixedKeyEncoding ? this.subkey : JSON.stringify(this.subkey)
3405
- };
3406
- }
3407
- }
3408
-
3409
- class SyncDataBucket {
3410
- bucket;
3411
- data;
3412
- has_more;
3413
- after;
3414
- next_after;
3415
- static fromRow(row) {
3416
- return new SyncDataBucket(row.bucket, row.data.map((entry) => OplogEntry.fromRow(entry)), row.has_more ?? false, row.after, row.next_after);
3417
- }
3418
- constructor(bucket, data,
3419
- /**
3420
- * True if the response does not contain all the data for this bucket, and another request must be made.
3421
- */
3422
- has_more,
3423
- /**
3424
- * The `after` specified in the request.
3425
- */
3426
- after,
3427
- /**
3428
- * Use this for the next request.
3429
- */
3430
- next_after) {
3431
- this.bucket = bucket;
3432
- this.data = data;
3433
- this.has_more = has_more;
3434
- this.after = after;
3435
- this.next_after = next_after;
3436
- }
3437
- toJSON(fixedKeyEncoding = false) {
3438
- return {
3439
- bucket: this.bucket,
3440
- has_more: this.has_more,
3441
- after: this.after,
3442
- next_after: this.next_after,
3443
- data: this.data.map((entry) => entry.toJSON(fixedKeyEncoding))
3444
- };
3445
- }
3446
- }
3447
-
3448
3349
  var dist = {};
3449
3350
 
3450
3351
  var Codecs = {};
@@ -8229,177 +8130,10 @@ function requireDist () {
8229
8130
 
8230
8131
  var distExports = requireDist();
8231
8132
 
8232
- var version = "1.51.0";
8133
+ var version = "1.53.0";
8233
8134
  var PACKAGE = {
8234
8135
  version: version};
8235
8136
 
8236
- const DEFAULT_PRESSURE_LIMITS = {
8237
- highWater: 10,
8238
- lowWater: 0
8239
- };
8240
- /**
8241
- * A very basic implementation of a data stream with backpressure support which does not use
8242
- * native JS streams or async iterators.
8243
- * This is handy for environments such as React Native which need polyfills for the above.
8244
- */
8245
- class DataStream extends BaseObserver {
8246
- options;
8247
- dataQueue;
8248
- isClosed;
8249
- processingPromise;
8250
- notifyDataAdded;
8251
- logger;
8252
- mapLine;
8253
- constructor(options) {
8254
- super();
8255
- this.options = options;
8256
- this.processingPromise = null;
8257
- this.isClosed = false;
8258
- this.dataQueue = [];
8259
- this.mapLine = options?.mapLine ?? ((line) => line);
8260
- this.logger = options?.logger ?? Logger.get('DataStream');
8261
- if (options?.closeOnError) {
8262
- const l = this.registerListener({
8263
- error: (ex) => {
8264
- l?.();
8265
- this.close();
8266
- }
8267
- });
8268
- }
8269
- }
8270
- get highWatermark() {
8271
- return this.options?.pressure?.highWaterMark ?? DEFAULT_PRESSURE_LIMITS.highWater;
8272
- }
8273
- get lowWatermark() {
8274
- return this.options?.pressure?.lowWaterMark ?? DEFAULT_PRESSURE_LIMITS.lowWater;
8275
- }
8276
- get closed() {
8277
- return this.isClosed;
8278
- }
8279
- async close() {
8280
- this.isClosed = true;
8281
- await this.processingPromise;
8282
- this.iterateListeners((l) => l.closed?.());
8283
- // Discard any data in the queue
8284
- this.dataQueue = [];
8285
- this.listeners.clear();
8286
- }
8287
- /**
8288
- * Enqueues data for the consumers to read
8289
- */
8290
- enqueueData(data) {
8291
- if (this.isClosed) {
8292
- throw new Error('Cannot enqueue data into closed stream.');
8293
- }
8294
- this.dataQueue.push(data);
8295
- this.notifyDataAdded?.();
8296
- this.processQueue();
8297
- }
8298
- /**
8299
- * Reads data once from the data stream
8300
- * @returns a Data payload or Null if the stream closed.
8301
- */
8302
- async read() {
8303
- if (this.closed) {
8304
- return null;
8305
- }
8306
- // Wait for any pending processing to complete first.
8307
- // This ensures we register our listener before calling processQueue(),
8308
- // avoiding a race where processQueue() sees no reader and returns early.
8309
- if (this.processingPromise) {
8310
- await this.processingPromise;
8311
- }
8312
- // Re-check after await - stream may have closed while we were waiting
8313
- if (this.closed) {
8314
- return null;
8315
- }
8316
- return new Promise((resolve, reject) => {
8317
- const l = this.registerListener({
8318
- data: async (data) => {
8319
- resolve(data);
8320
- // Remove the listener
8321
- l?.();
8322
- },
8323
- closed: () => {
8324
- resolve(null);
8325
- l?.();
8326
- },
8327
- error: (ex) => {
8328
- reject(ex);
8329
- l?.();
8330
- }
8331
- });
8332
- this.processQueue();
8333
- });
8334
- }
8335
- /**
8336
- * Executes a callback for each data item in the stream
8337
- */
8338
- forEach(callback) {
8339
- if (this.dataQueue.length <= this.lowWatermark) {
8340
- this.iterateAsyncErrored(async (l) => l.lowWater?.());
8341
- }
8342
- return this.registerListener({
8343
- data: callback
8344
- });
8345
- }
8346
- processQueue() {
8347
- if (this.processingPromise) {
8348
- return;
8349
- }
8350
- const promise = (this.processingPromise = this._processQueue());
8351
- promise.finally(() => {
8352
- this.processingPromise = null;
8353
- });
8354
- return promise;
8355
- }
8356
- hasDataReader() {
8357
- return Array.from(this.listeners.values()).some((l) => !!l.data);
8358
- }
8359
- async _processQueue() {
8360
- /**
8361
- * Allow listeners to mutate the queue before processing.
8362
- * This allows for operations such as dropping or compressing data
8363
- * on high water or requesting more data on low water.
8364
- */
8365
- if (this.dataQueue.length >= this.highWatermark) {
8366
- await this.iterateAsyncErrored(async (l) => l.highWater?.());
8367
- }
8368
- if (this.isClosed || !this.hasDataReader()) {
8369
- return;
8370
- }
8371
- if (this.dataQueue.length) {
8372
- const data = this.dataQueue.shift();
8373
- const mapped = this.mapLine(data);
8374
- await this.iterateAsyncErrored(async (l) => l.data?.(mapped));
8375
- }
8376
- if (this.dataQueue.length <= this.lowWatermark) {
8377
- const dataAdded = new Promise((resolve) => {
8378
- this.notifyDataAdded = resolve;
8379
- });
8380
- await Promise.race([this.iterateAsyncErrored(async (l) => l.lowWater?.()), dataAdded]);
8381
- this.notifyDataAdded = null;
8382
- }
8383
- if (this.dataQueue.length > 0) {
8384
- setTimeout(() => this.processQueue());
8385
- }
8386
- }
8387
- async iterateAsyncErrored(cb) {
8388
- // Important: We need to copy the listeners, as calling a listener could result in adding another
8389
- // listener, resulting in infinite loops.
8390
- const listeners = Array.from(this.listeners.values());
8391
- for (let i of listeners) {
8392
- try {
8393
- await cb(i);
8394
- }
8395
- catch (ex) {
8396
- this.logger.error(ex);
8397
- this.iterateListeners((l) => l.error?.(ex));
8398
- }
8399
- }
8400
- }
8401
- }
8402
-
8403
8137
  var WebsocketDuplexConnection = {};
8404
8138
 
8405
8139
  var hasRequiredWebsocketDuplexConnection;
@@ -8562,8 +8296,199 @@ class WebsocketClientTransport {
8562
8296
  }
8563
8297
  }
8564
8298
 
8299
+ const doneResult = { done: true, value: undefined };
8300
+ function valueResult(value) {
8301
+ return { done: false, value };
8302
+ }
8303
+ /**
8304
+ * Expands a source async iterator by allowing to inject events asynchronously.
8305
+ *
8306
+ * The resulting iterator will emit all events from its source. Additionally though, events can be injected. These
8307
+ * events are dropped once the main iterator completes, but are otherwise forwarded.
8308
+ *
8309
+ * The iterator completes when its source completes, and it supports backpressure by only calling `next()` on the source
8310
+ * in response to a `next()` call from downstream if no pending injected events can be dispatched.
8311
+ */
8312
+ function injectable(source) {
8313
+ let sourceIsDone = false;
8314
+ let waiter = undefined; // An active, waiting next() call.
8315
+ // A pending upstream event that couldn't be dispatched because inject() has been called before it was resolved.
8316
+ let pendingSourceEvent = null;
8317
+ let pendingInjectedEvents = [];
8318
+ const consumeWaiter = () => {
8319
+ const pending = waiter;
8320
+ waiter = undefined;
8321
+ return pending;
8322
+ };
8323
+ const fetchFromSource = () => {
8324
+ const resolveWaiter = (propagate) => {
8325
+ const active = consumeWaiter();
8326
+ if (active) {
8327
+ propagate(active);
8328
+ }
8329
+ else {
8330
+ pendingSourceEvent = propagate;
8331
+ }
8332
+ };
8333
+ const nextFromSource = source.next();
8334
+ nextFromSource.then((value) => {
8335
+ sourceIsDone = value.done == true;
8336
+ resolveWaiter((w) => w.resolve(value));
8337
+ }, (error) => {
8338
+ resolveWaiter((w) => w.reject(error));
8339
+ });
8340
+ };
8341
+ return {
8342
+ next: () => {
8343
+ return new Promise((resolve, reject) => {
8344
+ // First priority: Dispatch ready upstream events.
8345
+ if (sourceIsDone) {
8346
+ return resolve(doneResult);
8347
+ }
8348
+ if (pendingSourceEvent) {
8349
+ pendingSourceEvent({ resolve, reject });
8350
+ pendingSourceEvent = null;
8351
+ return;
8352
+ }
8353
+ // Second priority: Dispatch injected events
8354
+ if (pendingInjectedEvents.length) {
8355
+ return resolve(valueResult(pendingInjectedEvents.shift()));
8356
+ }
8357
+ // Nothing pending? Fetch from source
8358
+ waiter = { resolve, reject };
8359
+ return fetchFromSource();
8360
+ });
8361
+ },
8362
+ inject: (event) => {
8363
+ const pending = consumeWaiter();
8364
+ if (pending != null) {
8365
+ pending.resolve(valueResult(event));
8366
+ }
8367
+ else {
8368
+ pendingInjectedEvents.push(event);
8369
+ }
8370
+ }
8371
+ };
8372
+ }
8373
+ /**
8374
+ * Splits a byte stream at line endings, emitting each line as a string.
8375
+ */
8376
+ function extractJsonLines(source, decoder) {
8377
+ let buffer = '';
8378
+ const pendingLines = [];
8379
+ let isFinalEvent = false;
8380
+ return {
8381
+ next: async () => {
8382
+ while (true) {
8383
+ if (isFinalEvent) {
8384
+ return doneResult;
8385
+ }
8386
+ {
8387
+ const first = pendingLines.shift();
8388
+ if (first) {
8389
+ return { done: false, value: first };
8390
+ }
8391
+ }
8392
+ const { done, value } = await source.next();
8393
+ if (done) {
8394
+ const remaining = buffer.trim();
8395
+ if (remaining.length != 0) {
8396
+ isFinalEvent = true;
8397
+ return { done: false, value: remaining };
8398
+ }
8399
+ return doneResult;
8400
+ }
8401
+ const data = decoder.decode(value, { stream: true });
8402
+ buffer += data;
8403
+ const lines = buffer.split('\n');
8404
+ for (let i = 0; i < lines.length - 1; i++) {
8405
+ const l = lines[i].trim();
8406
+ if (l.length > 0) {
8407
+ pendingLines.push(l);
8408
+ }
8409
+ }
8410
+ buffer = lines[lines.length - 1];
8411
+ }
8412
+ }
8413
+ };
8414
+ }
8415
+ /**
8416
+ * Splits a concatenated stream of BSON objects by emitting individual objects.
8417
+ */
8418
+ function extractBsonObjects(source) {
8419
+ // Fully read but not emitted yet.
8420
+ const completedObjects = [];
8421
+ // Whether source has returned { done: true }. We do the same once completed objects have been emitted.
8422
+ let isDone = false;
8423
+ const lengthBuffer = new DataView(new ArrayBuffer(4));
8424
+ let objectBody = null;
8425
+ // If we're parsing the length field, a number between 1 and 4 (inclusive) describing remaining bytes in the header.
8426
+ // If we're consuming a document, the bytes remaining.
8427
+ let remainingLength = 4;
8428
+ return {
8429
+ async next() {
8430
+ while (true) {
8431
+ // Before fetching new data from upstream, return completed objects.
8432
+ if (completedObjects.length) {
8433
+ return valueResult(completedObjects.shift());
8434
+ }
8435
+ if (isDone) {
8436
+ return doneResult;
8437
+ }
8438
+ const upstreamEvent = await source.next();
8439
+ if (upstreamEvent.done) {
8440
+ isDone = true;
8441
+ if (objectBody || remainingLength != 4) {
8442
+ throw new Error('illegal end of stream in BSON object');
8443
+ }
8444
+ return doneResult;
8445
+ }
8446
+ const chunk = upstreamEvent.value;
8447
+ for (let i = 0; i < chunk.length;) {
8448
+ const availableInData = chunk.length - i;
8449
+ if (objectBody) {
8450
+ // We're in the middle of reading a BSON document.
8451
+ const bytesToRead = Math.min(availableInData, remainingLength);
8452
+ const copySource = new Uint8Array(chunk.buffer, chunk.byteOffset + i, bytesToRead);
8453
+ objectBody.set(copySource, objectBody.length - remainingLength);
8454
+ i += bytesToRead;
8455
+ remainingLength -= bytesToRead;
8456
+ if (remainingLength == 0) {
8457
+ completedObjects.push(objectBody);
8458
+ // Prepare to read another document, starting with its length
8459
+ objectBody = null;
8460
+ remainingLength = 4;
8461
+ }
8462
+ }
8463
+ else {
8464
+ // Copy up to 4 bytes into lengthBuffer, depending on how many we still need.
8465
+ const bytesToRead = Math.min(availableInData, remainingLength);
8466
+ for (let j = 0; j < bytesToRead; j++) {
8467
+ lengthBuffer.setUint8(4 - remainingLength + j, chunk[i + j]);
8468
+ }
8469
+ i += bytesToRead;
8470
+ remainingLength -= bytesToRead;
8471
+ if (remainingLength == 0) {
8472
+ // Transition from reading length header to reading document. Subtracting 4 because the length of the
8473
+ // header is included in length.
8474
+ const length = lengthBuffer.getInt32(0, true /* little endian */);
8475
+ remainingLength = length - 4;
8476
+ if (remainingLength < 1) {
8477
+ throw new Error(`invalid length for bson: ${length}`);
8478
+ }
8479
+ objectBody = new Uint8Array(length);
8480
+ new DataView(objectBody.buffer).setInt32(0, length, true);
8481
+ }
8482
+ }
8483
+ }
8484
+ }
8485
+ }
8486
+ };
8487
+ }
8488
+
8565
8489
  const POWERSYNC_TRAILING_SLASH_MATCH = /\/+$/;
8566
8490
  const POWERSYNC_JS_VERSION = PACKAGE.version;
8491
+ const SYNC_QUEUE_REQUEST_HIGH_WATER = 10;
8567
8492
  const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
8568
8493
  // Keep alive message is sent every period
8569
8494
  const KEEP_ALIVE_MS = 20_000;
@@ -8743,73 +8668,67 @@ class AbstractRemote {
8743
8668
  return new WebSocket(url);
8744
8669
  }
8745
8670
  /**
8746
- * Returns a data stream of sync line data.
8671
+ * Returns a data stream of sync line data, fetched via RSocket-over-WebSocket.
8747
8672
  *
8748
- * @param map Maps received payload frames to the typed event value.
8749
- * @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
8750
- * (required for compatibility with older sync services).
8673
+ * The only mechanism to abort the returned stream is to use the abort signal in {@link SocketSyncStreamOptions}.
8751
8674
  */
8752
- async socketStreamRaw(options, map, bson) {
8675
+ async socketStreamRaw(options) {
8753
8676
  const { path, fetchStrategy = FetchStrategy.Buffered } = options;
8754
- const mimeType = bson == null ? 'application/json' : 'application/bson';
8677
+ const mimeType = 'application/json';
8755
8678
  function toBuffer(js) {
8756
- let contents;
8757
- if (bson != null) {
8758
- contents = bson.serialize(js);
8759
- }
8760
- else {
8761
- contents = JSON.stringify(js);
8762
- }
8763
- return Buffer.from(contents);
8679
+ return Buffer.from(JSON.stringify(js));
8764
8680
  }
8765
8681
  const syncQueueRequestSize = fetchStrategy == FetchStrategy.Buffered ? 10 : 1;
8766
8682
  const request = await this.buildRequest(path);
8683
+ const url = this.options.socketUrlTransformer(request.url);
8767
8684
  // Add the user agent in the setup payload - we can't set custom
8768
8685
  // headers with websockets on web. The browser userAgent is however added
8769
8686
  // automatically as a header.
8770
8687
  const userAgent = this.getUserAgent();
8771
- const stream = new DataStream({
8772
- logger: this.logger,
8773
- pressure: {
8774
- lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER
8775
- },
8776
- mapLine: map
8777
- });
8688
+ // While we're connecting (a process that can't be aborted in RSocket), the WebSocket instance to close if we wanted
8689
+ // to abort the connection.
8690
+ let pendingSocket = null;
8691
+ let keepAliveTimeout;
8692
+ let rsocket = null;
8693
+ let queue = null;
8694
+ let didClose = false;
8695
+ const abortRequest = () => {
8696
+ if (didClose) {
8697
+ return;
8698
+ }
8699
+ didClose = true;
8700
+ clearTimeout(keepAliveTimeout);
8701
+ if (pendingSocket) {
8702
+ pendingSocket.close();
8703
+ }
8704
+ if (rsocket) {
8705
+ rsocket.close();
8706
+ }
8707
+ if (queue) {
8708
+ queue.stop();
8709
+ }
8710
+ };
8778
8711
  // Handle upstream abort
8779
- if (options.abortSignal?.aborted) {
8712
+ if (options.abortSignal.aborted) {
8780
8713
  throw new AbortOperation('Connection request aborted');
8781
8714
  }
8782
8715
  else {
8783
- options.abortSignal?.addEventListener('abort', () => {
8784
- stream.close();
8785
- }, { once: true });
8716
+ options.abortSignal.addEventListener('abort', abortRequest);
8786
8717
  }
8787
- let keepAliveTimeout;
8788
8718
  const resetTimeout = () => {
8789
8719
  clearTimeout(keepAliveTimeout);
8790
8720
  keepAliveTimeout = setTimeout(() => {
8791
8721
  this.logger.error(`No data received on WebSocket in ${SOCKET_TIMEOUT_MS}ms, closing connection.`);
8792
- stream.close();
8722
+ abortRequest();
8793
8723
  }, SOCKET_TIMEOUT_MS);
8794
8724
  };
8795
8725
  resetTimeout();
8796
- // Typescript complains about this being `never` if it's not assigned here.
8797
- // This is assigned in `wsCreator`.
8798
- let disposeSocketConnectionTimeout = () => { };
8799
- const url = this.options.socketUrlTransformer(request.url);
8800
8726
  const connector = new distExports.RSocketConnector({
8801
8727
  transport: new WebsocketClientTransport({
8802
8728
  url,
8803
8729
  wsCreator: (url) => {
8804
- const socket = this.createSocket(url);
8805
- disposeSocketConnectionTimeout = stream.registerListener({
8806
- closed: () => {
8807
- // Allow closing the underlying WebSocket if the stream was closed before the
8808
- // RSocket connect completed. This should effectively abort the request.
8809
- socket.close();
8810
- }
8811
- });
8812
- socket.addEventListener('message', (event) => {
8730
+ const socket = (pendingSocket = this.createSocket(url));
8731
+ socket.addEventListener('message', () => {
8813
8732
  resetTimeout();
8814
8733
  });
8815
8734
  return socket;
@@ -8829,43 +8748,40 @@ class AbstractRemote {
8829
8748
  }
8830
8749
  }
8831
8750
  });
8832
- let rsocket;
8833
8751
  try {
8834
8752
  rsocket = await connector.connect();
8835
8753
  // The connection is established, we no longer need to monitor the initial timeout
8836
- disposeSocketConnectionTimeout();
8754
+ pendingSocket = null;
8837
8755
  }
8838
8756
  catch (ex) {
8839
8757
  this.logger.error(`Failed to connect WebSocket`, ex);
8840
- clearTimeout(keepAliveTimeout);
8841
- if (!stream.closed) {
8842
- await stream.close();
8843
- }
8758
+ abortRequest();
8844
8759
  throw ex;
8845
8760
  }
8846
8761
  resetTimeout();
8847
- let socketIsClosed = false;
8848
- const closeSocket = () => {
8849
- clearTimeout(keepAliveTimeout);
8850
- if (socketIsClosed) {
8851
- return;
8852
- }
8853
- socketIsClosed = true;
8854
- rsocket.close();
8855
- };
8856
8762
  // Helps to prevent double close scenarios
8857
- rsocket.onClose(() => (socketIsClosed = true));
8858
- // We initially request this amount and expect these to arrive eventually
8859
- let pendingEventsCount = syncQueueRequestSize;
8860
- const disposeClosedListener = stream.registerListener({
8861
- closed: () => {
8862
- closeSocket();
8863
- disposeClosedListener();
8864
- }
8865
- });
8866
- const socket = await new Promise((resolve, reject) => {
8763
+ rsocket.onClose(() => (rsocket = null));
8764
+ return await new Promise((resolve, reject) => {
8867
8765
  let connectionEstablished = false;
8868
- const res = rsocket.requestStream({
8766
+ let pendingEventsCount = syncQueueRequestSize;
8767
+ let paused = false;
8768
+ let res = null;
8769
+ function requestMore() {
8770
+ const delta = syncQueueRequestSize - pendingEventsCount;
8771
+ if (!paused && delta > 0) {
8772
+ res?.request(delta);
8773
+ pendingEventsCount = syncQueueRequestSize;
8774
+ }
8775
+ }
8776
+ const events = new EventIterator((q) => {
8777
+ queue = q;
8778
+ q.on('highWater', () => (paused = true));
8779
+ q.on('lowWater', () => {
8780
+ paused = false;
8781
+ requestMore();
8782
+ });
8783
+ }, { highWaterMark: SYNC_QUEUE_REQUEST_HIGH_WATER, lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER })[Symbol.asyncIterator]();
8784
+ res = rsocket.requestStream({
8869
8785
  data: toBuffer(options.data),
8870
8786
  metadata: toBuffer({
8871
8787
  path
@@ -8890,7 +8806,7 @@ class AbstractRemote {
8890
8806
  }
8891
8807
  // RSocket will close the RSocket stream automatically
8892
8808
  // Close the downstream stream as well - this will close the RSocket connection and WebSocket
8893
- stream.close();
8809
+ abortRequest();
8894
8810
  // Handles cases where the connection failed e.g. auth error or connection error
8895
8811
  if (!connectionEstablished) {
8896
8812
  reject(e);
@@ -8900,41 +8816,40 @@ class AbstractRemote {
8900
8816
  // The connection is active
8901
8817
  if (!connectionEstablished) {
8902
8818
  connectionEstablished = true;
8903
- resolve(res);
8819
+ resolve(events);
8904
8820
  }
8905
8821
  const { data } = payload;
8822
+ if (data) {
8823
+ queue.push(data);
8824
+ }
8906
8825
  // Less events are now pending
8907
8826
  pendingEventsCount--;
8908
- if (!data) {
8909
- return;
8910
- }
8911
- stream.enqueueData(data);
8827
+ // Request another event (unless the downstream consumer is paused).
8828
+ requestMore();
8912
8829
  },
8913
8830
  onComplete: () => {
8914
- stream.close();
8831
+ abortRequest(); // this will also emit a done event
8915
8832
  },
8916
8833
  onExtension: () => { }
8917
8834
  });
8918
8835
  });
8919
- const l = stream.registerListener({
8920
- lowWater: async () => {
8921
- // Request to fill up the queue
8922
- const required = syncQueueRequestSize - pendingEventsCount;
8923
- if (required > 0) {
8924
- socket.request(syncQueueRequestSize - pendingEventsCount);
8925
- pendingEventsCount = syncQueueRequestSize;
8926
- }
8927
- },
8928
- closed: () => {
8929
- l();
8930
- }
8931
- });
8932
- return stream;
8933
8836
  }
8934
8837
  /**
8935
- * Connects to the sync/stream http endpoint, mapping and emitting each received string line.
8838
+ * @returns Whether the HTTP implementation on this platform can receive streamed binary responses. This is true on
8839
+ * all platforms except React Native (who would have guessed...), where we must not request BSON responses.
8840
+ *
8841
+ * @see https://github.com/react-native-community/fetch?tab=readme-ov-file#motivation
8842
+ */
8843
+ get supportsStreamingBinaryResponses() {
8844
+ return true;
8845
+ }
8846
+ /**
8847
+ * Posts a `/sync/stream` request, asserts that it completes successfully and returns the streaming response as an
8848
+ * async iterator of byte blobs.
8849
+ *
8850
+ * To cancel the async iterator, use the abort signal from {@link SyncStreamOptions} passed to this method.
8936
8851
  */
8937
- async postStreamRaw(options, mapLine) {
8852
+ async fetchStreamRaw(options) {
8938
8853
  const { data, path, headers, abortSignal } = options;
8939
8854
  const request = await this.buildRequest(path);
8940
8855
  /**
@@ -8946,119 +8861,94 @@ class AbstractRemote {
8946
8861
  * Aborting the active fetch request while it is being consumed seems to throw
8947
8862
  * an unhandled exception on the window level.
8948
8863
  */
8949
- if (abortSignal?.aborted) {
8950
- throw new AbortOperation('Abort request received before making postStreamRaw request');
8864
+ if (abortSignal.aborted) {
8865
+ throw new AbortOperation('Abort request received before making fetchStreamRaw request');
8951
8866
  }
8952
8867
  const controller = new AbortController();
8953
- let requestResolved = false;
8954
- abortSignal?.addEventListener('abort', () => {
8955
- if (!requestResolved) {
8868
+ let reader = null;
8869
+ abortSignal.addEventListener('abort', () => {
8870
+ const reason = abortSignal.reason ??
8871
+ new AbortOperation('Cancelling network request before it resolves. Abort signal has been received.');
8872
+ if (reader == null) {
8956
8873
  // Only abort via the abort controller if the request has not resolved yet
8957
- controller.abort(abortSignal.reason ??
8958
- new AbortOperation('Cancelling network request before it resolves. Abort signal has been received.'));
8874
+ controller.abort(reason);
8875
+ }
8876
+ else {
8877
+ reader.cancel(reason).catch(() => {
8878
+ // Cancelling the reader might rethrow an exception we would have handled by throwing in next(). So we can
8879
+ // ignore it here.
8880
+ });
8959
8881
  }
8960
8882
  });
8961
- const res = await this.fetch(request.url, {
8962
- method: 'POST',
8963
- headers: { ...headers, ...request.headers },
8964
- body: JSON.stringify(data),
8965
- signal: controller.signal,
8966
- cache: 'no-store',
8967
- ...(this.options.fetchOptions ?? {}),
8968
- ...options.fetchOptions
8969
- }).catch((ex) => {
8883
+ let res;
8884
+ let responseIsBson = false;
8885
+ try {
8886
+ const ndJson = 'application/x-ndjson';
8887
+ const bson = 'application/vnd.powersync.bson-stream';
8888
+ res = await this.fetch(request.url, {
8889
+ method: 'POST',
8890
+ headers: {
8891
+ ...headers,
8892
+ ...request.headers,
8893
+ accept: this.supportsStreamingBinaryResponses ? `${bson};q=0.9,${ndJson};q=0.8` : ndJson
8894
+ },
8895
+ body: JSON.stringify(data),
8896
+ signal: controller.signal,
8897
+ cache: 'no-store',
8898
+ ...(this.options.fetchOptions ?? {}),
8899
+ ...options.fetchOptions
8900
+ });
8901
+ if (!res.ok || !res.body) {
8902
+ const text = await res.text();
8903
+ this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
8904
+ const error = new Error(`HTTP ${res.statusText}: ${text}`);
8905
+ error.status = res.status;
8906
+ throw error;
8907
+ }
8908
+ const contentType = res.headers.get('content-type');
8909
+ responseIsBson = contentType == bson;
8910
+ }
8911
+ catch (ex) {
8970
8912
  if (ex.name == 'AbortError') {
8971
8913
  throw new AbortOperation(`Pending fetch request to ${request.url} has been aborted.`);
8972
8914
  }
8973
8915
  throw ex;
8974
- });
8975
- if (!res) {
8976
- throw new Error('Fetch request was aborted');
8977
- }
8978
- requestResolved = true;
8979
- if (!res.ok || !res.body) {
8980
- const text = await res.text();
8981
- this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
8982
- const error = new Error(`HTTP ${res.statusText}: ${text}`);
8983
- error.status = res.status;
8984
- throw error;
8985
8916
  }
8986
- // Create a new stream splitting the response at line endings while also handling cancellations
8987
- // by closing the reader.
8988
- const reader = res.body.getReader();
8989
- let readerReleased = false;
8990
- // This will close the network request and read stream
8991
- const closeReader = async () => {
8992
- try {
8993
- readerReleased = true;
8994
- await reader.cancel();
8995
- }
8996
- catch (ex) {
8997
- // an error will throw if the reader hasn't been used yet
8998
- }
8999
- reader.releaseLock();
9000
- };
9001
- const stream = new DataStream({
9002
- logger: this.logger,
9003
- mapLine: mapLine,
9004
- pressure: {
9005
- highWaterMark: 20,
9006
- lowWaterMark: 10
9007
- }
9008
- });
9009
- abortSignal?.addEventListener('abort', () => {
9010
- closeReader();
9011
- stream.close();
9012
- });
9013
- const decoder = this.createTextDecoder();
9014
- let buffer = '';
9015
- const consumeStream = async () => {
9016
- while (!stream.closed && !abortSignal?.aborted && !readerReleased) {
9017
- const { done, value } = await reader.read();
9018
- if (done) {
9019
- const remaining = buffer.trim();
9020
- if (remaining.length != 0) {
9021
- stream.enqueueData(remaining);
9022
- }
9023
- stream.close();
9024
- await closeReader();
9025
- return;
8917
+ reader = res.body.getReader();
8918
+ const stream = {
8919
+ next: async () => {
8920
+ if (controller.signal.aborted) {
8921
+ return doneResult;
9026
8922
  }
9027
- const data = decoder.decode(value, { stream: true });
9028
- buffer += data;
9029
- const lines = buffer.split('\n');
9030
- for (var i = 0; i < lines.length - 1; i++) {
9031
- var l = lines[i].trim();
9032
- if (l.length > 0) {
9033
- stream.enqueueData(l);
9034
- }
8923
+ try {
8924
+ return await reader.read();
9035
8925
  }
9036
- buffer = lines[lines.length - 1];
9037
- // Implement backpressure by waiting for the low water mark to be reached
9038
- if (stream.dataQueue.length > stream.highWatermark) {
9039
- await new Promise((resolve) => {
9040
- const dispose = stream.registerListener({
9041
- lowWater: async () => {
9042
- resolve();
9043
- dispose();
9044
- },
9045
- closed: () => {
9046
- resolve();
9047
- dispose();
9048
- }
9049
- });
9050
- });
8926
+ catch (ex) {
8927
+ if (controller.signal.aborted) {
8928
+ // .read() completes with an error if we cancel the reader, which we do to disconnect. So this is just
8929
+ // things working as intended, we can return a done event and consider the exception handled.
8930
+ return doneResult;
8931
+ }
8932
+ throw ex;
9051
8933
  }
9052
8934
  }
9053
8935
  };
9054
- consumeStream().catch(ex => this.logger.error('Error consuming stream', ex));
9055
- const l = stream.registerListener({
9056
- closed: () => {
9057
- closeReader();
9058
- l?.();
9059
- }
9060
- });
9061
- return stream;
8936
+ return { isBson: responseIsBson, stream };
8937
+ }
8938
+ /**
8939
+ * Posts a `/sync/stream` request.
8940
+ *
8941
+ * Depending on the `Content-Type` of the response, this returns strings for sync lines or encoded BSON documents as
8942
+ * {@link Uint8Array}s.
8943
+ */
8944
+ async fetchStream(options) {
8945
+ const { isBson, stream } = await this.fetchStreamRaw(options);
8946
+ if (isBson) {
8947
+ return extractBsonObjects(stream);
8948
+ }
8949
+ else {
8950
+ return extractJsonLines(stream, this.createTextDecoder());
8951
+ }
9062
8952
  }
9063
8953
  }
9064
8954
 
@@ -9087,31 +8977,8 @@ function coreStatusToJs(status) {
9087
8977
  priorityStatusEntries: status.priority_status.map(priorityToJs)
9088
8978
  };
9089
8979
  }
9090
-
9091
- function isStreamingSyncData(line) {
9092
- return line.data != null;
9093
- }
9094
- function isStreamingKeepalive(line) {
9095
- return line.token_expires_in != null;
9096
- }
9097
- function isStreamingSyncCheckpoint(line) {
9098
- return line.checkpoint != null;
9099
- }
9100
- function isStreamingSyncCheckpointComplete(line) {
9101
- return line.checkpoint_complete != null;
9102
- }
9103
- function isStreamingSyncCheckpointPartiallyComplete(line) {
9104
- return line.partial_checkpoint_complete != null;
9105
- }
9106
- function isStreamingSyncCheckpointDiff(line) {
9107
- return line.checkpoint_diff != null;
9108
- }
9109
- function isContinueCheckpointRequest(request) {
9110
- return (Array.isArray(request.buckets) &&
9111
- typeof request.checkpoint_token == 'string');
9112
- }
9113
- function isSyncNewCheckpointRequest(request) {
9114
- return typeof request.request_checkpoint == 'object';
8980
+ function isInterruptingInstruction(instruction) {
8981
+ return 'EstablishSyncStream' in instruction || 'CloseSyncStream' in instruction;
9115
8982
  }
9116
8983
 
9117
8984
  var LockType;
@@ -9126,35 +8993,21 @@ var SyncStreamConnectionMethod;
9126
8993
  })(SyncStreamConnectionMethod || (SyncStreamConnectionMethod = {}));
9127
8994
  var SyncClientImplementation;
9128
8995
  (function (SyncClientImplementation) {
9129
- /**
9130
- * Decodes and handles sync lines received from the sync service in JavaScript.
9131
- *
9132
- * This is the default option.
9133
- *
9134
- * @deprecated We recommend the {@link RUST} client implementation for all apps. If you have issues with
9135
- * the Rust client, please file an issue or reach out to us. The JavaScript client will be removed in a future
9136
- * version of the PowerSync SDK.
9137
- */
9138
- SyncClientImplementation["JAVASCRIPT"] = "js";
9139
8996
  /**
9140
8997
  * This implementation offloads the sync line decoding and handling into the PowerSync
9141
8998
  * core extension.
9142
8999
  *
9143
- * This option is more performant than the {@link JAVASCRIPT} client, enabled by default and the
9144
- * recommended client implementation for all apps.
9000
+ * This is the only option, as an older JavaScript client implementation has been removed from the SDK.
9145
9001
  *
9146
9002
  * ## Compatibility warning
9147
9003
  *
9148
9004
  * The Rust sync client stores sync data in a format that is slightly different than the one used
9149
- * by the old {@link JAVASCRIPT} implementation. When adopting the {@link RUST} client on existing
9150
- * databases, the PowerSync SDK will migrate the format automatically.
9151
- * Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
9152
- * the version introducing {@link RUST} as an option) also supports the new format, so you can switch
9153
- * back to {@link JAVASCRIPT} later.
9005
+ * by the old JavaScript client. When adopting the {@link RUST} client on existing databases, the PowerSync SDK will
9006
+ * migrate the format automatically.
9154
9007
  *
9155
- * __However__: Upgrading the SDK version, then adopting {@link RUST} as a sync client and later
9156
- * downgrading the SDK to an older version (necessarily using the JavaScript-based implementation then)
9157
- * can lead to sync issues.
9008
+ * SDK versions supporting both the JavaScript and the Rust client support both formats with the JavaScript client
9009
+ * implementaiton. However, downgrading to an SDK version that only supports the JavaScript client would not be
9010
+ * possible anymore. Problematic SDK versions have been released before 2025-06-09.
9158
9011
  */
9159
9012
  SyncClientImplementation["RUST"] = "rust";
9160
9013
  })(SyncClientImplementation || (SyncClientImplementation = {}));
@@ -9177,13 +9030,7 @@ const DEFAULT_STREAM_CONNECTION_OPTIONS = {
9177
9030
  serializedSchema: undefined,
9178
9031
  includeDefaultStreams: true
9179
9032
  };
9180
- // The priority we assume when we receive checkpoint lines where no priority is set.
9181
- // This is the default priority used by the sync service, but can be set to an arbitrary
9182
- // value since sync services without priorities also won't send partial sync completion
9183
- // messages.
9184
- const FALLBACK_PRIORITY = 3;
9185
9033
  class AbstractStreamingSyncImplementation extends BaseObserver {
9186
- _lastSyncedAt;
9187
9034
  options;
9188
9035
  abortController;
9189
9036
  // In rare cases, mostly for tests, uploads can be triggered without being properly connected.
@@ -9193,6 +9040,7 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
9193
9040
  streamingSyncPromise;
9194
9041
  logger;
9195
9042
  activeStreams;
9043
+ connectionMayHaveChanged = false;
9196
9044
  isUploadingCrud = false;
9197
9045
  notifyCompletedUploads;
9198
9046
  handleActiveStreamsChange;
@@ -9272,9 +9120,6 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
9272
9120
  this.crudUpdateListener = undefined;
9273
9121
  this.uploadAbortController?.abort();
9274
9122
  }
9275
- async hasCompletedSync() {
9276
- return this.options.adapter.hasCompletedSync();
9277
- }
9278
9123
  async getWriteCheckpoint() {
9279
9124
  const clientId = await this.options.adapter.getClientId();
9280
9125
  let path = `/write-checkpoint2.json?client_id=${clientId}`;
@@ -9356,7 +9201,7 @@ The next upload iteration will be delayed.`);
9356
9201
  });
9357
9202
  }
9358
9203
  }
9359
- this.uploadAbortController = null;
9204
+ this.uploadAbortController = undefined;
9360
9205
  }
9361
9206
  });
9362
9207
  }
@@ -9472,6 +9317,11 @@ The next upload iteration will be delayed.`);
9472
9317
  shouldDelayRetry = false;
9473
9318
  // A disconnect was requested, we should not delay since there is no explicit retry
9474
9319
  }
9320
+ else if (this.connectionMayHaveChanged && ex.message?.indexOf('No iteration is active') >= 0) {
9321
+ this.connectionMayHaveChanged = false;
9322
+ this.logger.info('Sync error after changed connection, retrying immediately');
9323
+ shouldDelayRetry = false;
9324
+ }
9475
9325
  else {
9476
9326
  this.logger.error(ex);
9477
9327
  }
@@ -9502,17 +9352,14 @@ The next upload iteration will be delayed.`);
9502
9352
  // Mark as disconnected if here
9503
9353
  this.updateSyncStatus({ connected: false, connecting: false });
9504
9354
  }
9505
- async collectLocalBucketState() {
9506
- const bucketEntries = await this.options.adapter.getBucketStates();
9507
- const req = bucketEntries.map((entry) => ({
9508
- name: entry.bucket,
9509
- after: entry.op_id
9510
- }));
9511
- const localDescriptions = new Map();
9512
- for (const entry of bucketEntries) {
9513
- localDescriptions.set(entry.bucket, null);
9514
- }
9515
- return [req, localDescriptions];
9355
+ markConnectionMayHaveChanged() {
9356
+ // By setting this field, we'll immediately retry if the next sync event causes an error triggered by us not having
9357
+ // an active sync iteration on the connection in use.
9358
+ this.connectionMayHaveChanged = true;
9359
+ // This triggers a `powersync_control` invocation if a sync iteration is currently active. This is a cheap call to
9360
+ // make when no subscriptions have actually changed, we're mainly interested in this immediately throwing if no
9361
+ // iteration is active. That allows us to reconnect ASAP, instead of having to wait for the next sync line.
9362
+ this.handleActiveStreamsChange?.();
9516
9363
  }
9517
9364
  /**
9518
9365
  * Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
@@ -9553,344 +9400,98 @@ The next upload iteration will be delayed.`);
9553
9400
  if (invalidMetadata.length > 0) {
9554
9401
  throw new Error(`Invalid appMetadata provided. Only string values are allowed. Invalid values: ${invalidMetadata.map(([key, value]) => `${key}: ${value}`).join(', ')}`);
9555
9402
  }
9556
- const clientImplementation = resolvedOptions.clientImplementation;
9557
- this.updateSyncStatus({ clientImplementation });
9558
- if (clientImplementation == SyncClientImplementation.JAVASCRIPT) {
9559
- await this.legacyStreamingSyncIteration(signal, resolvedOptions);
9560
- return null;
9561
- }
9562
- else {
9563
- await this.requireKeyFormat(true);
9564
- return await this.rustSyncIteration(signal, resolvedOptions);
9565
- }
9403
+ await this.requireKeyFormat(true);
9404
+ return await this.rustSyncIteration(signal, resolvedOptions);
9566
9405
  }
9567
9406
  });
9568
9407
  }
9569
- async legacyStreamingSyncIteration(signal, resolvedOptions) {
9570
- const rawTables = resolvedOptions.serializedSchema?.raw_tables;
9571
- if (rawTables != null && rawTables.length) {
9572
- this.logger.warn('Raw tables require the Rust-based sync client. The JS client will ignore them.');
9573
- }
9574
- if (this.activeStreams.length) {
9575
- this.logger.error('Sync streams require `clientImplementation: SyncClientImplementation.RUST` when connecting.');
9576
- }
9577
- this.logger.debug('Streaming sync iteration started');
9578
- this.options.adapter.startSession();
9579
- let [req, bucketMap] = await this.collectLocalBucketState();
9580
- let targetCheckpoint = null;
9581
- // A checkpoint that has been validated but not applied (e.g. due to pending local writes)
9582
- let pendingValidatedCheckpoint = null;
9583
- const clientId = await this.options.adapter.getClientId();
9584
- const usingFixedKeyFormat = await this.requireKeyFormat(false);
9585
- this.logger.debug('Requesting stream from server');
9586
- const syncOptions = {
9587
- path: '/sync/stream',
9588
- abortSignal: signal,
9589
- data: {
9590
- buckets: req,
9591
- include_checksum: true,
9592
- raw_data: true,
9593
- parameters: resolvedOptions.params,
9594
- app_metadata: resolvedOptions.appMetadata,
9595
- client_id: clientId
9408
+ receiveSyncLines(data) {
9409
+ const { options, connection } = data;
9410
+ const remote = this.options.remote;
9411
+ const openInner = async () => {
9412
+ if (connection.connectionMethod == SyncStreamConnectionMethod.HTTP) {
9413
+ return await remote.fetchStream(options);
9596
9414
  }
9597
- };
9598
- let stream;
9599
- if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
9600
- stream = await this.options.remote.postStreamRaw(syncOptions, (line) => {
9601
- if (typeof line == 'string') {
9602
- return JSON.parse(line);
9603
- }
9604
- else {
9605
- // Directly enqueued by us
9606
- return line;
9607
- }
9608
- });
9609
- }
9610
- else {
9611
- const bson = await this.options.remote.getBSON();
9612
- stream = await this.options.remote.socketStreamRaw({
9613
- ...syncOptions,
9614
- ...{ fetchStrategy: resolvedOptions.fetchStrategy }
9615
- }, (payload) => {
9616
- if (payload instanceof Uint8Array) {
9617
- return bson.deserialize(payload);
9618
- }
9619
- else {
9620
- // Directly enqueued by us
9621
- return payload;
9622
- }
9623
- }, bson);
9624
- }
9625
- this.logger.debug('Stream established. Processing events');
9626
- this.notifyCompletedUploads = () => {
9627
- if (!stream.closed) {
9628
- stream.enqueueData({ crud_upload_completed: null });
9415
+ else {
9416
+ return await this.options.remote.socketStreamRaw({
9417
+ ...options,
9418
+ ...{ fetchStrategy: connection.fetchStrategy }
9419
+ });
9629
9420
  }
9630
9421
  };
9631
- while (!stream.closed) {
9632
- const line = await stream.read();
9633
- if (!line) {
9634
- // The stream has closed while waiting
9635
- return;
9636
- }
9637
- if ('crud_upload_completed' in line) {
9638
- if (pendingValidatedCheckpoint != null) {
9639
- const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
9640
- if (applied) {
9641
- pendingValidatedCheckpoint = null;
9642
- }
9643
- else if (endIteration) {
9644
- break;
9645
- }
9422
+ let inner;
9423
+ let done = false;
9424
+ return {
9425
+ async next() {
9426
+ if (done) {
9427
+ return doneResult;
9646
9428
  }
9647
- continue;
9648
- }
9649
- // A connection is active and messages are being received
9650
- if (!this.syncStatus.connected) {
9651
- // There is a connection now
9652
- Promise.resolve().then(() => this.triggerCrudUpload());
9653
- this.updateSyncStatus({
9654
- connected: true
9655
- });
9656
- }
9657
- if (isStreamingSyncCheckpoint(line)) {
9658
- targetCheckpoint = line.checkpoint;
9659
- // New checkpoint - existing validated checkpoint is no longer valid
9660
- pendingValidatedCheckpoint = null;
9661
- const bucketsToDelete = new Set(bucketMap.keys());
9662
- const newBuckets = new Map();
9663
- for (const checksum of line.checkpoint.buckets) {
9664
- newBuckets.set(checksum.bucket, {
9665
- name: checksum.bucket,
9666
- priority: checksum.priority ?? FALLBACK_PRIORITY
9429
+ else if (inner == null) {
9430
+ inner = await openInner();
9431
+ // We're connected here, so we can tell the core extension about it.
9432
+ return valueResult({
9433
+ command: PowerSyncControlCommand.CONNECTION_STATE,
9434
+ payload: 'established'
9667
9435
  });
9668
- bucketsToDelete.delete(checksum.bucket);
9669
- }
9670
- if (bucketsToDelete.size > 0) {
9671
- this.logger.debug('Removing buckets', [...bucketsToDelete]);
9672
- }
9673
- bucketMap = newBuckets;
9674
- await this.options.adapter.removeBuckets([...bucketsToDelete]);
9675
- await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
9676
- await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
9677
- }
9678
- else if (isStreamingSyncCheckpointComplete(line)) {
9679
- const result = await this.applyCheckpoint(targetCheckpoint);
9680
- if (result.endIteration) {
9681
- return;
9682
- }
9683
- else if (!result.applied) {
9684
- // "Could not apply checkpoint due to local data". We need to retry after
9685
- // finishing uploads.
9686
- pendingValidatedCheckpoint = targetCheckpoint;
9687
9436
  }
9688
9437
  else {
9689
- // Nothing to retry later. This would likely already be null from the last
9690
- // checksum or checksum_diff operation, but we make sure.
9691
- pendingValidatedCheckpoint = null;
9692
- }
9693
- }
9694
- else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
9695
- const priority = line.partial_checkpoint_complete.priority;
9696
- this.logger.debug('Partial checkpoint complete', priority);
9697
- const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint, priority);
9698
- if (!result.checkpointValid) {
9699
- // This means checksums failed. Start again with a new checkpoint.
9700
- // TODO: better back-off
9701
- await new Promise((resolve) => setTimeout(resolve, 50));
9702
- return;
9703
- }
9704
- else if (!result.ready) ;
9705
- else {
9706
- // We'll keep on downloading, but can report that this priority is synced now.
9707
- this.logger.debug('partial checkpoint validation succeeded');
9708
- // All states with a higher priority can be deleted since this partial sync includes them.
9709
- const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
9710
- priorityStates.push({
9711
- priority,
9712
- lastSyncedAt: new Date(),
9713
- hasSynced: true
9714
- });
9715
- this.updateSyncStatus({
9716
- connected: true,
9717
- priorityStatusEntries: priorityStates
9718
- });
9719
- }
9720
- }
9721
- else if (isStreamingSyncCheckpointDiff(line)) {
9722
- // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
9723
- if (targetCheckpoint == null) {
9724
- throw new Error('Checkpoint diff without previous checkpoint');
9725
- }
9726
- // New checkpoint - existing validated checkpoint is no longer valid
9727
- pendingValidatedCheckpoint = null;
9728
- const diff = line.checkpoint_diff;
9729
- const newBuckets = new Map();
9730
- for (const checksum of targetCheckpoint.buckets) {
9731
- newBuckets.set(checksum.bucket, checksum);
9732
- }
9733
- for (const checksum of diff.updated_buckets) {
9734
- newBuckets.set(checksum.bucket, checksum);
9735
- }
9736
- for (const bucket of diff.removed_buckets) {
9737
- newBuckets.delete(bucket);
9738
- }
9739
- const newCheckpoint = {
9740
- last_op_id: diff.last_op_id,
9741
- buckets: [...newBuckets.values()],
9742
- write_checkpoint: diff.write_checkpoint
9743
- };
9744
- targetCheckpoint = newCheckpoint;
9745
- await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
9746
- bucketMap = new Map();
9747
- newBuckets.forEach((checksum, name) => bucketMap.set(name, {
9748
- name: checksum.bucket,
9749
- priority: checksum.priority ?? FALLBACK_PRIORITY
9750
- }));
9751
- const bucketsToDelete = diff.removed_buckets;
9752
- if (bucketsToDelete.length > 0) {
9753
- this.logger.debug('Remove buckets', bucketsToDelete);
9754
- }
9755
- await this.options.adapter.removeBuckets(bucketsToDelete);
9756
- await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
9757
- }
9758
- else if (isStreamingSyncData(line)) {
9759
- const { data } = line;
9760
- const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
9761
- let updatedProgress = null;
9762
- if (previousProgress) {
9763
- updatedProgress = { ...previousProgress };
9764
- const progressForBucket = updatedProgress[data.bucket];
9765
- if (progressForBucket) {
9766
- updatedProgress[data.bucket] = {
9767
- ...progressForBucket,
9768
- since_last: progressForBucket.since_last + data.data.length
9769
- };
9438
+ const event = await inner.next();
9439
+ if (event.done) {
9440
+ done = true;
9441
+ return valueResult({ command: PowerSyncControlCommand.CONNECTION_STATE, payload: 'end' });
9770
9442
  }
9771
- }
9772
- this.updateSyncStatus({
9773
- dataFlow: {
9774
- downloading: true,
9775
- downloadProgress: updatedProgress
9443
+ else {
9444
+ return valueResult({
9445
+ command: typeof event.value == 'string'
9446
+ ? PowerSyncControlCommand.PROCESS_TEXT_LINE
9447
+ : PowerSyncControlCommand.PROCESS_BSON_LINE,
9448
+ payload: event.value
9449
+ });
9776
9450
  }
9777
- });
9778
- await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] }, usingFixedKeyFormat);
9779
- }
9780
- else if (isStreamingKeepalive(line)) {
9781
- const remaining_seconds = line.token_expires_in;
9782
- if (remaining_seconds == 0) {
9783
- // Connection would be closed automatically right after this
9784
- this.logger.debug('Token expiring; reconnect');
9785
- /**
9786
- * For a rare case where the backend connector does not update the token
9787
- * (uses the same one), this should have some delay.
9788
- */
9789
- await this.delayRetry();
9790
- return;
9791
- }
9792
- else if (remaining_seconds < 30) {
9793
- this.logger.debug('Token will expire soon; reconnect');
9794
- // Pre-emptively refresh the token
9795
- this.options.remote.invalidateCredentials();
9796
- return;
9797
9451
  }
9798
- this.triggerCrudUpload();
9799
- }
9800
- else {
9801
- this.logger.debug('Received unknown sync line', line);
9802
9452
  }
9803
- }
9804
- this.logger.debug('Stream input empty');
9805
- // Connection closed. Likely due to auth issue.
9806
- return;
9453
+ };
9807
9454
  }
9808
9455
  async rustSyncIteration(signal, resolvedOptions) {
9809
9456
  const syncImplementation = this;
9810
9457
  const adapter = this.options.adapter;
9811
9458
  const remote = this.options.remote;
9812
- let receivingLines = null;
9813
- let hadSyncLine = false;
9814
9459
  let hideDisconnectOnRestart = false;
9460
+ let notifyTokenRefreshed;
9815
9461
  if (signal.aborted) {
9816
9462
  throw new AbortOperation('Connection request has been aborted');
9817
9463
  }
9818
- const abortController = new AbortController();
9819
- signal.addEventListener('abort', () => abortController.abort());
9820
- // Pending sync lines received from the service, as well as local events that trigger a powersync_control
9821
- // invocation (local events include refreshed tokens and completed uploads).
9822
- // This is a single data stream so that we can handle all control calls from a single place.
9823
- let controlInvocations = null;
9824
- async function connect(instr) {
9825
- const syncOptions = {
9826
- path: '/sync/stream',
9827
- abortSignal: abortController.signal,
9828
- data: instr.request
9464
+ function startCommand() {
9465
+ const options = {
9466
+ parameters: resolvedOptions.params,
9467
+ app_metadata: resolvedOptions.appMetadata,
9468
+ active_streams: syncImplementation.activeStreams,
9469
+ include_defaults: resolvedOptions.includeDefaultStreams
9829
9470
  };
9830
- if (resolvedOptions.connectionMethod == SyncStreamConnectionMethod.HTTP) {
9831
- controlInvocations = await remote.postStreamRaw(syncOptions, (line) => {
9832
- if (typeof line == 'string') {
9833
- return {
9834
- command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
9835
- payload: line
9836
- };
9837
- }
9838
- else {
9839
- // Directly enqueued by us
9840
- return line;
9841
- }
9842
- });
9843
- }
9844
- else {
9845
- controlInvocations = await remote.socketStreamRaw({
9846
- ...syncOptions,
9847
- fetchStrategy: resolvedOptions.fetchStrategy
9848
- }, (payload) => {
9849
- if (payload instanceof Uint8Array) {
9850
- return {
9851
- command: PowerSyncControlCommand.PROCESS_BSON_LINE,
9852
- payload: payload
9853
- };
9854
- }
9855
- else {
9856
- // Directly enqueued by us
9857
- return payload;
9858
- }
9859
- });
9860
- }
9861
- // The rust client will set connected: true after the first sync line because that's when it gets invoked, but
9862
- // we're already connected here and can report that.
9863
- syncImplementation.updateSyncStatus({ connected: true });
9864
- try {
9865
- while (!controlInvocations.closed) {
9866
- const line = await controlInvocations.read();
9867
- if (line == null) {
9868
- return;
9869
- }
9870
- await control(line.command, line.payload);
9871
- if (!hadSyncLine) {
9872
- syncImplementation.triggerCrudUpload();
9873
- hadSyncLine = true;
9874
- }
9875
- }
9876
- }
9877
- finally {
9878
- const activeInstructions = controlInvocations;
9879
- // We concurrently add events to the active data stream when e.g. a CRUD upload is completed or a token is
9880
- // refreshed. That would throw after closing (and we can't handle those events either way), so set this back
9881
- // to null.
9882
- controlInvocations = null;
9883
- await activeInstructions.close();
9471
+ if (resolvedOptions.serializedSchema) {
9472
+ options.schema = resolvedOptions.serializedSchema;
9884
9473
  }
9474
+ return invokePowerSyncControl(PowerSyncControlCommand.START, JSON.stringify(options));
9885
9475
  }
9886
9476
  async function stop() {
9887
- await control(PowerSyncControlCommand.STOP);
9477
+ const instructions = await invokePowerSyncControl(PowerSyncControlCommand.STOP);
9478
+ for (const instruction of instructions) {
9479
+ // We don't need to handle interrupting instructions since we're unconditionally ending the sync iteration at
9480
+ // this point.
9481
+ if (isInterruptingInstruction(instruction))
9482
+ continue;
9483
+ await handleInstruction(instruction);
9484
+ }
9888
9485
  }
9889
- async function control(op, payload) {
9486
+ async function invokePowerSyncControl(op, payload) {
9890
9487
  const rawResponse = await adapter.control(op, payload ?? null);
9891
9488
  const logger = syncImplementation.logger;
9892
9489
  logger.trace('powersync_control', op, payload == null || typeof payload == 'string' ? payload : '<bytes>', rawResponse);
9893
- await handleInstructions(JSON.parse(rawResponse));
9490
+ if (op != PowerSyncControlCommand.STOP) {
9491
+ // Evidently we have a working connection here, otherwise powersync_control would have failed.
9492
+ syncImplementation.connectionMayHaveChanged = false;
9493
+ }
9494
+ return JSON.parse(rawResponse);
9894
9495
  }
9895
9496
  async function handleInstruction(instruction) {
9896
9497
  if ('LogLine' in instruction) {
@@ -9909,13 +9510,6 @@ The next upload iteration will be delayed.`);
9909
9510
  else if ('UpdateSyncStatus' in instruction) {
9910
9511
  syncImplementation.updateSyncStatus(coreStatusToJs(instruction.UpdateSyncStatus.status));
9911
9512
  }
9912
- else if ('EstablishSyncStream' in instruction) {
9913
- if (receivingLines != null) {
9914
- // Already connected, this shouldn't happen during a single iteration.
9915
- throw 'Unexpected request to establish sync stream, already connected';
9916
- }
9917
- receivingLines = connect(instruction.EstablishSyncStream);
9918
- }
9919
9513
  else if ('FetchCredentials' in instruction) {
9920
9514
  if (instruction.FetchCredentials.did_expire) {
9921
9515
  remote.invalidateCredentials();
@@ -9924,16 +9518,12 @@ The next upload iteration will be delayed.`);
9924
9518
  remote.invalidateCredentials();
9925
9519
  // Restart iteration after the credentials have been refreshed.
9926
9520
  remote.fetchCredentials().then((_) => {
9927
- controlInvocations?.enqueueData({ command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED });
9521
+ notifyTokenRefreshed?.();
9928
9522
  }, (err) => {
9929
9523
  syncImplementation.logger.warn('Could not prefetch credentials', err);
9930
9524
  });
9931
9525
  }
9932
9526
  }
9933
- else if ('CloseSyncStream' in instruction) {
9934
- abortController.abort();
9935
- hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
9936
- }
9937
9527
  else if ('FlushFileSystem' in instruction) ;
9938
9528
  else if ('DidCompleteSync' in instruction) {
9939
9529
  syncImplementation.updateSyncStatus({
@@ -9943,105 +9533,83 @@ The next upload iteration will be delayed.`);
9943
9533
  });
9944
9534
  }
9945
9535
  }
9946
- async function handleInstructions(instructions) {
9947
- for (const instr of instructions) {
9948
- await handleInstruction(instr);
9949
- }
9950
- }
9951
9536
  try {
9952
- const options = {
9953
- parameters: resolvedOptions.params,
9954
- app_metadata: resolvedOptions.appMetadata,
9955
- active_streams: this.activeStreams,
9956
- include_defaults: resolvedOptions.includeDefaultStreams
9957
- };
9958
- if (resolvedOptions.serializedSchema) {
9959
- options.schema = resolvedOptions.serializedSchema;
9537
+ const defaultResult = { immediateRestart: false };
9538
+ // Pending sync lines received from the service, as well as local events that trigger a powersync_control
9539
+ // invocation (local events include refreshed tokens and completed uploads).
9540
+ // This is a single data stream so that we can handle all control calls from a single place.
9541
+ let controlInvocations = null;
9542
+ for (const startInstruction of await startCommand()) {
9543
+ if ('EstablishSyncStream' in startInstruction) {
9544
+ const syncOptions = {
9545
+ path: '/sync/stream',
9546
+ abortSignal: signal,
9547
+ data: startInstruction.EstablishSyncStream.request
9548
+ };
9549
+ controlInvocations = injectable(syncImplementation.receiveSyncLines({
9550
+ options: syncOptions,
9551
+ connection: resolvedOptions
9552
+ }));
9553
+ }
9554
+ else if ('CloseSyncStream' in startInstruction) {
9555
+ return defaultResult;
9556
+ }
9557
+ else {
9558
+ await handleInstruction(startInstruction);
9559
+ }
9960
9560
  }
9961
- await control(PowerSyncControlCommand.START, JSON.stringify(options));
9561
+ if (controlInvocations == null)
9562
+ return defaultResult;
9962
9563
  this.notifyCompletedUploads = () => {
9963
- if (controlInvocations && !controlInvocations?.closed) {
9964
- controlInvocations.enqueueData({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
9965
- }
9564
+ controlInvocations.inject({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
9966
9565
  };
9967
9566
  this.handleActiveStreamsChange = () => {
9968
- if (controlInvocations && !controlInvocations?.closed) {
9969
- controlInvocations.enqueueData({
9970
- command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
9971
- payload: JSON.stringify(this.activeStreams)
9972
- });
9973
- }
9567
+ controlInvocations.inject({
9568
+ command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
9569
+ payload: JSON.stringify(this.activeStreams)
9570
+ });
9974
9571
  };
9975
- await receivingLines;
9572
+ notifyTokenRefreshed = () => {
9573
+ controlInvocations.inject({
9574
+ command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED
9575
+ });
9576
+ };
9577
+ let hadSyncLine = false;
9578
+ loop: while (true) {
9579
+ const { done, value } = await controlInvocations.next();
9580
+ if (done)
9581
+ break;
9582
+ if (!hadSyncLine) {
9583
+ // Trigger a local CRUD upload when the first sync line has been received, this allows uploading local changes
9584
+ // that have been made while offline or disconnected.
9585
+ if (value.command == PowerSyncControlCommand.PROCESS_TEXT_LINE ||
9586
+ value.command == PowerSyncControlCommand.PROCESS_BSON_LINE) {
9587
+ hadSyncLine = true;
9588
+ this.triggerCrudUpload?.();
9589
+ }
9590
+ }
9591
+ const instructions = await invokePowerSyncControl(value.command, value.payload);
9592
+ for (const instruction of instructions) {
9593
+ if ('EstablishSyncStream' in instruction) {
9594
+ throw new Error('Received EstablishSyncStream while already connected.');
9595
+ }
9596
+ else if ('CloseSyncStream' in instruction) {
9597
+ hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
9598
+ break loop;
9599
+ }
9600
+ else {
9601
+ await handleInstruction(instruction);
9602
+ }
9603
+ }
9604
+ }
9976
9605
  }
9977
9606
  finally {
9978
9607
  this.notifyCompletedUploads = this.handleActiveStreamsChange = undefined;
9608
+ notifyTokenRefreshed = undefined;
9979
9609
  await stop();
9980
9610
  }
9981
9611
  return { immediateRestart: hideDisconnectOnRestart };
9982
9612
  }
9983
- async updateSyncStatusForStartingCheckpoint(checkpoint) {
9984
- const localProgress = await this.options.adapter.getBucketOperationProgress();
9985
- const progress = {};
9986
- let invalidated = false;
9987
- for (const bucket of checkpoint.buckets) {
9988
- const savedProgress = localProgress[bucket.bucket];
9989
- const atLast = savedProgress?.atLast ?? 0;
9990
- const sinceLast = savedProgress?.sinceLast ?? 0;
9991
- progress[bucket.bucket] = {
9992
- // The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
9993
- // will use by default.
9994
- priority: bucket.priority ?? 3,
9995
- at_last: atLast,
9996
- since_last: sinceLast,
9997
- target_count: bucket.count ?? 0
9998
- };
9999
- if (bucket.count != null && bucket.count < atLast + sinceLast) {
10000
- // Either due to a defrag / sync rule deploy or a compaction operation, the size
10001
- // of the bucket shrank so much that the local ops exceed the ops in the updated
10002
- // bucket. We can't prossibly report progress in this case (it would overshoot 100%).
10003
- invalidated = true;
10004
- }
10005
- }
10006
- if (invalidated) {
10007
- for (const bucket in progress) {
10008
- const bucketProgress = progress[bucket];
10009
- bucketProgress.at_last = 0;
10010
- bucketProgress.since_last = 0;
10011
- }
10012
- }
10013
- this.updateSyncStatus({
10014
- dataFlow: {
10015
- downloading: true,
10016
- downloadProgress: progress
10017
- }
10018
- });
10019
- }
10020
- async applyCheckpoint(checkpoint) {
10021
- let result = await this.options.adapter.syncLocalDatabase(checkpoint);
10022
- if (!result.checkpointValid) {
10023
- this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
10024
- // This means checksums failed. Start again with a new checkpoint.
10025
- // TODO: better back-off
10026
- await new Promise((resolve) => setTimeout(resolve, 50));
10027
- return { applied: false, endIteration: true };
10028
- }
10029
- else if (!result.ready) {
10030
- this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`);
10031
- return { applied: false, endIteration: false };
10032
- }
10033
- this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
10034
- this.updateSyncStatus({
10035
- connected: true,
10036
- lastSyncedAt: new Date(),
10037
- dataFlow: {
10038
- downloading: false,
10039
- downloadProgress: null,
10040
- downloadError: undefined
10041
- }
10042
- });
10043
- return { applied: true, endIteration: false };
10044
- }
10045
9613
  updateSyncStatus(options) {
10046
9614
  const updatedStatus = new SyncStatus({
10047
9615
  connected: options.connected ?? this.syncStatus.connected,
@@ -10987,7 +10555,7 @@ class AbstractPowerSyncDatabase extends BaseObserver {
10987
10555
  * @returns A transaction of CRUD operations to upload, or null if there are none
10988
10556
  */
10989
10557
  async getNextCrudTransaction() {
10990
- const iterator = this.getCrudTransactions()[symbolAsyncIterator]();
10558
+ const iterator = this.getCrudTransactions()[Symbol.asyncIterator]();
10991
10559
  return (await iterator.next()).value;
10992
10560
  }
10993
10561
  /**
@@ -11023,7 +10591,7 @@ class AbstractPowerSyncDatabase extends BaseObserver {
11023
10591
  */
11024
10592
  getCrudTransactions() {
11025
10593
  return {
11026
- [symbolAsyncIterator]: () => {
10594
+ [Symbol.asyncIterator]: () => {
11027
10595
  let lastCrudItemId = -1;
11028
10596
  const sql = `
11029
10597
  WITH RECURSIVE crud_entries AS (
@@ -11580,14 +11148,12 @@ class SqliteBucketStorage extends BaseObserver {
11580
11148
  db;
11581
11149
  logger;
11582
11150
  tableNames;
11583
- _hasCompletedSync;
11584
11151
  updateListener;
11585
11152
  _clientId;
11586
11153
  constructor(db, logger = Logger.get('SqliteBucketStorage')) {
11587
11154
  super();
11588
11155
  this.db = db;
11589
11156
  this.logger = logger;
11590
- this._hasCompletedSync = false;
11591
11157
  this.tableNames = new Set();
11592
11158
  this.updateListener = db.registerListener({
11593
11159
  tablesUpdated: (update) => {
@@ -11599,7 +11165,6 @@ class SqliteBucketStorage extends BaseObserver {
11599
11165
  });
11600
11166
  }
11601
11167
  async init() {
11602
- this._hasCompletedSync = false;
11603
11168
  const existingTableRows = await this.db.getAll(`SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'`);
11604
11169
  for (const row of existingTableRows ?? []) {
11605
11170
  this.tableNames.add(row.name);
@@ -11621,156 +11186,6 @@ class SqliteBucketStorage extends BaseObserver {
11621
11186
  getMaxOpId() {
11622
11187
  return MAX_OP_ID;
11623
11188
  }
11624
- /**
11625
- * Reset any caches.
11626
- */
11627
- startSession() { }
11628
- async getBucketStates() {
11629
- const result = await this.db.getAll("SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != '$local'");
11630
- return result;
11631
- }
11632
- async getBucketOperationProgress() {
11633
- const rows = await this.db.getAll('SELECT name, count_at_last, count_since_last FROM ps_buckets');
11634
- return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
11635
- }
11636
- async saveSyncData(batch, fixedKeyFormat = false) {
11637
- await this.writeTransaction(async (tx) => {
11638
- for (const b of batch.buckets) {
11639
- await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
11640
- 'save',
11641
- JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
11642
- ]);
11643
- this.logger.debug(`Saved batch of data for bucket: ${b.bucket}, operations: ${b.data.length}`);
11644
- }
11645
- });
11646
- }
11647
- async removeBuckets(buckets) {
11648
- for (const bucket of buckets) {
11649
- await this.deleteBucket(bucket);
11650
- }
11651
- }
11652
- /**
11653
- * Mark a bucket for deletion.
11654
- */
11655
- async deleteBucket(bucket) {
11656
- await this.writeTransaction(async (tx) => {
11657
- await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
11658
- });
11659
- this.logger.debug(`Done deleting bucket ${bucket}`);
11660
- }
11661
- async hasCompletedSync() {
11662
- if (this._hasCompletedSync) {
11663
- return true;
11664
- }
11665
- const r = await this.db.get(`SELECT powersync_last_synced_at() as synced_at`);
11666
- const completed = r.synced_at != null;
11667
- if (completed) {
11668
- this._hasCompletedSync = true;
11669
- }
11670
- return completed;
11671
- }
11672
- async syncLocalDatabase(checkpoint, priority) {
11673
- const r = await this.validateChecksums(checkpoint, priority);
11674
- if (!r.checkpointValid) {
11675
- this.logger.error('Checksums failed for', r.checkpointFailures);
11676
- for (const b of r.checkpointFailures ?? []) {
11677
- await this.deleteBucket(b);
11678
- }
11679
- return { ready: false, checkpointValid: false, checkpointFailures: r.checkpointFailures };
11680
- }
11681
- if (priority == null) {
11682
- this.logger.debug(`Validated checksums checkpoint ${checkpoint.last_op_id}`);
11683
- }
11684
- else {
11685
- this.logger.debug(`Validated checksums for partial checkpoint ${checkpoint.last_op_id}, priority ${priority}`);
11686
- }
11687
- let buckets = checkpoint.buckets;
11688
- if (priority !== undefined) {
11689
- buckets = buckets.filter((b) => hasMatchingPriority(priority, b));
11690
- }
11691
- const bucketNames = buckets.map((b) => b.bucket);
11692
- await this.writeTransaction(async (tx) => {
11693
- await tx.execute(`UPDATE ps_buckets SET last_op = ? WHERE name IN (SELECT json_each.value FROM json_each(?))`, [
11694
- checkpoint.last_op_id,
11695
- JSON.stringify(bucketNames)
11696
- ]);
11697
- if (priority == null && checkpoint.write_checkpoint) {
11698
- await tx.execute("UPDATE ps_buckets SET last_op = ? WHERE name = '$local'", [checkpoint.write_checkpoint]);
11699
- }
11700
- });
11701
- const valid = await this.updateObjectsFromBuckets(checkpoint, priority);
11702
- if (!valid) {
11703
- return { ready: false, checkpointValid: true };
11704
- }
11705
- return {
11706
- ready: true,
11707
- checkpointValid: true
11708
- };
11709
- }
11710
- /**
11711
- * Atomically update the local state to the current checkpoint.
11712
- *
11713
- * This includes creating new tables, dropping old tables, and copying data over from the oplog.
11714
- */
11715
- async updateObjectsFromBuckets(checkpoint, priority) {
11716
- let arg = '';
11717
- if (priority !== undefined) {
11718
- const affectedBuckets = [];
11719
- for (const desc of checkpoint.buckets) {
11720
- if (hasMatchingPriority(priority, desc)) {
11721
- affectedBuckets.push(desc.bucket);
11722
- }
11723
- }
11724
- arg = JSON.stringify({ priority, buckets: affectedBuckets });
11725
- }
11726
- return this.writeTransaction(async (tx) => {
11727
- const { insertId: result } = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
11728
- 'sync_local',
11729
- arg
11730
- ]);
11731
- if (result == 1) {
11732
- if (priority == null) {
11733
- const bucketToCount = Object.fromEntries(checkpoint.buckets.map((b) => [b.bucket, b.count]));
11734
- // The two parameters could be replaced with one, but: https://github.com/powersync-ja/better-sqlite3/pull/6
11735
- const jsonBucketCount = JSON.stringify(bucketToCount);
11736
- await tx.execute("UPDATE ps_buckets SET count_since_last = 0, count_at_last = ?->name WHERE name != '$local' AND ?->name IS NOT NULL", [jsonBucketCount, jsonBucketCount]);
11737
- }
11738
- return true;
11739
- }
11740
- else {
11741
- return false;
11742
- }
11743
- });
11744
- }
11745
- async validateChecksums(checkpoint, priority) {
11746
- if (priority !== undefined) {
11747
- // Only validate the buckets within the priority we care about
11748
- const newBuckets = checkpoint.buckets.filter((cs) => hasMatchingPriority(priority, cs));
11749
- checkpoint = { ...checkpoint, buckets: newBuckets };
11750
- }
11751
- const rs = await this.db.execute('SELECT powersync_validate_checkpoint(?) as result', [
11752
- JSON.stringify({ ...checkpoint })
11753
- ]);
11754
- const resultItem = rs.rows?.item(0);
11755
- if (!resultItem) {
11756
- return {
11757
- checkpointValid: false,
11758
- ready: false,
11759
- checkpointFailures: []
11760
- };
11761
- }
11762
- const result = JSON.parse(resultItem['result']);
11763
- if (result['valid']) {
11764
- return { ready: true, checkpointValid: true };
11765
- }
11766
- else {
11767
- return {
11768
- checkpointValid: false,
11769
- ready: false,
11770
- checkpointFailures: result['failed_buckets']
11771
- };
11772
- }
11773
- }
11774
11189
  async updateLocalTarget(cb) {
11775
11190
  const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", [MAX_OP_ID]);
11776
11191
  if (!rs1.length) {
@@ -11861,12 +11276,6 @@ class SqliteBucketStorage extends BaseObserver {
11861
11276
  async writeTransaction(callback, options) {
11862
11277
  return this.db.writeTransaction(callback, options);
11863
11278
  }
11864
- /**
11865
- * Set a target checkpoint.
11866
- */
11867
- async setTargetCheckpoint(checkpoint) {
11868
- // No-op for now
11869
- }
11870
11279
  async control(op, payload) {
11871
11280
  return await this.writeTransaction(async (tx) => {
11872
11281
  const [[raw]] = await tx.executeRaw('SELECT powersync_control(?, ?)', [op, payload]);
@@ -11890,20 +11299,6 @@ class SqliteBucketStorage extends BaseObserver {
11890
11299
  }
11891
11300
  static _subkeyMigrationKey = 'powersync_js_migrated_subkeys';
11892
11301
  }
11893
- function hasMatchingPriority(priority, bucket) {
11894
- return bucket.priority != null && bucket.priority <= priority;
11895
- }
11896
-
11897
- // TODO JSON
11898
- class SyncDataBatch {
11899
- buckets;
11900
- static fromJSON(json) {
11901
- return new SyncDataBatch(json.buckets.map((bucket) => SyncDataBucket.fromRow(bucket)));
11902
- }
11903
- constructor(buckets) {
11904
- this.buckets = buckets;
11905
- }
11906
- }
11907
11302
 
11908
11303
  /**
11909
11304
  * Thrown when an underlying database connection is closed.
@@ -11963,10 +11358,8 @@ class Schema {
11963
11358
  * developer instead of automatically by PowerSync.
11964
11359
  * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
11965
11360
  * using client-side table and column constraints.
11966
- * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
11967
11361
  *
11968
11362
  * @param tables An object of (table name, raw table definition) entries.
11969
- * @experimental Note that the raw tables API is still experimental and may change in the future.
11970
11363
  */
11971
11364
  withRawTables(tables) {
11972
11365
  for (const [name, rawTableDefinition] of Object.entries(tables)) {
@@ -12163,5 +11556,5 @@ const parseQuery = (query, parameters) => {
12163
11556
  return { sqlStatement, parameters: parameters };
12164
11557
  };
12165
11558
 
12166
- export { ATTACHMENT_TABLE, AbortOperation, AbstractPowerSyncDatabase, AbstractPowerSyncDatabaseOpenFactory, AbstractQueryProcessor, AbstractRemote, AbstractStreamingSyncImplementation, ArrayComparator, AttachmentContext, AttachmentQueue, AttachmentService, AttachmentState, AttachmentTable, BaseObserver, Column, ColumnType, ConnectionClosedError, ConnectionManager, ControlledExecutor, CrudBatch, CrudEntry, CrudTransaction, DBAdapterDefaultMixin, DBGetUtilsDefaultMixin, DEFAULT_CRUD_BATCH_LIMIT, DEFAULT_CRUD_UPLOAD_THROTTLE_MS, DEFAULT_INDEX_COLUMN_OPTIONS, DEFAULT_INDEX_OPTIONS, DEFAULT_LOCK_TIMEOUT_MS, DEFAULT_POWERSYNC_CLOSE_OPTIONS, DEFAULT_POWERSYNC_DB_OPTIONS, DEFAULT_PRESSURE_LIMITS, DEFAULT_REMOTE_LOGGER, DEFAULT_REMOTE_OPTIONS, DEFAULT_RETRY_DELAY_MS, DEFAULT_ROW_COMPARATOR, DEFAULT_STREAMING_SYNC_OPTIONS, DEFAULT_STREAM_CONNECTION_OPTIONS, DEFAULT_SYNC_CLIENT_IMPLEMENTATION, DEFAULT_TABLE_OPTIONS, DEFAULT_WATCH_QUERY_OPTIONS, DEFAULT_WATCH_THROTTLE_MS, DataStream, DiffTriggerOperation, DifferentialQueryProcessor, EMPTY_DIFFERENTIAL, EncodingType, FalsyComparator, FetchImplementationProvider, FetchStrategy, GetAllQuery, Index, IndexedColumn, InvalidSQLCharacters, LockType, LogLevel, MAX_AMOUNT_OF_COLUMNS, MAX_OP_ID, MEMORY_TRIGGER_CLAIM_MANAGER, Mutex, OnChangeQueryProcessor, OpType, OpTypeEnum, OplogEntry, PSInternalTable, PowerSyncControlCommand, RowUpdateType, Schema, Semaphore, SqliteBucketStorage, SyncClientImplementation, SyncDataBatch, SyncDataBucket, SyncProgress, SyncStatus, SyncStreamConnectionMethod, SyncingService, Table, TableV2, TriggerManagerImpl, UpdateType, UploadQueueStats, WatchedQueryListenerEvent, attachmentFromSql, column, compilableQueryWatch, createBaseLogger, createLogger, extractTableUpdates, isBatchedUpdateNotification, isContinueCheckpointRequest, isDBAdapter, isPowerSyncDatabaseOptionsWithSettings, isSQLOpenFactory, isSQLOpenOptions, isStreamingKeepalive, isStreamingSyncCheckpoint, isStreamingSyncCheckpointComplete, isStreamingSyncCheckpointDiff, isStreamingSyncCheckpointPartiallyComplete, isStreamingSyncData, isSyncNewCheckpointRequest, parseQuery, runOnSchemaChange, sanitizeSQL, sanitizeUUID, timeoutSignal };
11559
+ export { ATTACHMENT_TABLE, AbortOperation, AbstractPowerSyncDatabase, AbstractPowerSyncDatabaseOpenFactory, AbstractQueryProcessor, AbstractRemote, AbstractStreamingSyncImplementation, ArrayComparator, AttachmentContext, AttachmentQueue, AttachmentService, AttachmentState, AttachmentTable, BaseObserver, Column, ColumnType, ConnectionClosedError, ConnectionManager, ControlledExecutor, CrudBatch, CrudEntry, CrudTransaction, DBAdapterDefaultMixin, DBGetUtilsDefaultMixin, DEFAULT_CRUD_BATCH_LIMIT, DEFAULT_CRUD_UPLOAD_THROTTLE_MS, DEFAULT_INDEX_COLUMN_OPTIONS, DEFAULT_INDEX_OPTIONS, DEFAULT_LOCK_TIMEOUT_MS, DEFAULT_POWERSYNC_CLOSE_OPTIONS, DEFAULT_POWERSYNC_DB_OPTIONS, DEFAULT_REMOTE_LOGGER, DEFAULT_REMOTE_OPTIONS, DEFAULT_RETRY_DELAY_MS, DEFAULT_ROW_COMPARATOR, DEFAULT_STREAMING_SYNC_OPTIONS, DEFAULT_STREAM_CONNECTION_OPTIONS, DEFAULT_SYNC_CLIENT_IMPLEMENTATION, DEFAULT_TABLE_OPTIONS, DEFAULT_WATCH_QUERY_OPTIONS, DEFAULT_WATCH_THROTTLE_MS, DiffTriggerOperation, DifferentialQueryProcessor, EMPTY_DIFFERENTIAL, EncodingType, FalsyComparator, FetchImplementationProvider, FetchStrategy, GetAllQuery, Index, IndexedColumn, InvalidSQLCharacters, LockType, LogLevel, MAX_AMOUNT_OF_COLUMNS, MAX_OP_ID, MEMORY_TRIGGER_CLAIM_MANAGER, Mutex, OnChangeQueryProcessor, PSInternalTable, PowerSyncControlCommand, RowUpdateType, Schema, Semaphore, SqliteBucketStorage, SyncClientImplementation, SyncProgress, SyncStatus, SyncStreamConnectionMethod, SyncingService, Table, TableV2, TriggerManagerImpl, UpdateType, UploadQueueStats, WatchedQueryListenerEvent, attachmentFromSql, column, compilableQueryWatch, createBaseLogger, createLogger, extractTableUpdates, isBatchedUpdateNotification, isDBAdapter, isPowerSyncDatabaseOptionsWithSettings, isSQLOpenFactory, isSQLOpenOptions, parseQuery, runOnSchemaChange, sanitizeSQL, sanitizeUUID, timeoutSignal };
12167
11560
  //# sourceMappingURL=bundle.node.mjs.map