@powersync/common 1.51.0 → 1.53.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/dist/bundle.cjs +510 -1129
  2. package/dist/bundle.cjs.map +1 -1
  3. package/dist/bundle.mjs +511 -1116
  4. package/dist/bundle.mjs.map +1 -1
  5. package/dist/bundle.node.cjs +508 -1129
  6. package/dist/bundle.node.cjs.map +1 -1
  7. package/dist/bundle.node.mjs +509 -1116
  8. package/dist/bundle.node.mjs.map +1 -1
  9. package/dist/index.d.cts +73 -433
  10. package/legacy/sync_protocol.d.ts +103 -0
  11. package/lib/client/AbstractPowerSyncDatabase.js +3 -3
  12. package/lib/client/AbstractPowerSyncDatabase.js.map +1 -1
  13. package/lib/client/ConnectionManager.js +1 -1
  14. package/lib/client/ConnectionManager.js.map +1 -1
  15. package/lib/client/sync/bucket/BucketStorageAdapter.d.ts +6 -64
  16. package/lib/client/sync/bucket/BucketStorageAdapter.js +4 -0
  17. package/lib/client/sync/bucket/BucketStorageAdapter.js.map +1 -1
  18. package/lib/client/sync/bucket/SqliteBucketStorage.d.ts +1 -28
  19. package/lib/client/sync/bucket/SqliteBucketStorage.js +0 -162
  20. package/lib/client/sync/bucket/SqliteBucketStorage.js.map +1 -1
  21. package/lib/client/sync/stream/AbstractRemote.d.ts +29 -18
  22. package/lib/client/sync/stream/AbstractRemote.js +155 -188
  23. package/lib/client/sync/stream/AbstractRemote.js.map +1 -1
  24. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +13 -35
  25. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +150 -448
  26. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js.map +1 -1
  27. package/lib/client/sync/stream/JsonValue.d.ts +7 -0
  28. package/lib/client/sync/stream/JsonValue.js +2 -0
  29. package/lib/client/sync/stream/JsonValue.js.map +1 -0
  30. package/lib/client/sync/stream/core-instruction.d.ts +14 -9
  31. package/lib/client/sync/stream/core-instruction.js +3 -0
  32. package/lib/client/sync/stream/core-instruction.js.map +1 -1
  33. package/lib/db/DBAdapter.d.ts +9 -0
  34. package/lib/db/DBAdapter.js +8 -1
  35. package/lib/db/DBAdapter.js.map +1 -1
  36. package/lib/db/crud/SyncStatus.d.ts +3 -4
  37. package/lib/db/crud/SyncStatus.js +0 -4
  38. package/lib/db/crud/SyncStatus.js.map +1 -1
  39. package/lib/db/schema/RawTable.d.ts +0 -5
  40. package/lib/db/schema/Schema.d.ts +0 -2
  41. package/lib/db/schema/Schema.js +0 -2
  42. package/lib/db/schema/Schema.js.map +1 -1
  43. package/lib/index.d.ts +2 -6
  44. package/lib/index.js +1 -6
  45. package/lib/index.js.map +1 -1
  46. package/lib/utils/async.d.ts +0 -9
  47. package/lib/utils/async.js +0 -9
  48. package/lib/utils/async.js.map +1 -1
  49. package/lib/utils/stream_transform.d.ts +39 -0
  50. package/lib/utils/stream_transform.js +206 -0
  51. package/lib/utils/stream_transform.js.map +1 -0
  52. package/package.json +15 -10
  53. package/src/client/AbstractPowerSyncDatabase.ts +3 -3
  54. package/src/client/ConnectionManager.ts +1 -1
  55. package/src/client/sync/bucket/BucketStorageAdapter.ts +6 -71
  56. package/src/client/sync/bucket/SqliteBucketStorage.ts +1 -197
  57. package/src/client/sync/stream/AbstractRemote.ts +183 -229
  58. package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +181 -510
  59. package/src/client/sync/stream/JsonValue.ts +8 -0
  60. package/src/client/sync/stream/core-instruction.ts +15 -5
  61. package/src/db/DBAdapter.ts +20 -2
  62. package/src/db/crud/SyncStatus.ts +4 -5
  63. package/src/db/schema/RawTable.ts +0 -5
  64. package/src/db/schema/Schema.ts +0 -2
  65. package/src/index.ts +2 -6
  66. package/src/utils/async.ts +0 -11
  67. package/src/utils/stream_transform.ts +252 -0
  68. package/lib/client/sync/bucket/OpType.d.ts +0 -16
  69. package/lib/client/sync/bucket/OpType.js +0 -23
  70. package/lib/client/sync/bucket/OpType.js.map +0 -1
  71. package/lib/client/sync/bucket/OplogEntry.d.ts +0 -23
  72. package/lib/client/sync/bucket/OplogEntry.js +0 -36
  73. package/lib/client/sync/bucket/OplogEntry.js.map +0 -1
  74. package/lib/client/sync/bucket/SyncDataBatch.d.ts +0 -6
  75. package/lib/client/sync/bucket/SyncDataBatch.js +0 -12
  76. package/lib/client/sync/bucket/SyncDataBatch.js.map +0 -1
  77. package/lib/client/sync/bucket/SyncDataBucket.d.ts +0 -40
  78. package/lib/client/sync/bucket/SyncDataBucket.js +0 -40
  79. package/lib/client/sync/bucket/SyncDataBucket.js.map +0 -1
  80. package/lib/client/sync/stream/streaming-sync-types.d.ts +0 -143
  81. package/lib/client/sync/stream/streaming-sync-types.js +0 -26
  82. package/lib/client/sync/stream/streaming-sync-types.js.map +0 -1
  83. package/lib/utils/DataStream.d.ts +0 -62
  84. package/lib/utils/DataStream.js +0 -169
  85. package/lib/utils/DataStream.js.map +0 -1
  86. package/src/client/sync/bucket/OpType.ts +0 -23
  87. package/src/client/sync/bucket/OplogEntry.ts +0 -50
  88. package/src/client/sync/bucket/SyncDataBatch.ts +0 -11
  89. package/src/client/sync/bucket/SyncDataBucket.ts +0 -49
  90. package/src/client/sync/stream/streaming-sync-types.ts +0 -210
  91. package/src/utils/DataStream.ts +0 -222
@@ -1819,8 +1819,15 @@ class BaseTransaction {
1819
1819
  class TransactionImplementation extends DBGetUtilsDefaultMixin(BaseTransaction) {
1820
1820
  static async runWith(ctx, fn) {
1821
1821
  let tx = new TransactionImplementation(ctx);
1822
+ // For write transactions, use BEGIN IMMEDIATE to immediately obtain a write lock on the database (instead of doing
1823
+ // that on the first statement). If we have a genuine read-only connection, we also use BEGIN IMMEDIATE there: In
1824
+ // WAL mode, that ensures we pin the current state of the database (instead of the state at the first statement in
1825
+ // the transaction). But if we have a "fake" read-only connection implemented through `pragma query_only = true`, we
1826
+ // can't use this trick because it would attempt to lock the connection. So there, we use a regular `BEGIN`
1827
+ // statement.
1828
+ const useBeginImmediate = ctx.connectionType != 'queryOnly';
1822
1829
  try {
1823
- await ctx.execute('BEGIN IMMEDIATE');
1830
+ await ctx.execute(useBeginImmediate ? 'BEGIN IMMEDIATE' : 'BEGIN');
1824
1831
  const result = await fn(tx);
1825
1832
  await tx.commit();
1826
1833
  return result;
@@ -1979,16 +1986,12 @@ class SyncStatus {
1979
1986
  *
1980
1987
  * This returns null when the database is currently being opened and we don't have reliable information about all
1981
1988
  * included streams yet.
1982
- *
1983
- * @experimental Sync streams are currently in alpha.
1984
1989
  */
1985
1990
  get syncStreams() {
1986
1991
  return this.options.dataFlow?.internalStreamSubscriptions?.map((core) => new SyncStreamStatusView(this, core));
1987
1992
  }
1988
1993
  /**
1989
1994
  * If the `stream` appears in {@link syncStreams}, returns the current status for that stream.
1990
- *
1991
- * @experimental Sync streams are currently in alpha.
1992
1995
  */
1993
1996
  forStream(stream) {
1994
1997
  const asJson = JSON.stringify(stream.parameters);
@@ -2257,15 +2260,6 @@ class ControlledExecutor {
2257
2260
  }
2258
2261
  }
2259
2262
 
2260
- /**
2261
- * A ponyfill for `Symbol.asyncIterator` that is compatible with the
2262
- * [recommended polyfill](https://github.com/Azure/azure-sdk-for-js/blob/%40azure/core-asynciterator-polyfill_1.0.2/sdk/core/core-asynciterator-polyfill/src/index.ts#L4-L6)
2263
- * we recommend for React Native.
2264
- *
2265
- * As long as we use this symbol (instead of `for await` and `async *`) in this package, we can be compatible with async
2266
- * iterators without requiring them.
2267
- */
2268
- const symbolAsyncIterator = Symbol.asyncIterator ?? Symbol.for('Symbol.asyncIterator');
2269
2263
  /**
2270
2264
  * Throttle a function to be called at most once every "wait" milliseconds,
2271
2265
  * on the trailing edge.
@@ -2580,7 +2574,7 @@ class SyncStreamSubscriptionHandle {
2580
2574
  constructor(subscription) {
2581
2575
  this.subscription = subscription;
2582
2576
  subscription.refcount++;
2583
- _finalizer?.register(this, subscription);
2577
+ _finalizer?.register(this, subscription, this);
2584
2578
  }
2585
2579
  get name() {
2586
2580
  return this.subscription.name;
@@ -3169,6 +3163,10 @@ exports.PowerSyncControlCommand = void 0;
3169
3163
  PowerSyncControlCommand["NOTIFY_TOKEN_REFRESHED"] = "refreshed_token";
3170
3164
  PowerSyncControlCommand["NOTIFY_CRUD_UPLOAD_COMPLETED"] = "completed_upload";
3171
3165
  PowerSyncControlCommand["UPDATE_SUBSCRIPTIONS"] = "update_subscriptions";
3166
+ /**
3167
+ * An `established` or `end` event for response streams.
3168
+ */
3169
+ PowerSyncControlCommand["CONNECTION_STATE"] = "connection";
3172
3170
  })(exports.PowerSyncControlCommand || (exports.PowerSyncControlCommand = {}));
3173
3171
 
3174
3172
  /**
@@ -3350,103 +3348,6 @@ class AbortOperation extends Error {
3350
3348
  }
3351
3349
  }
3352
3350
 
3353
- exports.OpTypeEnum = void 0;
3354
- (function (OpTypeEnum) {
3355
- OpTypeEnum[OpTypeEnum["CLEAR"] = 1] = "CLEAR";
3356
- OpTypeEnum[OpTypeEnum["MOVE"] = 2] = "MOVE";
3357
- OpTypeEnum[OpTypeEnum["PUT"] = 3] = "PUT";
3358
- OpTypeEnum[OpTypeEnum["REMOVE"] = 4] = "REMOVE";
3359
- })(exports.OpTypeEnum || (exports.OpTypeEnum = {}));
3360
- /**
3361
- * Used internally for sync buckets.
3362
- */
3363
- class OpType {
3364
- value;
3365
- static fromJSON(jsonValue) {
3366
- return new OpType(exports.OpTypeEnum[jsonValue]);
3367
- }
3368
- constructor(value) {
3369
- this.value = value;
3370
- }
3371
- toJSON() {
3372
- return Object.entries(exports.OpTypeEnum).find(([, value]) => value === this.value)[0];
3373
- }
3374
- }
3375
-
3376
- class OplogEntry {
3377
- op_id;
3378
- op;
3379
- checksum;
3380
- subkey;
3381
- object_type;
3382
- object_id;
3383
- data;
3384
- static fromRow(row) {
3385
- return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, row.subkey, row.object_type, row.object_id, row.data);
3386
- }
3387
- constructor(op_id, op, checksum, subkey, object_type, object_id, data) {
3388
- this.op_id = op_id;
3389
- this.op = op;
3390
- this.checksum = checksum;
3391
- this.subkey = subkey;
3392
- this.object_type = object_type;
3393
- this.object_id = object_id;
3394
- this.data = data;
3395
- }
3396
- toJSON(fixedKeyEncoding = false) {
3397
- return {
3398
- op_id: this.op_id,
3399
- op: this.op.toJSON(),
3400
- object_type: this.object_type,
3401
- object_id: this.object_id,
3402
- checksum: this.checksum,
3403
- data: this.data,
3404
- // Older versions of the JS SDK used to always JSON.stringify here. That has always been wrong,
3405
- // but we need to migrate gradually to not break existing databases.
3406
- subkey: fixedKeyEncoding ? this.subkey : JSON.stringify(this.subkey)
3407
- };
3408
- }
3409
- }
3410
-
3411
- class SyncDataBucket {
3412
- bucket;
3413
- data;
3414
- has_more;
3415
- after;
3416
- next_after;
3417
- static fromRow(row) {
3418
- return new SyncDataBucket(row.bucket, row.data.map((entry) => OplogEntry.fromRow(entry)), row.has_more ?? false, row.after, row.next_after);
3419
- }
3420
- constructor(bucket, data,
3421
- /**
3422
- * True if the response does not contain all the data for this bucket, and another request must be made.
3423
- */
3424
- has_more,
3425
- /**
3426
- * The `after` specified in the request.
3427
- */
3428
- after,
3429
- /**
3430
- * Use this for the next request.
3431
- */
3432
- next_after) {
3433
- this.bucket = bucket;
3434
- this.data = data;
3435
- this.has_more = has_more;
3436
- this.after = after;
3437
- this.next_after = next_after;
3438
- }
3439
- toJSON(fixedKeyEncoding = false) {
3440
- return {
3441
- bucket: this.bucket,
3442
- has_more: this.has_more,
3443
- after: this.after,
3444
- next_after: this.next_after,
3445
- data: this.data.map((entry) => entry.toJSON(fixedKeyEncoding))
3446
- };
3447
- }
3448
- }
3449
-
3450
3351
  var dist = {};
3451
3352
 
3452
3353
  var Codecs = {};
@@ -8231,177 +8132,10 @@ function requireDist () {
8231
8132
 
8232
8133
  var distExports = requireDist();
8233
8134
 
8234
- var version = "1.51.0";
8135
+ var version = "1.53.0";
8235
8136
  var PACKAGE = {
8236
8137
  version: version};
8237
8138
 
8238
- const DEFAULT_PRESSURE_LIMITS = {
8239
- highWater: 10,
8240
- lowWater: 0
8241
- };
8242
- /**
8243
- * A very basic implementation of a data stream with backpressure support which does not use
8244
- * native JS streams or async iterators.
8245
- * This is handy for environments such as React Native which need polyfills for the above.
8246
- */
8247
- class DataStream extends BaseObserver {
8248
- options;
8249
- dataQueue;
8250
- isClosed;
8251
- processingPromise;
8252
- notifyDataAdded;
8253
- logger;
8254
- mapLine;
8255
- constructor(options) {
8256
- super();
8257
- this.options = options;
8258
- this.processingPromise = null;
8259
- this.isClosed = false;
8260
- this.dataQueue = [];
8261
- this.mapLine = options?.mapLine ?? ((line) => line);
8262
- this.logger = options?.logger ?? Logger.get('DataStream');
8263
- if (options?.closeOnError) {
8264
- const l = this.registerListener({
8265
- error: (ex) => {
8266
- l?.();
8267
- this.close();
8268
- }
8269
- });
8270
- }
8271
- }
8272
- get highWatermark() {
8273
- return this.options?.pressure?.highWaterMark ?? DEFAULT_PRESSURE_LIMITS.highWater;
8274
- }
8275
- get lowWatermark() {
8276
- return this.options?.pressure?.lowWaterMark ?? DEFAULT_PRESSURE_LIMITS.lowWater;
8277
- }
8278
- get closed() {
8279
- return this.isClosed;
8280
- }
8281
- async close() {
8282
- this.isClosed = true;
8283
- await this.processingPromise;
8284
- this.iterateListeners((l) => l.closed?.());
8285
- // Discard any data in the queue
8286
- this.dataQueue = [];
8287
- this.listeners.clear();
8288
- }
8289
- /**
8290
- * Enqueues data for the consumers to read
8291
- */
8292
- enqueueData(data) {
8293
- if (this.isClosed) {
8294
- throw new Error('Cannot enqueue data into closed stream.');
8295
- }
8296
- this.dataQueue.push(data);
8297
- this.notifyDataAdded?.();
8298
- this.processQueue();
8299
- }
8300
- /**
8301
- * Reads data once from the data stream
8302
- * @returns a Data payload or Null if the stream closed.
8303
- */
8304
- async read() {
8305
- if (this.closed) {
8306
- return null;
8307
- }
8308
- // Wait for any pending processing to complete first.
8309
- // This ensures we register our listener before calling processQueue(),
8310
- // avoiding a race where processQueue() sees no reader and returns early.
8311
- if (this.processingPromise) {
8312
- await this.processingPromise;
8313
- }
8314
- // Re-check after await - stream may have closed while we were waiting
8315
- if (this.closed) {
8316
- return null;
8317
- }
8318
- return new Promise((resolve, reject) => {
8319
- const l = this.registerListener({
8320
- data: async (data) => {
8321
- resolve(data);
8322
- // Remove the listener
8323
- l?.();
8324
- },
8325
- closed: () => {
8326
- resolve(null);
8327
- l?.();
8328
- },
8329
- error: (ex) => {
8330
- reject(ex);
8331
- l?.();
8332
- }
8333
- });
8334
- this.processQueue();
8335
- });
8336
- }
8337
- /**
8338
- * Executes a callback for each data item in the stream
8339
- */
8340
- forEach(callback) {
8341
- if (this.dataQueue.length <= this.lowWatermark) {
8342
- this.iterateAsyncErrored(async (l) => l.lowWater?.());
8343
- }
8344
- return this.registerListener({
8345
- data: callback
8346
- });
8347
- }
8348
- processQueue() {
8349
- if (this.processingPromise) {
8350
- return;
8351
- }
8352
- const promise = (this.processingPromise = this._processQueue());
8353
- promise.finally(() => {
8354
- this.processingPromise = null;
8355
- });
8356
- return promise;
8357
- }
8358
- hasDataReader() {
8359
- return Array.from(this.listeners.values()).some((l) => !!l.data);
8360
- }
8361
- async _processQueue() {
8362
- /**
8363
- * Allow listeners to mutate the queue before processing.
8364
- * This allows for operations such as dropping or compressing data
8365
- * on high water or requesting more data on low water.
8366
- */
8367
- if (this.dataQueue.length >= this.highWatermark) {
8368
- await this.iterateAsyncErrored(async (l) => l.highWater?.());
8369
- }
8370
- if (this.isClosed || !this.hasDataReader()) {
8371
- return;
8372
- }
8373
- if (this.dataQueue.length) {
8374
- const data = this.dataQueue.shift();
8375
- const mapped = this.mapLine(data);
8376
- await this.iterateAsyncErrored(async (l) => l.data?.(mapped));
8377
- }
8378
- if (this.dataQueue.length <= this.lowWatermark) {
8379
- const dataAdded = new Promise((resolve) => {
8380
- this.notifyDataAdded = resolve;
8381
- });
8382
- await Promise.race([this.iterateAsyncErrored(async (l) => l.lowWater?.()), dataAdded]);
8383
- this.notifyDataAdded = null;
8384
- }
8385
- if (this.dataQueue.length > 0) {
8386
- setTimeout(() => this.processQueue());
8387
- }
8388
- }
8389
- async iterateAsyncErrored(cb) {
8390
- // Important: We need to copy the listeners, as calling a listener could result in adding another
8391
- // listener, resulting in infinite loops.
8392
- const listeners = Array.from(this.listeners.values());
8393
- for (let i of listeners) {
8394
- try {
8395
- await cb(i);
8396
- }
8397
- catch (ex) {
8398
- this.logger.error(ex);
8399
- this.iterateListeners((l) => l.error?.(ex));
8400
- }
8401
- }
8402
- }
8403
- }
8404
-
8405
8139
  var WebsocketDuplexConnection = {};
8406
8140
 
8407
8141
  var hasRequiredWebsocketDuplexConnection;
@@ -8564,8 +8298,199 @@ class WebsocketClientTransport {
8564
8298
  }
8565
8299
  }
8566
8300
 
8301
+ const doneResult = { done: true, value: undefined };
8302
+ function valueResult(value) {
8303
+ return { done: false, value };
8304
+ }
8305
+ /**
8306
+ * Expands a source async iterator by allowing to inject events asynchronously.
8307
+ *
8308
+ * The resulting iterator will emit all events from its source. Additionally though, events can be injected. These
8309
+ * events are dropped once the main iterator completes, but are otherwise forwarded.
8310
+ *
8311
+ * The iterator completes when its source completes, and it supports backpressure by only calling `next()` on the source
8312
+ * in response to a `next()` call from downstream if no pending injected events can be dispatched.
8313
+ */
8314
+ function injectable(source) {
8315
+ let sourceIsDone = false;
8316
+ let waiter = undefined; // An active, waiting next() call.
8317
+ // A pending upstream event that couldn't be dispatched because inject() has been called before it was resolved.
8318
+ let pendingSourceEvent = null;
8319
+ let pendingInjectedEvents = [];
8320
+ const consumeWaiter = () => {
8321
+ const pending = waiter;
8322
+ waiter = undefined;
8323
+ return pending;
8324
+ };
8325
+ const fetchFromSource = () => {
8326
+ const resolveWaiter = (propagate) => {
8327
+ const active = consumeWaiter();
8328
+ if (active) {
8329
+ propagate(active);
8330
+ }
8331
+ else {
8332
+ pendingSourceEvent = propagate;
8333
+ }
8334
+ };
8335
+ const nextFromSource = source.next();
8336
+ nextFromSource.then((value) => {
8337
+ sourceIsDone = value.done == true;
8338
+ resolveWaiter((w) => w.resolve(value));
8339
+ }, (error) => {
8340
+ resolveWaiter((w) => w.reject(error));
8341
+ });
8342
+ };
8343
+ return {
8344
+ next: () => {
8345
+ return new Promise((resolve, reject) => {
8346
+ // First priority: Dispatch ready upstream events.
8347
+ if (sourceIsDone) {
8348
+ return resolve(doneResult);
8349
+ }
8350
+ if (pendingSourceEvent) {
8351
+ pendingSourceEvent({ resolve, reject });
8352
+ pendingSourceEvent = null;
8353
+ return;
8354
+ }
8355
+ // Second priority: Dispatch injected events
8356
+ if (pendingInjectedEvents.length) {
8357
+ return resolve(valueResult(pendingInjectedEvents.shift()));
8358
+ }
8359
+ // Nothing pending? Fetch from source
8360
+ waiter = { resolve, reject };
8361
+ return fetchFromSource();
8362
+ });
8363
+ },
8364
+ inject: (event) => {
8365
+ const pending = consumeWaiter();
8366
+ if (pending != null) {
8367
+ pending.resolve(valueResult(event));
8368
+ }
8369
+ else {
8370
+ pendingInjectedEvents.push(event);
8371
+ }
8372
+ }
8373
+ };
8374
+ }
8375
+ /**
8376
+ * Splits a byte stream at line endings, emitting each line as a string.
8377
+ */
8378
+ function extractJsonLines(source, decoder) {
8379
+ let buffer = '';
8380
+ const pendingLines = [];
8381
+ let isFinalEvent = false;
8382
+ return {
8383
+ next: async () => {
8384
+ while (true) {
8385
+ if (isFinalEvent) {
8386
+ return doneResult;
8387
+ }
8388
+ {
8389
+ const first = pendingLines.shift();
8390
+ if (first) {
8391
+ return { done: false, value: first };
8392
+ }
8393
+ }
8394
+ const { done, value } = await source.next();
8395
+ if (done) {
8396
+ const remaining = buffer.trim();
8397
+ if (remaining.length != 0) {
8398
+ isFinalEvent = true;
8399
+ return { done: false, value: remaining };
8400
+ }
8401
+ return doneResult;
8402
+ }
8403
+ const data = decoder.decode(value, { stream: true });
8404
+ buffer += data;
8405
+ const lines = buffer.split('\n');
8406
+ for (let i = 0; i < lines.length - 1; i++) {
8407
+ const l = lines[i].trim();
8408
+ if (l.length > 0) {
8409
+ pendingLines.push(l);
8410
+ }
8411
+ }
8412
+ buffer = lines[lines.length - 1];
8413
+ }
8414
+ }
8415
+ };
8416
+ }
8417
+ /**
8418
+ * Splits a concatenated stream of BSON objects by emitting individual objects.
8419
+ */
8420
+ function extractBsonObjects(source) {
8421
+ // Fully read but not emitted yet.
8422
+ const completedObjects = [];
8423
+ // Whether source has returned { done: true }. We do the same once completed objects have been emitted.
8424
+ let isDone = false;
8425
+ const lengthBuffer = new DataView(new ArrayBuffer(4));
8426
+ let objectBody = null;
8427
+ // If we're parsing the length field, a number between 1 and 4 (inclusive) describing remaining bytes in the header.
8428
+ // If we're consuming a document, the bytes remaining.
8429
+ let remainingLength = 4;
8430
+ return {
8431
+ async next() {
8432
+ while (true) {
8433
+ // Before fetching new data from upstream, return completed objects.
8434
+ if (completedObjects.length) {
8435
+ return valueResult(completedObjects.shift());
8436
+ }
8437
+ if (isDone) {
8438
+ return doneResult;
8439
+ }
8440
+ const upstreamEvent = await source.next();
8441
+ if (upstreamEvent.done) {
8442
+ isDone = true;
8443
+ if (objectBody || remainingLength != 4) {
8444
+ throw new Error('illegal end of stream in BSON object');
8445
+ }
8446
+ return doneResult;
8447
+ }
8448
+ const chunk = upstreamEvent.value;
8449
+ for (let i = 0; i < chunk.length;) {
8450
+ const availableInData = chunk.length - i;
8451
+ if (objectBody) {
8452
+ // We're in the middle of reading a BSON document.
8453
+ const bytesToRead = Math.min(availableInData, remainingLength);
8454
+ const copySource = new Uint8Array(chunk.buffer, chunk.byteOffset + i, bytesToRead);
8455
+ objectBody.set(copySource, objectBody.length - remainingLength);
8456
+ i += bytesToRead;
8457
+ remainingLength -= bytesToRead;
8458
+ if (remainingLength == 0) {
8459
+ completedObjects.push(objectBody);
8460
+ // Prepare to read another document, starting with its length
8461
+ objectBody = null;
8462
+ remainingLength = 4;
8463
+ }
8464
+ }
8465
+ else {
8466
+ // Copy up to 4 bytes into lengthBuffer, depending on how many we still need.
8467
+ const bytesToRead = Math.min(availableInData, remainingLength);
8468
+ for (let j = 0; j < bytesToRead; j++) {
8469
+ lengthBuffer.setUint8(4 - remainingLength + j, chunk[i + j]);
8470
+ }
8471
+ i += bytesToRead;
8472
+ remainingLength -= bytesToRead;
8473
+ if (remainingLength == 0) {
8474
+ // Transition from reading length header to reading document. Subtracting 4 because the length of the
8475
+ // header is included in length.
8476
+ const length = lengthBuffer.getInt32(0, true /* little endian */);
8477
+ remainingLength = length - 4;
8478
+ if (remainingLength < 1) {
8479
+ throw new Error(`invalid length for bson: ${length}`);
8480
+ }
8481
+ objectBody = new Uint8Array(length);
8482
+ new DataView(objectBody.buffer).setInt32(0, length, true);
8483
+ }
8484
+ }
8485
+ }
8486
+ }
8487
+ }
8488
+ };
8489
+ }
8490
+
8567
8491
  const POWERSYNC_TRAILING_SLASH_MATCH = /\/+$/;
8568
8492
  const POWERSYNC_JS_VERSION = PACKAGE.version;
8493
+ const SYNC_QUEUE_REQUEST_HIGH_WATER = 10;
8569
8494
  const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
8570
8495
  // Keep alive message is sent every period
8571
8496
  const KEEP_ALIVE_MS = 20_000;
@@ -8745,73 +8670,67 @@ class AbstractRemote {
8745
8670
  return new WebSocket(url);
8746
8671
  }
8747
8672
  /**
8748
- * Returns a data stream of sync line data.
8673
+ * Returns a data stream of sync line data, fetched via RSocket-over-WebSocket.
8749
8674
  *
8750
- * @param map Maps received payload frames to the typed event value.
8751
- * @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
8752
- * (required for compatibility with older sync services).
8675
+ * The only mechanism to abort the returned stream is to use the abort signal in {@link SocketSyncStreamOptions}.
8753
8676
  */
8754
- async socketStreamRaw(options, map, bson) {
8677
+ async socketStreamRaw(options) {
8755
8678
  const { path, fetchStrategy = exports.FetchStrategy.Buffered } = options;
8756
- const mimeType = bson == null ? 'application/json' : 'application/bson';
8679
+ const mimeType = 'application/json';
8757
8680
  function toBuffer(js) {
8758
- let contents;
8759
- if (bson != null) {
8760
- contents = bson.serialize(js);
8761
- }
8762
- else {
8763
- contents = JSON.stringify(js);
8764
- }
8765
- return node_buffer.Buffer.from(contents);
8681
+ return node_buffer.Buffer.from(JSON.stringify(js));
8766
8682
  }
8767
8683
  const syncQueueRequestSize = fetchStrategy == exports.FetchStrategy.Buffered ? 10 : 1;
8768
8684
  const request = await this.buildRequest(path);
8685
+ const url = this.options.socketUrlTransformer(request.url);
8769
8686
  // Add the user agent in the setup payload - we can't set custom
8770
8687
  // headers with websockets on web. The browser userAgent is however added
8771
8688
  // automatically as a header.
8772
8689
  const userAgent = this.getUserAgent();
8773
- const stream = new DataStream({
8774
- logger: this.logger,
8775
- pressure: {
8776
- lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER
8777
- },
8778
- mapLine: map
8779
- });
8690
+ // While we're connecting (a process that can't be aborted in RSocket), the WebSocket instance to close if we wanted
8691
+ // to abort the connection.
8692
+ let pendingSocket = null;
8693
+ let keepAliveTimeout;
8694
+ let rsocket = null;
8695
+ let queue = null;
8696
+ let didClose = false;
8697
+ const abortRequest = () => {
8698
+ if (didClose) {
8699
+ return;
8700
+ }
8701
+ didClose = true;
8702
+ clearTimeout(keepAliveTimeout);
8703
+ if (pendingSocket) {
8704
+ pendingSocket.close();
8705
+ }
8706
+ if (rsocket) {
8707
+ rsocket.close();
8708
+ }
8709
+ if (queue) {
8710
+ queue.stop();
8711
+ }
8712
+ };
8780
8713
  // Handle upstream abort
8781
- if (options.abortSignal?.aborted) {
8714
+ if (options.abortSignal.aborted) {
8782
8715
  throw new AbortOperation('Connection request aborted');
8783
8716
  }
8784
8717
  else {
8785
- options.abortSignal?.addEventListener('abort', () => {
8786
- stream.close();
8787
- }, { once: true });
8718
+ options.abortSignal.addEventListener('abort', abortRequest);
8788
8719
  }
8789
- let keepAliveTimeout;
8790
8720
  const resetTimeout = () => {
8791
8721
  clearTimeout(keepAliveTimeout);
8792
8722
  keepAliveTimeout = setTimeout(() => {
8793
8723
  this.logger.error(`No data received on WebSocket in ${SOCKET_TIMEOUT_MS}ms, closing connection.`);
8794
- stream.close();
8724
+ abortRequest();
8795
8725
  }, SOCKET_TIMEOUT_MS);
8796
8726
  };
8797
8727
  resetTimeout();
8798
- // Typescript complains about this being `never` if it's not assigned here.
8799
- // This is assigned in `wsCreator`.
8800
- let disposeSocketConnectionTimeout = () => { };
8801
- const url = this.options.socketUrlTransformer(request.url);
8802
8728
  const connector = new distExports.RSocketConnector({
8803
8729
  transport: new WebsocketClientTransport({
8804
8730
  url,
8805
8731
  wsCreator: (url) => {
8806
- const socket = this.createSocket(url);
8807
- disposeSocketConnectionTimeout = stream.registerListener({
8808
- closed: () => {
8809
- // Allow closing the underlying WebSocket if the stream was closed before the
8810
- // RSocket connect completed. This should effectively abort the request.
8811
- socket.close();
8812
- }
8813
- });
8814
- socket.addEventListener('message', (event) => {
8732
+ const socket = (pendingSocket = this.createSocket(url));
8733
+ socket.addEventListener('message', () => {
8815
8734
  resetTimeout();
8816
8735
  });
8817
8736
  return socket;
@@ -8831,43 +8750,40 @@ class AbstractRemote {
8831
8750
  }
8832
8751
  }
8833
8752
  });
8834
- let rsocket;
8835
8753
  try {
8836
8754
  rsocket = await connector.connect();
8837
8755
  // The connection is established, we no longer need to monitor the initial timeout
8838
- disposeSocketConnectionTimeout();
8756
+ pendingSocket = null;
8839
8757
  }
8840
8758
  catch (ex) {
8841
8759
  this.logger.error(`Failed to connect WebSocket`, ex);
8842
- clearTimeout(keepAliveTimeout);
8843
- if (!stream.closed) {
8844
- await stream.close();
8845
- }
8760
+ abortRequest();
8846
8761
  throw ex;
8847
8762
  }
8848
8763
  resetTimeout();
8849
- let socketIsClosed = false;
8850
- const closeSocket = () => {
8851
- clearTimeout(keepAliveTimeout);
8852
- if (socketIsClosed) {
8853
- return;
8854
- }
8855
- socketIsClosed = true;
8856
- rsocket.close();
8857
- };
8858
8764
  // Helps to prevent double close scenarios
8859
- rsocket.onClose(() => (socketIsClosed = true));
8860
- // We initially request this amount and expect these to arrive eventually
8861
- let pendingEventsCount = syncQueueRequestSize;
8862
- const disposeClosedListener = stream.registerListener({
8863
- closed: () => {
8864
- closeSocket();
8865
- disposeClosedListener();
8866
- }
8867
- });
8868
- const socket = await new Promise((resolve, reject) => {
8765
+ rsocket.onClose(() => (rsocket = null));
8766
+ return await new Promise((resolve, reject) => {
8869
8767
  let connectionEstablished = false;
8870
- const res = rsocket.requestStream({
8768
+ let pendingEventsCount = syncQueueRequestSize;
8769
+ let paused = false;
8770
+ let res = null;
8771
+ function requestMore() {
8772
+ const delta = syncQueueRequestSize - pendingEventsCount;
8773
+ if (!paused && delta > 0) {
8774
+ res?.request(delta);
8775
+ pendingEventsCount = syncQueueRequestSize;
8776
+ }
8777
+ }
8778
+ const events = new eventIterator.EventIterator((q) => {
8779
+ queue = q;
8780
+ q.on('highWater', () => (paused = true));
8781
+ q.on('lowWater', () => {
8782
+ paused = false;
8783
+ requestMore();
8784
+ });
8785
+ }, { highWaterMark: SYNC_QUEUE_REQUEST_HIGH_WATER, lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER })[Symbol.asyncIterator]();
8786
+ res = rsocket.requestStream({
8871
8787
  data: toBuffer(options.data),
8872
8788
  metadata: toBuffer({
8873
8789
  path
@@ -8892,7 +8808,7 @@ class AbstractRemote {
8892
8808
  }
8893
8809
  // RSocket will close the RSocket stream automatically
8894
8810
  // Close the downstream stream as well - this will close the RSocket connection and WebSocket
8895
- stream.close();
8811
+ abortRequest();
8896
8812
  // Handles cases where the connection failed e.g. auth error or connection error
8897
8813
  if (!connectionEstablished) {
8898
8814
  reject(e);
@@ -8902,41 +8818,40 @@ class AbstractRemote {
8902
8818
  // The connection is active
8903
8819
  if (!connectionEstablished) {
8904
8820
  connectionEstablished = true;
8905
- resolve(res);
8821
+ resolve(events);
8906
8822
  }
8907
8823
  const { data } = payload;
8824
+ if (data) {
8825
+ queue.push(data);
8826
+ }
8908
8827
  // Less events are now pending
8909
8828
  pendingEventsCount--;
8910
- if (!data) {
8911
- return;
8912
- }
8913
- stream.enqueueData(data);
8829
+ // Request another event (unless the downstream consumer is paused).
8830
+ requestMore();
8914
8831
  },
8915
8832
  onComplete: () => {
8916
- stream.close();
8833
+ abortRequest(); // this will also emit a done event
8917
8834
  },
8918
8835
  onExtension: () => { }
8919
8836
  });
8920
8837
  });
8921
- const l = stream.registerListener({
8922
- lowWater: async () => {
8923
- // Request to fill up the queue
8924
- const required = syncQueueRequestSize - pendingEventsCount;
8925
- if (required > 0) {
8926
- socket.request(syncQueueRequestSize - pendingEventsCount);
8927
- pendingEventsCount = syncQueueRequestSize;
8928
- }
8929
- },
8930
- closed: () => {
8931
- l();
8932
- }
8933
- });
8934
- return stream;
8935
8838
  }
8936
8839
  /**
8937
- * Connects to the sync/stream http endpoint, mapping and emitting each received string line.
8840
+ * @returns Whether the HTTP implementation on this platform can receive streamed binary responses. This is true on
8841
+ * all platforms except React Native (who would have guessed...), where we must not request BSON responses.
8842
+ *
8843
+ * @see https://github.com/react-native-community/fetch?tab=readme-ov-file#motivation
8844
+ */
8845
+ get supportsStreamingBinaryResponses() {
8846
+ return true;
8847
+ }
8848
+ /**
8849
+ * Posts a `/sync/stream` request, asserts that it completes successfully and returns the streaming response as an
8850
+ * async iterator of byte blobs.
8851
+ *
8852
+ * To cancel the async iterator, use the abort signal from {@link SyncStreamOptions} passed to this method.
8938
8853
  */
8939
- async postStreamRaw(options, mapLine) {
8854
+ async fetchStreamRaw(options) {
8940
8855
  const { data, path, headers, abortSignal } = options;
8941
8856
  const request = await this.buildRequest(path);
8942
8857
  /**
@@ -8948,119 +8863,94 @@ class AbstractRemote {
8948
8863
  * Aborting the active fetch request while it is being consumed seems to throw
8949
8864
  * an unhandled exception on the window level.
8950
8865
  */
8951
- if (abortSignal?.aborted) {
8952
- throw new AbortOperation('Abort request received before making postStreamRaw request');
8866
+ if (abortSignal.aborted) {
8867
+ throw new AbortOperation('Abort request received before making fetchStreamRaw request');
8953
8868
  }
8954
8869
  const controller = new AbortController();
8955
- let requestResolved = false;
8956
- abortSignal?.addEventListener('abort', () => {
8957
- if (!requestResolved) {
8870
+ let reader = null;
8871
+ abortSignal.addEventListener('abort', () => {
8872
+ const reason = abortSignal.reason ??
8873
+ new AbortOperation('Cancelling network request before it resolves. Abort signal has been received.');
8874
+ if (reader == null) {
8958
8875
  // Only abort via the abort controller if the request has not resolved yet
8959
- controller.abort(abortSignal.reason ??
8960
- new AbortOperation('Cancelling network request before it resolves. Abort signal has been received.'));
8876
+ controller.abort(reason);
8877
+ }
8878
+ else {
8879
+ reader.cancel(reason).catch(() => {
8880
+ // Cancelling the reader might rethrow an exception we would have handled by throwing in next(). So we can
8881
+ // ignore it here.
8882
+ });
8961
8883
  }
8962
8884
  });
8963
- const res = await this.fetch(request.url, {
8964
- method: 'POST',
8965
- headers: { ...headers, ...request.headers },
8966
- body: JSON.stringify(data),
8967
- signal: controller.signal,
8968
- cache: 'no-store',
8969
- ...(this.options.fetchOptions ?? {}),
8970
- ...options.fetchOptions
8971
- }).catch((ex) => {
8885
+ let res;
8886
+ let responseIsBson = false;
8887
+ try {
8888
+ const ndJson = 'application/x-ndjson';
8889
+ const bson = 'application/vnd.powersync.bson-stream';
8890
+ res = await this.fetch(request.url, {
8891
+ method: 'POST',
8892
+ headers: {
8893
+ ...headers,
8894
+ ...request.headers,
8895
+ accept: this.supportsStreamingBinaryResponses ? `${bson};q=0.9,${ndJson};q=0.8` : ndJson
8896
+ },
8897
+ body: JSON.stringify(data),
8898
+ signal: controller.signal,
8899
+ cache: 'no-store',
8900
+ ...(this.options.fetchOptions ?? {}),
8901
+ ...options.fetchOptions
8902
+ });
8903
+ if (!res.ok || !res.body) {
8904
+ const text = await res.text();
8905
+ this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
8906
+ const error = new Error(`HTTP ${res.statusText}: ${text}`);
8907
+ error.status = res.status;
8908
+ throw error;
8909
+ }
8910
+ const contentType = res.headers.get('content-type');
8911
+ responseIsBson = contentType == bson;
8912
+ }
8913
+ catch (ex) {
8972
8914
  if (ex.name == 'AbortError') {
8973
8915
  throw new AbortOperation(`Pending fetch request to ${request.url} has been aborted.`);
8974
8916
  }
8975
8917
  throw ex;
8976
- });
8977
- if (!res) {
8978
- throw new Error('Fetch request was aborted');
8979
- }
8980
- requestResolved = true;
8981
- if (!res.ok || !res.body) {
8982
- const text = await res.text();
8983
- this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
8984
- const error = new Error(`HTTP ${res.statusText}: ${text}`);
8985
- error.status = res.status;
8986
- throw error;
8987
8918
  }
8988
- // Create a new stream splitting the response at line endings while also handling cancellations
8989
- // by closing the reader.
8990
- const reader = res.body.getReader();
8991
- let readerReleased = false;
8992
- // This will close the network request and read stream
8993
- const closeReader = async () => {
8994
- try {
8995
- readerReleased = true;
8996
- await reader.cancel();
8997
- }
8998
- catch (ex) {
8999
- // an error will throw if the reader hasn't been used yet
9000
- }
9001
- reader.releaseLock();
9002
- };
9003
- const stream = new DataStream({
9004
- logger: this.logger,
9005
- mapLine: mapLine,
9006
- pressure: {
9007
- highWaterMark: 20,
9008
- lowWaterMark: 10
9009
- }
9010
- });
9011
- abortSignal?.addEventListener('abort', () => {
9012
- closeReader();
9013
- stream.close();
9014
- });
9015
- const decoder = this.createTextDecoder();
9016
- let buffer = '';
9017
- const consumeStream = async () => {
9018
- while (!stream.closed && !abortSignal?.aborted && !readerReleased) {
9019
- const { done, value } = await reader.read();
9020
- if (done) {
9021
- const remaining = buffer.trim();
9022
- if (remaining.length != 0) {
9023
- stream.enqueueData(remaining);
9024
- }
9025
- stream.close();
9026
- await closeReader();
9027
- return;
8919
+ reader = res.body.getReader();
8920
+ const stream = {
8921
+ next: async () => {
8922
+ if (controller.signal.aborted) {
8923
+ return doneResult;
9028
8924
  }
9029
- const data = decoder.decode(value, { stream: true });
9030
- buffer += data;
9031
- const lines = buffer.split('\n');
9032
- for (var i = 0; i < lines.length - 1; i++) {
9033
- var l = lines[i].trim();
9034
- if (l.length > 0) {
9035
- stream.enqueueData(l);
9036
- }
8925
+ try {
8926
+ return await reader.read();
9037
8927
  }
9038
- buffer = lines[lines.length - 1];
9039
- // Implement backpressure by waiting for the low water mark to be reached
9040
- if (stream.dataQueue.length > stream.highWatermark) {
9041
- await new Promise((resolve) => {
9042
- const dispose = stream.registerListener({
9043
- lowWater: async () => {
9044
- resolve();
9045
- dispose();
9046
- },
9047
- closed: () => {
9048
- resolve();
9049
- dispose();
9050
- }
9051
- });
9052
- });
8928
+ catch (ex) {
8929
+ if (controller.signal.aborted) {
8930
+ // .read() completes with an error if we cancel the reader, which we do to disconnect. So this is just
8931
+ // things working as intended, we can return a done event and consider the exception handled.
8932
+ return doneResult;
8933
+ }
8934
+ throw ex;
9053
8935
  }
9054
8936
  }
9055
8937
  };
9056
- consumeStream().catch(ex => this.logger.error('Error consuming stream', ex));
9057
- const l = stream.registerListener({
9058
- closed: () => {
9059
- closeReader();
9060
- l?.();
9061
- }
9062
- });
9063
- return stream;
8938
+ return { isBson: responseIsBson, stream };
8939
+ }
8940
+ /**
8941
+ * Posts a `/sync/stream` request.
8942
+ *
8943
+ * Depending on the `Content-Type` of the response, this returns strings for sync lines or encoded BSON documents as
8944
+ * {@link Uint8Array}s.
8945
+ */
8946
+ async fetchStream(options) {
8947
+ const { isBson, stream } = await this.fetchStreamRaw(options);
8948
+ if (isBson) {
8949
+ return extractBsonObjects(stream);
8950
+ }
8951
+ else {
8952
+ return extractJsonLines(stream, this.createTextDecoder());
8953
+ }
9064
8954
  }
9065
8955
  }
9066
8956
 
@@ -9089,31 +8979,8 @@ function coreStatusToJs(status) {
9089
8979
  priorityStatusEntries: status.priority_status.map(priorityToJs)
9090
8980
  };
9091
8981
  }
9092
-
9093
- function isStreamingSyncData(line) {
9094
- return line.data != null;
9095
- }
9096
- function isStreamingKeepalive(line) {
9097
- return line.token_expires_in != null;
9098
- }
9099
- function isStreamingSyncCheckpoint(line) {
9100
- return line.checkpoint != null;
9101
- }
9102
- function isStreamingSyncCheckpointComplete(line) {
9103
- return line.checkpoint_complete != null;
9104
- }
9105
- function isStreamingSyncCheckpointPartiallyComplete(line) {
9106
- return line.partial_checkpoint_complete != null;
9107
- }
9108
- function isStreamingSyncCheckpointDiff(line) {
9109
- return line.checkpoint_diff != null;
9110
- }
9111
- function isContinueCheckpointRequest(request) {
9112
- return (Array.isArray(request.buckets) &&
9113
- typeof request.checkpoint_token == 'string');
9114
- }
9115
- function isSyncNewCheckpointRequest(request) {
9116
- return typeof request.request_checkpoint == 'object';
8982
+ function isInterruptingInstruction(instruction) {
8983
+ return 'EstablishSyncStream' in instruction || 'CloseSyncStream' in instruction;
9117
8984
  }
9118
8985
 
9119
8986
  exports.LockType = void 0;
@@ -9128,35 +8995,21 @@ exports.SyncStreamConnectionMethod = void 0;
9128
8995
  })(exports.SyncStreamConnectionMethod || (exports.SyncStreamConnectionMethod = {}));
9129
8996
  exports.SyncClientImplementation = void 0;
9130
8997
  (function (SyncClientImplementation) {
9131
- /**
9132
- * Decodes and handles sync lines received from the sync service in JavaScript.
9133
- *
9134
- * This is the default option.
9135
- *
9136
- * @deprecated We recommend the {@link RUST} client implementation for all apps. If you have issues with
9137
- * the Rust client, please file an issue or reach out to us. The JavaScript client will be removed in a future
9138
- * version of the PowerSync SDK.
9139
- */
9140
- SyncClientImplementation["JAVASCRIPT"] = "js";
9141
8998
  /**
9142
8999
  * This implementation offloads the sync line decoding and handling into the PowerSync
9143
9000
  * core extension.
9144
9001
  *
9145
- * This option is more performant than the {@link JAVASCRIPT} client, enabled by default and the
9146
- * recommended client implementation for all apps.
9002
+ * This is the only option, as an older JavaScript client implementation has been removed from the SDK.
9147
9003
  *
9148
9004
  * ## Compatibility warning
9149
9005
  *
9150
9006
  * The Rust sync client stores sync data in a format that is slightly different than the one used
9151
- * by the old {@link JAVASCRIPT} implementation. When adopting the {@link RUST} client on existing
9152
- * databases, the PowerSync SDK will migrate the format automatically.
9153
- * Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
9154
- * the version introducing {@link RUST} as an option) also supports the new format, so you can switch
9155
- * back to {@link JAVASCRIPT} later.
9007
+ * by the old JavaScript client. When adopting the {@link RUST} client on existing databases, the PowerSync SDK will
9008
+ * migrate the format automatically.
9156
9009
  *
9157
- * __However__: Upgrading the SDK version, then adopting {@link RUST} as a sync client and later
9158
- * downgrading the SDK to an older version (necessarily using the JavaScript-based implementation then)
9159
- * can lead to sync issues.
9010
+ * SDK versions supporting both the JavaScript and the Rust client support both formats with the JavaScript client
9011
+ * implementaiton. However, downgrading to an SDK version that only supports the JavaScript client would not be
9012
+ * possible anymore. Problematic SDK versions have been released before 2025-06-09.
9160
9013
  */
9161
9014
  SyncClientImplementation["RUST"] = "rust";
9162
9015
  })(exports.SyncClientImplementation || (exports.SyncClientImplementation = {}));
@@ -9179,13 +9032,7 @@ const DEFAULT_STREAM_CONNECTION_OPTIONS = {
9179
9032
  serializedSchema: undefined,
9180
9033
  includeDefaultStreams: true
9181
9034
  };
9182
- // The priority we assume when we receive checkpoint lines where no priority is set.
9183
- // This is the default priority used by the sync service, but can be set to an arbitrary
9184
- // value since sync services without priorities also won't send partial sync completion
9185
- // messages.
9186
- const FALLBACK_PRIORITY = 3;
9187
9035
  class AbstractStreamingSyncImplementation extends BaseObserver {
9188
- _lastSyncedAt;
9189
9036
  options;
9190
9037
  abortController;
9191
9038
  // In rare cases, mostly for tests, uploads can be triggered without being properly connected.
@@ -9195,6 +9042,7 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
9195
9042
  streamingSyncPromise;
9196
9043
  logger;
9197
9044
  activeStreams;
9045
+ connectionMayHaveChanged = false;
9198
9046
  isUploadingCrud = false;
9199
9047
  notifyCompletedUploads;
9200
9048
  handleActiveStreamsChange;
@@ -9274,9 +9122,6 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
9274
9122
  this.crudUpdateListener = undefined;
9275
9123
  this.uploadAbortController?.abort();
9276
9124
  }
9277
- async hasCompletedSync() {
9278
- return this.options.adapter.hasCompletedSync();
9279
- }
9280
9125
  async getWriteCheckpoint() {
9281
9126
  const clientId = await this.options.adapter.getClientId();
9282
9127
  let path = `/write-checkpoint2.json?client_id=${clientId}`;
@@ -9358,7 +9203,7 @@ The next upload iteration will be delayed.`);
9358
9203
  });
9359
9204
  }
9360
9205
  }
9361
- this.uploadAbortController = null;
9206
+ this.uploadAbortController = undefined;
9362
9207
  }
9363
9208
  });
9364
9209
  }
@@ -9474,6 +9319,11 @@ The next upload iteration will be delayed.`);
9474
9319
  shouldDelayRetry = false;
9475
9320
  // A disconnect was requested, we should not delay since there is no explicit retry
9476
9321
  }
9322
+ else if (this.connectionMayHaveChanged && ex.message?.indexOf('No iteration is active') >= 0) {
9323
+ this.connectionMayHaveChanged = false;
9324
+ this.logger.info('Sync error after changed connection, retrying immediately');
9325
+ shouldDelayRetry = false;
9326
+ }
9477
9327
  else {
9478
9328
  this.logger.error(ex);
9479
9329
  }
@@ -9504,17 +9354,14 @@ The next upload iteration will be delayed.`);
9504
9354
  // Mark as disconnected if here
9505
9355
  this.updateSyncStatus({ connected: false, connecting: false });
9506
9356
  }
9507
- async collectLocalBucketState() {
9508
- const bucketEntries = await this.options.adapter.getBucketStates();
9509
- const req = bucketEntries.map((entry) => ({
9510
- name: entry.bucket,
9511
- after: entry.op_id
9512
- }));
9513
- const localDescriptions = new Map();
9514
- for (const entry of bucketEntries) {
9515
- localDescriptions.set(entry.bucket, null);
9516
- }
9517
- return [req, localDescriptions];
9357
+ markConnectionMayHaveChanged() {
9358
+ // By setting this field, we'll immediately retry if the next sync event causes an error triggered by us not having
9359
+ // an active sync iteration on the connection in use.
9360
+ this.connectionMayHaveChanged = true;
9361
+ // This triggers a `powersync_control` invocation if a sync iteration is currently active. This is a cheap call to
9362
+ // make when no subscriptions have actually changed, we're mainly interested in this immediately throwing if no
9363
+ // iteration is active. That allows us to reconnect ASAP, instead of having to wait for the next sync line.
9364
+ this.handleActiveStreamsChange?.();
9518
9365
  }
9519
9366
  /**
9520
9367
  * Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
@@ -9555,344 +9402,98 @@ The next upload iteration will be delayed.`);
9555
9402
  if (invalidMetadata.length > 0) {
9556
9403
  throw new Error(`Invalid appMetadata provided. Only string values are allowed. Invalid values: ${invalidMetadata.map(([key, value]) => `${key}: ${value}`).join(', ')}`);
9557
9404
  }
9558
- const clientImplementation = resolvedOptions.clientImplementation;
9559
- this.updateSyncStatus({ clientImplementation });
9560
- if (clientImplementation == exports.SyncClientImplementation.JAVASCRIPT) {
9561
- await this.legacyStreamingSyncIteration(signal, resolvedOptions);
9562
- return null;
9563
- }
9564
- else {
9565
- await this.requireKeyFormat(true);
9566
- return await this.rustSyncIteration(signal, resolvedOptions);
9567
- }
9405
+ await this.requireKeyFormat(true);
9406
+ return await this.rustSyncIteration(signal, resolvedOptions);
9568
9407
  }
9569
9408
  });
9570
9409
  }
9571
- async legacyStreamingSyncIteration(signal, resolvedOptions) {
9572
- const rawTables = resolvedOptions.serializedSchema?.raw_tables;
9573
- if (rawTables != null && rawTables.length) {
9574
- this.logger.warn('Raw tables require the Rust-based sync client. The JS client will ignore them.');
9575
- }
9576
- if (this.activeStreams.length) {
9577
- this.logger.error('Sync streams require `clientImplementation: SyncClientImplementation.RUST` when connecting.');
9578
- }
9579
- this.logger.debug('Streaming sync iteration started');
9580
- this.options.adapter.startSession();
9581
- let [req, bucketMap] = await this.collectLocalBucketState();
9582
- let targetCheckpoint = null;
9583
- // A checkpoint that has been validated but not applied (e.g. due to pending local writes)
9584
- let pendingValidatedCheckpoint = null;
9585
- const clientId = await this.options.adapter.getClientId();
9586
- const usingFixedKeyFormat = await this.requireKeyFormat(false);
9587
- this.logger.debug('Requesting stream from server');
9588
- const syncOptions = {
9589
- path: '/sync/stream',
9590
- abortSignal: signal,
9591
- data: {
9592
- buckets: req,
9593
- include_checksum: true,
9594
- raw_data: true,
9595
- parameters: resolvedOptions.params,
9596
- app_metadata: resolvedOptions.appMetadata,
9597
- client_id: clientId
9410
+ receiveSyncLines(data) {
9411
+ const { options, connection } = data;
9412
+ const remote = this.options.remote;
9413
+ const openInner = async () => {
9414
+ if (connection.connectionMethod == exports.SyncStreamConnectionMethod.HTTP) {
9415
+ return await remote.fetchStream(options);
9598
9416
  }
9599
- };
9600
- let stream;
9601
- if (resolvedOptions?.connectionMethod == exports.SyncStreamConnectionMethod.HTTP) {
9602
- stream = await this.options.remote.postStreamRaw(syncOptions, (line) => {
9603
- if (typeof line == 'string') {
9604
- return JSON.parse(line);
9605
- }
9606
- else {
9607
- // Directly enqueued by us
9608
- return line;
9609
- }
9610
- });
9611
- }
9612
- else {
9613
- const bson = await this.options.remote.getBSON();
9614
- stream = await this.options.remote.socketStreamRaw({
9615
- ...syncOptions,
9616
- ...{ fetchStrategy: resolvedOptions.fetchStrategy }
9617
- }, (payload) => {
9618
- if (payload instanceof Uint8Array) {
9619
- return bson.deserialize(payload);
9620
- }
9621
- else {
9622
- // Directly enqueued by us
9623
- return payload;
9624
- }
9625
- }, bson);
9626
- }
9627
- this.logger.debug('Stream established. Processing events');
9628
- this.notifyCompletedUploads = () => {
9629
- if (!stream.closed) {
9630
- stream.enqueueData({ crud_upload_completed: null });
9417
+ else {
9418
+ return await this.options.remote.socketStreamRaw({
9419
+ ...options,
9420
+ ...{ fetchStrategy: connection.fetchStrategy }
9421
+ });
9631
9422
  }
9632
9423
  };
9633
- while (!stream.closed) {
9634
- const line = await stream.read();
9635
- if (!line) {
9636
- // The stream has closed while waiting
9637
- return;
9638
- }
9639
- if ('crud_upload_completed' in line) {
9640
- if (pendingValidatedCheckpoint != null) {
9641
- const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
9642
- if (applied) {
9643
- pendingValidatedCheckpoint = null;
9644
- }
9645
- else if (endIteration) {
9646
- break;
9647
- }
9424
+ let inner;
9425
+ let done = false;
9426
+ return {
9427
+ async next() {
9428
+ if (done) {
9429
+ return doneResult;
9648
9430
  }
9649
- continue;
9650
- }
9651
- // A connection is active and messages are being received
9652
- if (!this.syncStatus.connected) {
9653
- // There is a connection now
9654
- Promise.resolve().then(() => this.triggerCrudUpload());
9655
- this.updateSyncStatus({
9656
- connected: true
9657
- });
9658
- }
9659
- if (isStreamingSyncCheckpoint(line)) {
9660
- targetCheckpoint = line.checkpoint;
9661
- // New checkpoint - existing validated checkpoint is no longer valid
9662
- pendingValidatedCheckpoint = null;
9663
- const bucketsToDelete = new Set(bucketMap.keys());
9664
- const newBuckets = new Map();
9665
- for (const checksum of line.checkpoint.buckets) {
9666
- newBuckets.set(checksum.bucket, {
9667
- name: checksum.bucket,
9668
- priority: checksum.priority ?? FALLBACK_PRIORITY
9431
+ else if (inner == null) {
9432
+ inner = await openInner();
9433
+ // We're connected here, so we can tell the core extension about it.
9434
+ return valueResult({
9435
+ command: exports.PowerSyncControlCommand.CONNECTION_STATE,
9436
+ payload: 'established'
9669
9437
  });
9670
- bucketsToDelete.delete(checksum.bucket);
9671
- }
9672
- if (bucketsToDelete.size > 0) {
9673
- this.logger.debug('Removing buckets', [...bucketsToDelete]);
9674
- }
9675
- bucketMap = newBuckets;
9676
- await this.options.adapter.removeBuckets([...bucketsToDelete]);
9677
- await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
9678
- await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
9679
- }
9680
- else if (isStreamingSyncCheckpointComplete(line)) {
9681
- const result = await this.applyCheckpoint(targetCheckpoint);
9682
- if (result.endIteration) {
9683
- return;
9684
- }
9685
- else if (!result.applied) {
9686
- // "Could not apply checkpoint due to local data". We need to retry after
9687
- // finishing uploads.
9688
- pendingValidatedCheckpoint = targetCheckpoint;
9689
9438
  }
9690
9439
  else {
9691
- // Nothing to retry later. This would likely already be null from the last
9692
- // checksum or checksum_diff operation, but we make sure.
9693
- pendingValidatedCheckpoint = null;
9694
- }
9695
- }
9696
- else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
9697
- const priority = line.partial_checkpoint_complete.priority;
9698
- this.logger.debug('Partial checkpoint complete', priority);
9699
- const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint, priority);
9700
- if (!result.checkpointValid) {
9701
- // This means checksums failed. Start again with a new checkpoint.
9702
- // TODO: better back-off
9703
- await new Promise((resolve) => setTimeout(resolve, 50));
9704
- return;
9705
- }
9706
- else if (!result.ready) ;
9707
- else {
9708
- // We'll keep on downloading, but can report that this priority is synced now.
9709
- this.logger.debug('partial checkpoint validation succeeded');
9710
- // All states with a higher priority can be deleted since this partial sync includes them.
9711
- const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
9712
- priorityStates.push({
9713
- priority,
9714
- lastSyncedAt: new Date(),
9715
- hasSynced: true
9716
- });
9717
- this.updateSyncStatus({
9718
- connected: true,
9719
- priorityStatusEntries: priorityStates
9720
- });
9721
- }
9722
- }
9723
- else if (isStreamingSyncCheckpointDiff(line)) {
9724
- // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
9725
- if (targetCheckpoint == null) {
9726
- throw new Error('Checkpoint diff without previous checkpoint');
9727
- }
9728
- // New checkpoint - existing validated checkpoint is no longer valid
9729
- pendingValidatedCheckpoint = null;
9730
- const diff = line.checkpoint_diff;
9731
- const newBuckets = new Map();
9732
- for (const checksum of targetCheckpoint.buckets) {
9733
- newBuckets.set(checksum.bucket, checksum);
9734
- }
9735
- for (const checksum of diff.updated_buckets) {
9736
- newBuckets.set(checksum.bucket, checksum);
9737
- }
9738
- for (const bucket of diff.removed_buckets) {
9739
- newBuckets.delete(bucket);
9740
- }
9741
- const newCheckpoint = {
9742
- last_op_id: diff.last_op_id,
9743
- buckets: [...newBuckets.values()],
9744
- write_checkpoint: diff.write_checkpoint
9745
- };
9746
- targetCheckpoint = newCheckpoint;
9747
- await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
9748
- bucketMap = new Map();
9749
- newBuckets.forEach((checksum, name) => bucketMap.set(name, {
9750
- name: checksum.bucket,
9751
- priority: checksum.priority ?? FALLBACK_PRIORITY
9752
- }));
9753
- const bucketsToDelete = diff.removed_buckets;
9754
- if (bucketsToDelete.length > 0) {
9755
- this.logger.debug('Remove buckets', bucketsToDelete);
9756
- }
9757
- await this.options.adapter.removeBuckets(bucketsToDelete);
9758
- await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
9759
- }
9760
- else if (isStreamingSyncData(line)) {
9761
- const { data } = line;
9762
- const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
9763
- let updatedProgress = null;
9764
- if (previousProgress) {
9765
- updatedProgress = { ...previousProgress };
9766
- const progressForBucket = updatedProgress[data.bucket];
9767
- if (progressForBucket) {
9768
- updatedProgress[data.bucket] = {
9769
- ...progressForBucket,
9770
- since_last: progressForBucket.since_last + data.data.length
9771
- };
9440
+ const event = await inner.next();
9441
+ if (event.done) {
9442
+ done = true;
9443
+ return valueResult({ command: exports.PowerSyncControlCommand.CONNECTION_STATE, payload: 'end' });
9772
9444
  }
9773
- }
9774
- this.updateSyncStatus({
9775
- dataFlow: {
9776
- downloading: true,
9777
- downloadProgress: updatedProgress
9445
+ else {
9446
+ return valueResult({
9447
+ command: typeof event.value == 'string'
9448
+ ? exports.PowerSyncControlCommand.PROCESS_TEXT_LINE
9449
+ : exports.PowerSyncControlCommand.PROCESS_BSON_LINE,
9450
+ payload: event.value
9451
+ });
9778
9452
  }
9779
- });
9780
- await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] }, usingFixedKeyFormat);
9781
- }
9782
- else if (isStreamingKeepalive(line)) {
9783
- const remaining_seconds = line.token_expires_in;
9784
- if (remaining_seconds == 0) {
9785
- // Connection would be closed automatically right after this
9786
- this.logger.debug('Token expiring; reconnect');
9787
- /**
9788
- * For a rare case where the backend connector does not update the token
9789
- * (uses the same one), this should have some delay.
9790
- */
9791
- await this.delayRetry();
9792
- return;
9793
- }
9794
- else if (remaining_seconds < 30) {
9795
- this.logger.debug('Token will expire soon; reconnect');
9796
- // Pre-emptively refresh the token
9797
- this.options.remote.invalidateCredentials();
9798
- return;
9799
9453
  }
9800
- this.triggerCrudUpload();
9801
- }
9802
- else {
9803
- this.logger.debug('Received unknown sync line', line);
9804
9454
  }
9805
- }
9806
- this.logger.debug('Stream input empty');
9807
- // Connection closed. Likely due to auth issue.
9808
- return;
9455
+ };
9809
9456
  }
9810
9457
  async rustSyncIteration(signal, resolvedOptions) {
9811
9458
  const syncImplementation = this;
9812
9459
  const adapter = this.options.adapter;
9813
9460
  const remote = this.options.remote;
9814
- let receivingLines = null;
9815
- let hadSyncLine = false;
9816
9461
  let hideDisconnectOnRestart = false;
9462
+ let notifyTokenRefreshed;
9817
9463
  if (signal.aborted) {
9818
9464
  throw new AbortOperation('Connection request has been aborted');
9819
9465
  }
9820
- const abortController = new AbortController();
9821
- signal.addEventListener('abort', () => abortController.abort());
9822
- // Pending sync lines received from the service, as well as local events that trigger a powersync_control
9823
- // invocation (local events include refreshed tokens and completed uploads).
9824
- // This is a single data stream so that we can handle all control calls from a single place.
9825
- let controlInvocations = null;
9826
- async function connect(instr) {
9827
- const syncOptions = {
9828
- path: '/sync/stream',
9829
- abortSignal: abortController.signal,
9830
- data: instr.request
9466
+ function startCommand() {
9467
+ const options = {
9468
+ parameters: resolvedOptions.params,
9469
+ app_metadata: resolvedOptions.appMetadata,
9470
+ active_streams: syncImplementation.activeStreams,
9471
+ include_defaults: resolvedOptions.includeDefaultStreams
9831
9472
  };
9832
- if (resolvedOptions.connectionMethod == exports.SyncStreamConnectionMethod.HTTP) {
9833
- controlInvocations = await remote.postStreamRaw(syncOptions, (line) => {
9834
- if (typeof line == 'string') {
9835
- return {
9836
- command: exports.PowerSyncControlCommand.PROCESS_TEXT_LINE,
9837
- payload: line
9838
- };
9839
- }
9840
- else {
9841
- // Directly enqueued by us
9842
- return line;
9843
- }
9844
- });
9845
- }
9846
- else {
9847
- controlInvocations = await remote.socketStreamRaw({
9848
- ...syncOptions,
9849
- fetchStrategy: resolvedOptions.fetchStrategy
9850
- }, (payload) => {
9851
- if (payload instanceof Uint8Array) {
9852
- return {
9853
- command: exports.PowerSyncControlCommand.PROCESS_BSON_LINE,
9854
- payload: payload
9855
- };
9856
- }
9857
- else {
9858
- // Directly enqueued by us
9859
- return payload;
9860
- }
9861
- });
9862
- }
9863
- // The rust client will set connected: true after the first sync line because that's when it gets invoked, but
9864
- // we're already connected here and can report that.
9865
- syncImplementation.updateSyncStatus({ connected: true });
9866
- try {
9867
- while (!controlInvocations.closed) {
9868
- const line = await controlInvocations.read();
9869
- if (line == null) {
9870
- return;
9871
- }
9872
- await control(line.command, line.payload);
9873
- if (!hadSyncLine) {
9874
- syncImplementation.triggerCrudUpload();
9875
- hadSyncLine = true;
9876
- }
9877
- }
9878
- }
9879
- finally {
9880
- const activeInstructions = controlInvocations;
9881
- // We concurrently add events to the active data stream when e.g. a CRUD upload is completed or a token is
9882
- // refreshed. That would throw after closing (and we can't handle those events either way), so set this back
9883
- // to null.
9884
- controlInvocations = null;
9885
- await activeInstructions.close();
9473
+ if (resolvedOptions.serializedSchema) {
9474
+ options.schema = resolvedOptions.serializedSchema;
9886
9475
  }
9476
+ return invokePowerSyncControl(exports.PowerSyncControlCommand.START, JSON.stringify(options));
9887
9477
  }
9888
9478
  async function stop() {
9889
- await control(exports.PowerSyncControlCommand.STOP);
9479
+ const instructions = await invokePowerSyncControl(exports.PowerSyncControlCommand.STOP);
9480
+ for (const instruction of instructions) {
9481
+ // We don't need to handle interrupting instructions since we're unconditionally ending the sync iteration at
9482
+ // this point.
9483
+ if (isInterruptingInstruction(instruction))
9484
+ continue;
9485
+ await handleInstruction(instruction);
9486
+ }
9890
9487
  }
9891
- async function control(op, payload) {
9488
+ async function invokePowerSyncControl(op, payload) {
9892
9489
  const rawResponse = await adapter.control(op, payload ?? null);
9893
9490
  const logger = syncImplementation.logger;
9894
9491
  logger.trace('powersync_control', op, payload == null || typeof payload == 'string' ? payload : '<bytes>', rawResponse);
9895
- await handleInstructions(JSON.parse(rawResponse));
9492
+ if (op != exports.PowerSyncControlCommand.STOP) {
9493
+ // Evidently we have a working connection here, otherwise powersync_control would have failed.
9494
+ syncImplementation.connectionMayHaveChanged = false;
9495
+ }
9496
+ return JSON.parse(rawResponse);
9896
9497
  }
9897
9498
  async function handleInstruction(instruction) {
9898
9499
  if ('LogLine' in instruction) {
@@ -9911,13 +9512,6 @@ The next upload iteration will be delayed.`);
9911
9512
  else if ('UpdateSyncStatus' in instruction) {
9912
9513
  syncImplementation.updateSyncStatus(coreStatusToJs(instruction.UpdateSyncStatus.status));
9913
9514
  }
9914
- else if ('EstablishSyncStream' in instruction) {
9915
- if (receivingLines != null) {
9916
- // Already connected, this shouldn't happen during a single iteration.
9917
- throw 'Unexpected request to establish sync stream, already connected';
9918
- }
9919
- receivingLines = connect(instruction.EstablishSyncStream);
9920
- }
9921
9515
  else if ('FetchCredentials' in instruction) {
9922
9516
  if (instruction.FetchCredentials.did_expire) {
9923
9517
  remote.invalidateCredentials();
@@ -9926,16 +9520,12 @@ The next upload iteration will be delayed.`);
9926
9520
  remote.invalidateCredentials();
9927
9521
  // Restart iteration after the credentials have been refreshed.
9928
9522
  remote.fetchCredentials().then((_) => {
9929
- controlInvocations?.enqueueData({ command: exports.PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED });
9523
+ notifyTokenRefreshed?.();
9930
9524
  }, (err) => {
9931
9525
  syncImplementation.logger.warn('Could not prefetch credentials', err);
9932
9526
  });
9933
9527
  }
9934
9528
  }
9935
- else if ('CloseSyncStream' in instruction) {
9936
- abortController.abort();
9937
- hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
9938
- }
9939
9529
  else if ('FlushFileSystem' in instruction) ;
9940
9530
  else if ('DidCompleteSync' in instruction) {
9941
9531
  syncImplementation.updateSyncStatus({
@@ -9945,105 +9535,83 @@ The next upload iteration will be delayed.`);
9945
9535
  });
9946
9536
  }
9947
9537
  }
9948
- async function handleInstructions(instructions) {
9949
- for (const instr of instructions) {
9950
- await handleInstruction(instr);
9951
- }
9952
- }
9953
9538
  try {
9954
- const options = {
9955
- parameters: resolvedOptions.params,
9956
- app_metadata: resolvedOptions.appMetadata,
9957
- active_streams: this.activeStreams,
9958
- include_defaults: resolvedOptions.includeDefaultStreams
9959
- };
9960
- if (resolvedOptions.serializedSchema) {
9961
- options.schema = resolvedOptions.serializedSchema;
9539
+ const defaultResult = { immediateRestart: false };
9540
+ // Pending sync lines received from the service, as well as local events that trigger a powersync_control
9541
+ // invocation (local events include refreshed tokens and completed uploads).
9542
+ // This is a single data stream so that we can handle all control calls from a single place.
9543
+ let controlInvocations = null;
9544
+ for (const startInstruction of await startCommand()) {
9545
+ if ('EstablishSyncStream' in startInstruction) {
9546
+ const syncOptions = {
9547
+ path: '/sync/stream',
9548
+ abortSignal: signal,
9549
+ data: startInstruction.EstablishSyncStream.request
9550
+ };
9551
+ controlInvocations = injectable(syncImplementation.receiveSyncLines({
9552
+ options: syncOptions,
9553
+ connection: resolvedOptions
9554
+ }));
9555
+ }
9556
+ else if ('CloseSyncStream' in startInstruction) {
9557
+ return defaultResult;
9558
+ }
9559
+ else {
9560
+ await handleInstruction(startInstruction);
9561
+ }
9962
9562
  }
9963
- await control(exports.PowerSyncControlCommand.START, JSON.stringify(options));
9563
+ if (controlInvocations == null)
9564
+ return defaultResult;
9964
9565
  this.notifyCompletedUploads = () => {
9965
- if (controlInvocations && !controlInvocations?.closed) {
9966
- controlInvocations.enqueueData({ command: exports.PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
9967
- }
9566
+ controlInvocations.inject({ command: exports.PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
9968
9567
  };
9969
9568
  this.handleActiveStreamsChange = () => {
9970
- if (controlInvocations && !controlInvocations?.closed) {
9971
- controlInvocations.enqueueData({
9972
- command: exports.PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
9973
- payload: JSON.stringify(this.activeStreams)
9974
- });
9975
- }
9569
+ controlInvocations.inject({
9570
+ command: exports.PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
9571
+ payload: JSON.stringify(this.activeStreams)
9572
+ });
9976
9573
  };
9977
- await receivingLines;
9574
+ notifyTokenRefreshed = () => {
9575
+ controlInvocations.inject({
9576
+ command: exports.PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED
9577
+ });
9578
+ };
9579
+ let hadSyncLine = false;
9580
+ loop: while (true) {
9581
+ const { done, value } = await controlInvocations.next();
9582
+ if (done)
9583
+ break;
9584
+ if (!hadSyncLine) {
9585
+ // Trigger a local CRUD upload when the first sync line has been received, this allows uploading local changes
9586
+ // that have been made while offline or disconnected.
9587
+ if (value.command == exports.PowerSyncControlCommand.PROCESS_TEXT_LINE ||
9588
+ value.command == exports.PowerSyncControlCommand.PROCESS_BSON_LINE) {
9589
+ hadSyncLine = true;
9590
+ this.triggerCrudUpload?.();
9591
+ }
9592
+ }
9593
+ const instructions = await invokePowerSyncControl(value.command, value.payload);
9594
+ for (const instruction of instructions) {
9595
+ if ('EstablishSyncStream' in instruction) {
9596
+ throw new Error('Received EstablishSyncStream while already connected.');
9597
+ }
9598
+ else if ('CloseSyncStream' in instruction) {
9599
+ hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
9600
+ break loop;
9601
+ }
9602
+ else {
9603
+ await handleInstruction(instruction);
9604
+ }
9605
+ }
9606
+ }
9978
9607
  }
9979
9608
  finally {
9980
9609
  this.notifyCompletedUploads = this.handleActiveStreamsChange = undefined;
9610
+ notifyTokenRefreshed = undefined;
9981
9611
  await stop();
9982
9612
  }
9983
9613
  return { immediateRestart: hideDisconnectOnRestart };
9984
9614
  }
9985
- async updateSyncStatusForStartingCheckpoint(checkpoint) {
9986
- const localProgress = await this.options.adapter.getBucketOperationProgress();
9987
- const progress = {};
9988
- let invalidated = false;
9989
- for (const bucket of checkpoint.buckets) {
9990
- const savedProgress = localProgress[bucket.bucket];
9991
- const atLast = savedProgress?.atLast ?? 0;
9992
- const sinceLast = savedProgress?.sinceLast ?? 0;
9993
- progress[bucket.bucket] = {
9994
- // The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
9995
- // will use by default.
9996
- priority: bucket.priority ?? 3,
9997
- at_last: atLast,
9998
- since_last: sinceLast,
9999
- target_count: bucket.count ?? 0
10000
- };
10001
- if (bucket.count != null && bucket.count < atLast + sinceLast) {
10002
- // Either due to a defrag / sync rule deploy or a compaction operation, the size
10003
- // of the bucket shrank so much that the local ops exceed the ops in the updated
10004
- // bucket. We can't prossibly report progress in this case (it would overshoot 100%).
10005
- invalidated = true;
10006
- }
10007
- }
10008
- if (invalidated) {
10009
- for (const bucket in progress) {
10010
- const bucketProgress = progress[bucket];
10011
- bucketProgress.at_last = 0;
10012
- bucketProgress.since_last = 0;
10013
- }
10014
- }
10015
- this.updateSyncStatus({
10016
- dataFlow: {
10017
- downloading: true,
10018
- downloadProgress: progress
10019
- }
10020
- });
10021
- }
10022
- async applyCheckpoint(checkpoint) {
10023
- let result = await this.options.adapter.syncLocalDatabase(checkpoint);
10024
- if (!result.checkpointValid) {
10025
- this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
10026
- // This means checksums failed. Start again with a new checkpoint.
10027
- // TODO: better back-off
10028
- await new Promise((resolve) => setTimeout(resolve, 50));
10029
- return { applied: false, endIteration: true };
10030
- }
10031
- else if (!result.ready) {
10032
- this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`);
10033
- return { applied: false, endIteration: false };
10034
- }
10035
- this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
10036
- this.updateSyncStatus({
10037
- connected: true,
10038
- lastSyncedAt: new Date(),
10039
- dataFlow: {
10040
- downloading: false,
10041
- downloadProgress: null,
10042
- downloadError: undefined
10043
- }
10044
- });
10045
- return { applied: true, endIteration: false };
10046
- }
10047
9615
  updateSyncStatus(options) {
10048
9616
  const updatedStatus = new SyncStatus({
10049
9617
  connected: options.connected ?? this.syncStatus.connected,
@@ -10989,7 +10557,7 @@ class AbstractPowerSyncDatabase extends BaseObserver {
10989
10557
  * @returns A transaction of CRUD operations to upload, or null if there are none
10990
10558
  */
10991
10559
  async getNextCrudTransaction() {
10992
- const iterator = this.getCrudTransactions()[symbolAsyncIterator]();
10560
+ const iterator = this.getCrudTransactions()[Symbol.asyncIterator]();
10993
10561
  return (await iterator.next()).value;
10994
10562
  }
10995
10563
  /**
@@ -11025,7 +10593,7 @@ class AbstractPowerSyncDatabase extends BaseObserver {
11025
10593
  */
11026
10594
  getCrudTransactions() {
11027
10595
  return {
11028
- [symbolAsyncIterator]: () => {
10596
+ [Symbol.asyncIterator]: () => {
11029
10597
  let lastCrudItemId = -1;
11030
10598
  const sql = `
11031
10599
  WITH RECURSIVE crud_entries AS (
@@ -11582,14 +11150,12 @@ class SqliteBucketStorage extends BaseObserver {
11582
11150
  db;
11583
11151
  logger;
11584
11152
  tableNames;
11585
- _hasCompletedSync;
11586
11153
  updateListener;
11587
11154
  _clientId;
11588
11155
  constructor(db, logger = Logger.get('SqliteBucketStorage')) {
11589
11156
  super();
11590
11157
  this.db = db;
11591
11158
  this.logger = logger;
11592
- this._hasCompletedSync = false;
11593
11159
  this.tableNames = new Set();
11594
11160
  this.updateListener = db.registerListener({
11595
11161
  tablesUpdated: (update) => {
@@ -11601,7 +11167,6 @@ class SqliteBucketStorage extends BaseObserver {
11601
11167
  });
11602
11168
  }
11603
11169
  async init() {
11604
- this._hasCompletedSync = false;
11605
11170
  const existingTableRows = await this.db.getAll(`SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'`);
11606
11171
  for (const row of existingTableRows ?? []) {
11607
11172
  this.tableNames.add(row.name);
@@ -11623,156 +11188,6 @@ class SqliteBucketStorage extends BaseObserver {
11623
11188
  getMaxOpId() {
11624
11189
  return MAX_OP_ID;
11625
11190
  }
11626
- /**
11627
- * Reset any caches.
11628
- */
11629
- startSession() { }
11630
- async getBucketStates() {
11631
- const result = await this.db.getAll("SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != '$local'");
11632
- return result;
11633
- }
11634
- async getBucketOperationProgress() {
11635
- const rows = await this.db.getAll('SELECT name, count_at_last, count_since_last FROM ps_buckets');
11636
- return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
11637
- }
11638
- async saveSyncData(batch, fixedKeyFormat = false) {
11639
- await this.writeTransaction(async (tx) => {
11640
- for (const b of batch.buckets) {
11641
- await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
11642
- 'save',
11643
- JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
11644
- ]);
11645
- this.logger.debug(`Saved batch of data for bucket: ${b.bucket}, operations: ${b.data.length}`);
11646
- }
11647
- });
11648
- }
11649
- async removeBuckets(buckets) {
11650
- for (const bucket of buckets) {
11651
- await this.deleteBucket(bucket);
11652
- }
11653
- }
11654
- /**
11655
- * Mark a bucket for deletion.
11656
- */
11657
- async deleteBucket(bucket) {
11658
- await this.writeTransaction(async (tx) => {
11659
- await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
11660
- });
11661
- this.logger.debug(`Done deleting bucket ${bucket}`);
11662
- }
11663
- async hasCompletedSync() {
11664
- if (this._hasCompletedSync) {
11665
- return true;
11666
- }
11667
- const r = await this.db.get(`SELECT powersync_last_synced_at() as synced_at`);
11668
- const completed = r.synced_at != null;
11669
- if (completed) {
11670
- this._hasCompletedSync = true;
11671
- }
11672
- return completed;
11673
- }
11674
- async syncLocalDatabase(checkpoint, priority) {
11675
- const r = await this.validateChecksums(checkpoint, priority);
11676
- if (!r.checkpointValid) {
11677
- this.logger.error('Checksums failed for', r.checkpointFailures);
11678
- for (const b of r.checkpointFailures ?? []) {
11679
- await this.deleteBucket(b);
11680
- }
11681
- return { ready: false, checkpointValid: false, checkpointFailures: r.checkpointFailures };
11682
- }
11683
- if (priority == null) {
11684
- this.logger.debug(`Validated checksums checkpoint ${checkpoint.last_op_id}`);
11685
- }
11686
- else {
11687
- this.logger.debug(`Validated checksums for partial checkpoint ${checkpoint.last_op_id}, priority ${priority}`);
11688
- }
11689
- let buckets = checkpoint.buckets;
11690
- if (priority !== undefined) {
11691
- buckets = buckets.filter((b) => hasMatchingPriority(priority, b));
11692
- }
11693
- const bucketNames = buckets.map((b) => b.bucket);
11694
- await this.writeTransaction(async (tx) => {
11695
- await tx.execute(`UPDATE ps_buckets SET last_op = ? WHERE name IN (SELECT json_each.value FROM json_each(?))`, [
11696
- checkpoint.last_op_id,
11697
- JSON.stringify(bucketNames)
11698
- ]);
11699
- if (priority == null && checkpoint.write_checkpoint) {
11700
- await tx.execute("UPDATE ps_buckets SET last_op = ? WHERE name = '$local'", [checkpoint.write_checkpoint]);
11701
- }
11702
- });
11703
- const valid = await this.updateObjectsFromBuckets(checkpoint, priority);
11704
- if (!valid) {
11705
- return { ready: false, checkpointValid: true };
11706
- }
11707
- return {
11708
- ready: true,
11709
- checkpointValid: true
11710
- };
11711
- }
11712
- /**
11713
- * Atomically update the local state to the current checkpoint.
11714
- *
11715
- * This includes creating new tables, dropping old tables, and copying data over from the oplog.
11716
- */
11717
- async updateObjectsFromBuckets(checkpoint, priority) {
11718
- let arg = '';
11719
- if (priority !== undefined) {
11720
- const affectedBuckets = [];
11721
- for (const desc of checkpoint.buckets) {
11722
- if (hasMatchingPriority(priority, desc)) {
11723
- affectedBuckets.push(desc.bucket);
11724
- }
11725
- }
11726
- arg = JSON.stringify({ priority, buckets: affectedBuckets });
11727
- }
11728
- return this.writeTransaction(async (tx) => {
11729
- const { insertId: result } = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
11730
- 'sync_local',
11731
- arg
11732
- ]);
11733
- if (result == 1) {
11734
- if (priority == null) {
11735
- const bucketToCount = Object.fromEntries(checkpoint.buckets.map((b) => [b.bucket, b.count]));
11736
- // The two parameters could be replaced with one, but: https://github.com/powersync-ja/better-sqlite3/pull/6
11737
- const jsonBucketCount = JSON.stringify(bucketToCount);
11738
- await tx.execute("UPDATE ps_buckets SET count_since_last = 0, count_at_last = ?->name WHERE name != '$local' AND ?->name IS NOT NULL", [jsonBucketCount, jsonBucketCount]);
11739
- }
11740
- return true;
11741
- }
11742
- else {
11743
- return false;
11744
- }
11745
- });
11746
- }
11747
- async validateChecksums(checkpoint, priority) {
11748
- if (priority !== undefined) {
11749
- // Only validate the buckets within the priority we care about
11750
- const newBuckets = checkpoint.buckets.filter((cs) => hasMatchingPriority(priority, cs));
11751
- checkpoint = { ...checkpoint, buckets: newBuckets };
11752
- }
11753
- const rs = await this.db.execute('SELECT powersync_validate_checkpoint(?) as result', [
11754
- JSON.stringify({ ...checkpoint })
11755
- ]);
11756
- const resultItem = rs.rows?.item(0);
11757
- if (!resultItem) {
11758
- return {
11759
- checkpointValid: false,
11760
- ready: false,
11761
- checkpointFailures: []
11762
- };
11763
- }
11764
- const result = JSON.parse(resultItem['result']);
11765
- if (result['valid']) {
11766
- return { ready: true, checkpointValid: true };
11767
- }
11768
- else {
11769
- return {
11770
- checkpointValid: false,
11771
- ready: false,
11772
- checkpointFailures: result['failed_buckets']
11773
- };
11774
- }
11775
- }
11776
11191
  async updateLocalTarget(cb) {
11777
11192
  const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", [MAX_OP_ID]);
11778
11193
  if (!rs1.length) {
@@ -11863,12 +11278,6 @@ class SqliteBucketStorage extends BaseObserver {
11863
11278
  async writeTransaction(callback, options) {
11864
11279
  return this.db.writeTransaction(callback, options);
11865
11280
  }
11866
- /**
11867
- * Set a target checkpoint.
11868
- */
11869
- async setTargetCheckpoint(checkpoint) {
11870
- // No-op for now
11871
- }
11872
11281
  async control(op, payload) {
11873
11282
  return await this.writeTransaction(async (tx) => {
11874
11283
  const [[raw]] = await tx.executeRaw('SELECT powersync_control(?, ?)', [op, payload]);
@@ -11892,20 +11301,6 @@ class SqliteBucketStorage extends BaseObserver {
11892
11301
  }
11893
11302
  static _subkeyMigrationKey = 'powersync_js_migrated_subkeys';
11894
11303
  }
11895
- function hasMatchingPriority(priority, bucket) {
11896
- return bucket.priority != null && bucket.priority <= priority;
11897
- }
11898
-
11899
- // TODO JSON
11900
- class SyncDataBatch {
11901
- buckets;
11902
- static fromJSON(json) {
11903
- return new SyncDataBatch(json.buckets.map((bucket) => SyncDataBucket.fromRow(bucket)));
11904
- }
11905
- constructor(buckets) {
11906
- this.buckets = buckets;
11907
- }
11908
- }
11909
11304
 
11910
11305
  /**
11911
11306
  * Thrown when an underlying database connection is closed.
@@ -11965,10 +11360,8 @@ class Schema {
11965
11360
  * developer instead of automatically by PowerSync.
11966
11361
  * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
11967
11362
  * using client-side table and column constraints.
11968
- * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
11969
11363
  *
11970
11364
  * @param tables An object of (table name, raw table definition) entries.
11971
- * @experimental Note that the raw tables API is still experimental and may change in the future.
11972
11365
  */
11973
11366
  withRawTables(tables) {
11974
11367
  for (const [name, rawTableDefinition] of Object.entries(tables)) {
@@ -12194,7 +11587,6 @@ exports.DEFAULT_INDEX_OPTIONS = DEFAULT_INDEX_OPTIONS;
12194
11587
  exports.DEFAULT_LOCK_TIMEOUT_MS = DEFAULT_LOCK_TIMEOUT_MS;
12195
11588
  exports.DEFAULT_POWERSYNC_CLOSE_OPTIONS = DEFAULT_POWERSYNC_CLOSE_OPTIONS;
12196
11589
  exports.DEFAULT_POWERSYNC_DB_OPTIONS = DEFAULT_POWERSYNC_DB_OPTIONS;
12197
- exports.DEFAULT_PRESSURE_LIMITS = DEFAULT_PRESSURE_LIMITS;
12198
11590
  exports.DEFAULT_REMOTE_LOGGER = DEFAULT_REMOTE_LOGGER;
12199
11591
  exports.DEFAULT_REMOTE_OPTIONS = DEFAULT_REMOTE_OPTIONS;
12200
11592
  exports.DEFAULT_RETRY_DELAY_MS = DEFAULT_RETRY_DELAY_MS;
@@ -12205,7 +11597,6 @@ exports.DEFAULT_SYNC_CLIENT_IMPLEMENTATION = DEFAULT_SYNC_CLIENT_IMPLEMENTATION;
12205
11597
  exports.DEFAULT_TABLE_OPTIONS = DEFAULT_TABLE_OPTIONS;
12206
11598
  exports.DEFAULT_WATCH_QUERY_OPTIONS = DEFAULT_WATCH_QUERY_OPTIONS;
12207
11599
  exports.DEFAULT_WATCH_THROTTLE_MS = DEFAULT_WATCH_THROTTLE_MS;
12208
- exports.DataStream = DataStream;
12209
11600
  exports.DifferentialQueryProcessor = DifferentialQueryProcessor;
12210
11601
  exports.EMPTY_DIFFERENTIAL = EMPTY_DIFFERENTIAL;
12211
11602
  exports.FalsyComparator = FalsyComparator;
@@ -12220,13 +11611,9 @@ exports.MAX_OP_ID = MAX_OP_ID;
12220
11611
  exports.MEMORY_TRIGGER_CLAIM_MANAGER = MEMORY_TRIGGER_CLAIM_MANAGER;
12221
11612
  exports.Mutex = Mutex;
12222
11613
  exports.OnChangeQueryProcessor = OnChangeQueryProcessor;
12223
- exports.OpType = OpType;
12224
- exports.OplogEntry = OplogEntry;
12225
11614
  exports.Schema = Schema;
12226
11615
  exports.Semaphore = Semaphore;
12227
11616
  exports.SqliteBucketStorage = SqliteBucketStorage;
12228
- exports.SyncDataBatch = SyncDataBatch;
12229
- exports.SyncDataBucket = SyncDataBucket;
12230
11617
  exports.SyncProgress = SyncProgress;
12231
11618
  exports.SyncStatus = SyncStatus;
12232
11619
  exports.SyncingService = SyncingService;
@@ -12241,18 +11628,10 @@ exports.createBaseLogger = createBaseLogger;
12241
11628
  exports.createLogger = createLogger;
12242
11629
  exports.extractTableUpdates = extractTableUpdates;
12243
11630
  exports.isBatchedUpdateNotification = isBatchedUpdateNotification;
12244
- exports.isContinueCheckpointRequest = isContinueCheckpointRequest;
12245
11631
  exports.isDBAdapter = isDBAdapter;
12246
11632
  exports.isPowerSyncDatabaseOptionsWithSettings = isPowerSyncDatabaseOptionsWithSettings;
12247
11633
  exports.isSQLOpenFactory = isSQLOpenFactory;
12248
11634
  exports.isSQLOpenOptions = isSQLOpenOptions;
12249
- exports.isStreamingKeepalive = isStreamingKeepalive;
12250
- exports.isStreamingSyncCheckpoint = isStreamingSyncCheckpoint;
12251
- exports.isStreamingSyncCheckpointComplete = isStreamingSyncCheckpointComplete;
12252
- exports.isStreamingSyncCheckpointDiff = isStreamingSyncCheckpointDiff;
12253
- exports.isStreamingSyncCheckpointPartiallyComplete = isStreamingSyncCheckpointPartiallyComplete;
12254
- exports.isStreamingSyncData = isStreamingSyncData;
12255
- exports.isSyncNewCheckpointRequest = isSyncNewCheckpointRequest;
12256
11635
  exports.parseQuery = parseQuery;
12257
11636
  exports.runOnSchemaChange = runOnSchemaChange;
12258
11637
  exports.sanitizeSQL = sanitizeSQL;