@powersync/common 0.0.0-dev-20260414110516 → 0.0.0-dev-20260504100448

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/bundle.cjs +33 -665
  2. package/dist/bundle.cjs.map +1 -1
  3. package/dist/bundle.mjs +34 -654
  4. package/dist/bundle.mjs.map +1 -1
  5. package/dist/bundle.node.cjs +33 -665
  6. package/dist/bundle.node.cjs.map +1 -1
  7. package/dist/bundle.node.mjs +34 -654
  8. package/dist/bundle.node.mjs.map +1 -1
  9. package/dist/index.d.cts +22 -369
  10. package/legacy/sync_protocol.d.ts +103 -0
  11. package/lib/client/sync/bucket/BucketStorageAdapter.d.ts +1 -63
  12. package/lib/client/sync/bucket/BucketStorageAdapter.js.map +1 -1
  13. package/lib/client/sync/bucket/SqliteBucketStorage.d.ts +1 -28
  14. package/lib/client/sync/bucket/SqliteBucketStorage.js +0 -162
  15. package/lib/client/sync/bucket/SqliteBucketStorage.js.map +1 -1
  16. package/lib/client/sync/stream/AbstractRemote.d.ts +2 -12
  17. package/lib/client/sync/stream/AbstractRemote.js +3 -13
  18. package/lib/client/sync/stream/AbstractRemote.js.map +1 -1
  19. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +12 -35
  20. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +29 -337
  21. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js.map +1 -1
  22. package/lib/client/sync/stream/JsonValue.d.ts +7 -0
  23. package/lib/client/sync/stream/JsonValue.js +2 -0
  24. package/lib/client/sync/stream/JsonValue.js.map +1 -0
  25. package/lib/client/sync/stream/core-instruction.d.ts +1 -2
  26. package/lib/client/sync/stream/core-instruction.js.map +1 -1
  27. package/lib/db/crud/SyncStatus.d.ts +0 -4
  28. package/lib/db/crud/SyncStatus.js +0 -4
  29. package/lib/db/crud/SyncStatus.js.map +1 -1
  30. package/lib/db/schema/RawTable.d.ts +0 -5
  31. package/lib/db/schema/Schema.d.ts +0 -2
  32. package/lib/db/schema/Schema.js +0 -2
  33. package/lib/db/schema/Schema.js.map +1 -1
  34. package/lib/index.d.ts +1 -5
  35. package/lib/index.js +1 -5
  36. package/lib/index.js.map +1 -1
  37. package/package.json +7 -4
  38. package/src/client/sync/bucket/BucketStorageAdapter.ts +1 -70
  39. package/src/client/sync/bucket/SqliteBucketStorage.ts +1 -197
  40. package/src/client/sync/stream/AbstractRemote.ts +5 -27
  41. package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +41 -407
  42. package/src/client/sync/stream/JsonValue.ts +8 -0
  43. package/src/client/sync/stream/core-instruction.ts +1 -2
  44. package/src/db/crud/SyncStatus.ts +0 -4
  45. package/src/db/schema/RawTable.ts +0 -5
  46. package/src/db/schema/Schema.ts +0 -2
  47. package/src/index.ts +1 -5
  48. package/lib/client/sync/bucket/OpType.d.ts +0 -16
  49. package/lib/client/sync/bucket/OpType.js +0 -23
  50. package/lib/client/sync/bucket/OpType.js.map +0 -1
  51. package/lib/client/sync/bucket/OplogEntry.d.ts +0 -23
  52. package/lib/client/sync/bucket/OplogEntry.js +0 -36
  53. package/lib/client/sync/bucket/OplogEntry.js.map +0 -1
  54. package/lib/client/sync/bucket/SyncDataBatch.d.ts +0 -6
  55. package/lib/client/sync/bucket/SyncDataBatch.js +0 -12
  56. package/lib/client/sync/bucket/SyncDataBatch.js.map +0 -1
  57. package/lib/client/sync/bucket/SyncDataBucket.d.ts +0 -40
  58. package/lib/client/sync/bucket/SyncDataBucket.js +0 -40
  59. package/lib/client/sync/bucket/SyncDataBucket.js.map +0 -1
  60. package/lib/client/sync/stream/streaming-sync-types.d.ts +0 -143
  61. package/lib/client/sync/stream/streaming-sync-types.js +0 -26
  62. package/lib/client/sync/stream/streaming-sync-types.js.map +0 -1
  63. package/src/client/sync/bucket/OpType.ts +0 -23
  64. package/src/client/sync/bucket/OplogEntry.ts +0 -50
  65. package/src/client/sync/bucket/SyncDataBatch.ts +0 -11
  66. package/src/client/sync/bucket/SyncDataBucket.ts +0 -49
  67. package/src/client/sync/stream/streaming-sync-types.ts +0 -210
package/dist/bundle.cjs CHANGED
@@ -2131,16 +2131,12 @@ class SyncStatus {
2131
2131
  *
2132
2132
  * This returns null when the database is currently being opened and we don't have reliable information about all
2133
2133
  * included streams yet.
2134
- *
2135
- * @experimental Sync streams are currently in alpha.
2136
2134
  */
2137
2135
  get syncStreams() {
2138
2136
  return this.options.dataFlow?.internalStreamSubscriptions?.map((core) => new SyncStreamStatusView(this, core));
2139
2137
  }
2140
2138
  /**
2141
2139
  * If the `stream` appears in {@link syncStreams}, returns the current status for that stream.
2142
- *
2143
- * @experimental Sync streams are currently in alpha.
2144
2140
  */
2145
2141
  forStream(stream) {
2146
2142
  const asJson = JSON.stringify(stream.parameters);
@@ -3493,103 +3489,6 @@ class AbortOperation extends Error {
3493
3489
  }
3494
3490
  }
3495
3491
 
3496
- exports.OpTypeEnum = void 0;
3497
- (function (OpTypeEnum) {
3498
- OpTypeEnum[OpTypeEnum["CLEAR"] = 1] = "CLEAR";
3499
- OpTypeEnum[OpTypeEnum["MOVE"] = 2] = "MOVE";
3500
- OpTypeEnum[OpTypeEnum["PUT"] = 3] = "PUT";
3501
- OpTypeEnum[OpTypeEnum["REMOVE"] = 4] = "REMOVE";
3502
- })(exports.OpTypeEnum || (exports.OpTypeEnum = {}));
3503
- /**
3504
- * Used internally for sync buckets.
3505
- */
3506
- class OpType {
3507
- value;
3508
- static fromJSON(jsonValue) {
3509
- return new OpType(exports.OpTypeEnum[jsonValue]);
3510
- }
3511
- constructor(value) {
3512
- this.value = value;
3513
- }
3514
- toJSON() {
3515
- return Object.entries(exports.OpTypeEnum).find(([, value]) => value === this.value)[0];
3516
- }
3517
- }
3518
-
3519
- class OplogEntry {
3520
- op_id;
3521
- op;
3522
- checksum;
3523
- subkey;
3524
- object_type;
3525
- object_id;
3526
- data;
3527
- static fromRow(row) {
3528
- return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, row.subkey, row.object_type, row.object_id, row.data);
3529
- }
3530
- constructor(op_id, op, checksum, subkey, object_type, object_id, data) {
3531
- this.op_id = op_id;
3532
- this.op = op;
3533
- this.checksum = checksum;
3534
- this.subkey = subkey;
3535
- this.object_type = object_type;
3536
- this.object_id = object_id;
3537
- this.data = data;
3538
- }
3539
- toJSON(fixedKeyEncoding = false) {
3540
- return {
3541
- op_id: this.op_id,
3542
- op: this.op.toJSON(),
3543
- object_type: this.object_type,
3544
- object_id: this.object_id,
3545
- checksum: this.checksum,
3546
- data: this.data,
3547
- // Older versions of the JS SDK used to always JSON.stringify here. That has always been wrong,
3548
- // but we need to migrate gradually to not break existing databases.
3549
- subkey: fixedKeyEncoding ? this.subkey : JSON.stringify(this.subkey)
3550
- };
3551
- }
3552
- }
3553
-
3554
- class SyncDataBucket {
3555
- bucket;
3556
- data;
3557
- has_more;
3558
- after;
3559
- next_after;
3560
- static fromRow(row) {
3561
- return new SyncDataBucket(row.bucket, row.data.map((entry) => OplogEntry.fromRow(entry)), row.has_more ?? false, row.after, row.next_after);
3562
- }
3563
- constructor(bucket, data,
3564
- /**
3565
- * True if the response does not contain all the data for this bucket, and another request must be made.
3566
- */
3567
- has_more,
3568
- /**
3569
- * The `after` specified in the request.
3570
- */
3571
- after,
3572
- /**
3573
- * Use this for the next request.
3574
- */
3575
- next_after) {
3576
- this.bucket = bucket;
3577
- this.data = data;
3578
- this.has_more = has_more;
3579
- this.after = after;
3580
- this.next_after = next_after;
3581
- }
3582
- toJSON(fixedKeyEncoding = false) {
3583
- return {
3584
- bucket: this.bucket,
3585
- has_more: this.has_more,
3586
- after: this.after,
3587
- next_after: this.next_after,
3588
- data: this.data.map((entry) => entry.toJSON(fixedKeyEncoding))
3589
- };
3590
- }
3591
- }
3592
-
3593
3492
  var buffer = {};
3594
3493
 
3595
3494
  var base64Js = {};
@@ -10745,7 +10644,7 @@ function requireDist () {
10745
10644
 
10746
10645
  var distExports = requireDist();
10747
10646
 
10748
- var version = "1.51.0";
10647
+ var version = "1.52.0";
10749
10648
  var PACKAGE = {
10750
10649
  version: version};
10751
10650
 
@@ -11302,22 +11201,12 @@ class AbstractRemote {
11302
11201
  * Returns a data stream of sync line data, fetched via RSocket-over-WebSocket.
11303
11202
  *
11304
11203
  * The only mechanism to abort the returned stream is to use the abort signal in {@link SocketSyncStreamOptions}.
11305
- *
11306
- * @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
11307
- * (required for compatibility with older sync services).
11308
11204
  */
11309
- async socketStreamRaw(options, bson) {
11205
+ async socketStreamRaw(options) {
11310
11206
  const { path, fetchStrategy = exports.FetchStrategy.Buffered } = options;
11311
- const mimeType = bson == null ? 'application/json' : 'application/bson';
11207
+ const mimeType = 'application/json';
11312
11208
  function toBuffer(js) {
11313
- let contents;
11314
- if (bson != null) {
11315
- contents = bson.serialize(js);
11316
- }
11317
- else {
11318
- contents = JSON.stringify(js);
11319
- }
11320
- return bufferExports.Buffer.from(contents);
11209
+ return bufferExports.Buffer.from(JSON.stringify(js));
11321
11210
  }
11322
11211
  const syncQueueRequestSize = fetchStrategy == exports.FetchStrategy.Buffered ? 10 : 1;
11323
11212
  const request = await this.buildRequest(path);
@@ -11619,32 +11508,6 @@ function coreStatusToJs(status) {
11619
11508
  };
11620
11509
  }
11621
11510
 
11622
- function isStreamingSyncData(line) {
11623
- return line.data != null;
11624
- }
11625
- function isStreamingKeepalive(line) {
11626
- return line.token_expires_in != null;
11627
- }
11628
- function isStreamingSyncCheckpoint(line) {
11629
- return line.checkpoint != null;
11630
- }
11631
- function isStreamingSyncCheckpointComplete(line) {
11632
- return line.checkpoint_complete != null;
11633
- }
11634
- function isStreamingSyncCheckpointPartiallyComplete(line) {
11635
- return line.partial_checkpoint_complete != null;
11636
- }
11637
- function isStreamingSyncCheckpointDiff(line) {
11638
- return line.checkpoint_diff != null;
11639
- }
11640
- function isContinueCheckpointRequest(request) {
11641
- return (Array.isArray(request.buckets) &&
11642
- typeof request.checkpoint_token == 'string');
11643
- }
11644
- function isSyncNewCheckpointRequest(request) {
11645
- return typeof request.request_checkpoint == 'object';
11646
- }
11647
-
11648
11511
  exports.LockType = void 0;
11649
11512
  (function (LockType) {
11650
11513
  LockType["CRUD"] = "crud";
@@ -11657,35 +11520,21 @@ exports.SyncStreamConnectionMethod = void 0;
11657
11520
  })(exports.SyncStreamConnectionMethod || (exports.SyncStreamConnectionMethod = {}));
11658
11521
  exports.SyncClientImplementation = void 0;
11659
11522
  (function (SyncClientImplementation) {
11660
- /**
11661
- * Decodes and handles sync lines received from the sync service in JavaScript.
11662
- *
11663
- * This is the default option.
11664
- *
11665
- * @deprecated We recommend the {@link RUST} client implementation for all apps. If you have issues with
11666
- * the Rust client, please file an issue or reach out to us. The JavaScript client will be removed in a future
11667
- * version of the PowerSync SDK.
11668
- */
11669
- SyncClientImplementation["JAVASCRIPT"] = "js";
11670
11523
  /**
11671
11524
  * This implementation offloads the sync line decoding and handling into the PowerSync
11672
11525
  * core extension.
11673
11526
  *
11674
- * This option is more performant than the {@link JAVASCRIPT} client, enabled by default and the
11675
- * recommended client implementation for all apps.
11527
+ * This is the only option, as an older JavaScript client implementation has been removed from the SDK.
11676
11528
  *
11677
11529
  * ## Compatibility warning
11678
11530
  *
11679
11531
  * The Rust sync client stores sync data in a format that is slightly different than the one used
11680
- * by the old {@link JAVASCRIPT} implementation. When adopting the {@link RUST} client on existing
11681
- * databases, the PowerSync SDK will migrate the format automatically.
11682
- * Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
11683
- * the version introducing {@link RUST} as an option) also supports the new format, so you can switch
11684
- * back to {@link JAVASCRIPT} later.
11532
+ * by the old JavaScript client. When adopting the {@link RUST} client on existing databases, the PowerSync SDK will
11533
+ * migrate the format automatically.
11685
11534
  *
11686
- * __However__: Upgrading the SDK version, then adopting {@link RUST} as a sync client and later
11687
- * downgrading the SDK to an older version (necessarily using the JavaScript-based implementation then)
11688
- * can lead to sync issues.
11535
+ * SDK versions supporting both the JavaScript and the Rust client support both formats with the JavaScript client
11536
+ * implementaiton. However, downgrading to an SDK version that only supports the JavaScript client would not be
11537
+ * possible anymore. Problematic SDK versions have been released before 2025-06-09.
11689
11538
  */
11690
11539
  SyncClientImplementation["RUST"] = "rust";
11691
11540
  })(exports.SyncClientImplementation || (exports.SyncClientImplementation = {}));
@@ -11708,13 +11557,7 @@ const DEFAULT_STREAM_CONNECTION_OPTIONS = {
11708
11557
  serializedSchema: undefined,
11709
11558
  includeDefaultStreams: true
11710
11559
  };
11711
- // The priority we assume when we receive checkpoint lines where no priority is set.
11712
- // This is the default priority used by the sync service, but can be set to an arbitrary
11713
- // value since sync services without priorities also won't send partial sync completion
11714
- // messages.
11715
- const FALLBACK_PRIORITY = 3;
11716
11560
  class AbstractStreamingSyncImplementation extends BaseObserver {
11717
- _lastSyncedAt;
11718
11561
  options;
11719
11562
  abortController;
11720
11563
  // In rare cases, mostly for tests, uploads can be triggered without being properly connected.
@@ -11724,6 +11567,7 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
11724
11567
  streamingSyncPromise;
11725
11568
  logger;
11726
11569
  activeStreams;
11570
+ connectionMayHaveChanged = false;
11727
11571
  isUploadingCrud = false;
11728
11572
  notifyCompletedUploads;
11729
11573
  handleActiveStreamsChange;
@@ -11803,9 +11647,6 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
11803
11647
  this.crudUpdateListener = undefined;
11804
11648
  this.uploadAbortController?.abort();
11805
11649
  }
11806
- async hasCompletedSync() {
11807
- return this.options.adapter.hasCompletedSync();
11808
- }
11809
11650
  async getWriteCheckpoint() {
11810
11651
  const clientId = await this.options.adapter.getClientId();
11811
11652
  let path = `/write-checkpoint2.json?client_id=${clientId}`;
@@ -11887,7 +11728,7 @@ The next upload iteration will be delayed.`);
11887
11728
  });
11888
11729
  }
11889
11730
  }
11890
- this.uploadAbortController = null;
11731
+ this.uploadAbortController = undefined;
11891
11732
  }
11892
11733
  });
11893
11734
  }
@@ -12003,6 +11844,11 @@ The next upload iteration will be delayed.`);
12003
11844
  shouldDelayRetry = false;
12004
11845
  // A disconnect was requested, we should not delay since there is no explicit retry
12005
11846
  }
11847
+ else if (this.connectionMayHaveChanged && ex.message?.indexOf('No iteration is active') >= 0) {
11848
+ this.connectionMayHaveChanged = false;
11849
+ this.logger.info('Sync error after changed connection, retrying immediately');
11850
+ shouldDelayRetry = false;
11851
+ }
12006
11852
  else {
12007
11853
  this.logger.error(ex);
12008
11854
  }
@@ -12033,17 +11879,14 @@ The next upload iteration will be delayed.`);
12033
11879
  // Mark as disconnected if here
12034
11880
  this.updateSyncStatus({ connected: false, connecting: false });
12035
11881
  }
12036
- async collectLocalBucketState() {
12037
- const bucketEntries = await this.options.adapter.getBucketStates();
12038
- const req = bucketEntries.map((entry) => ({
12039
- name: entry.bucket,
12040
- after: entry.op_id
12041
- }));
12042
- const localDescriptions = new Map();
12043
- for (const entry of bucketEntries) {
12044
- localDescriptions.set(entry.bucket, null);
12045
- }
12046
- return [req, localDescriptions];
11882
+ markConnectionMayHaveChanged() {
11883
+ // By setting this field, we'll immediately retry if the next sync event causes an error triggered by us not having
11884
+ // an active sync iteration on the connection in use.
11885
+ this.connectionMayHaveChanged = true;
11886
+ // This triggers a `powersync_control` invocation if a sync iteration is currently active. This is a cheap call to
11887
+ // make when no subscriptions have actually changed, we're mainly interested in this immediately throwing if no
11888
+ // iteration is active. That allows us to reconnect ASAP, instead of having to wait for the next sync line.
11889
+ this.handleActiveStreamsChange?.();
12047
11890
  }
12048
11891
  /**
12049
11892
  * Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
@@ -12086,19 +11929,13 @@ The next upload iteration will be delayed.`);
12086
11929
  }
12087
11930
  const clientImplementation = resolvedOptions.clientImplementation;
12088
11931
  this.updateSyncStatus({ clientImplementation });
12089
- if (clientImplementation == exports.SyncClientImplementation.JAVASCRIPT) {
12090
- await this.legacyStreamingSyncIteration(signal, resolvedOptions);
12091
- return null;
12092
- }
12093
- else {
12094
- await this.requireKeyFormat(true);
12095
- return await this.rustSyncIteration(signal, resolvedOptions);
12096
- }
11932
+ await this.requireKeyFormat(true);
11933
+ return await this.rustSyncIteration(signal, resolvedOptions);
12097
11934
  }
12098
11935
  });
12099
11936
  }
12100
11937
  async receiveSyncLines(data) {
12101
- const { options, connection, bson } = data;
11938
+ const { options, connection } = data;
12102
11939
  const remote = this.options.remote;
12103
11940
  if (connection.connectionMethod == exports.SyncStreamConnectionMethod.HTTP) {
12104
11941
  return await remote.fetchStream(options);
@@ -12107,232 +11944,8 @@ The next upload iteration will be delayed.`);
12107
11944
  return await this.options.remote.socketStreamRaw({
12108
11945
  ...options,
12109
11946
  ...{ fetchStrategy: connection.fetchStrategy }
12110
- }, bson);
12111
- }
12112
- }
12113
- async legacyStreamingSyncIteration(signal, resolvedOptions) {
12114
- const rawTables = resolvedOptions.serializedSchema?.raw_tables;
12115
- if (rawTables != null && rawTables.length) {
12116
- this.logger.warn('Raw tables require the Rust-based sync client. The JS client will ignore them.');
12117
- }
12118
- if (this.activeStreams.length) {
12119
- this.logger.error('Sync streams require `clientImplementation: SyncClientImplementation.RUST` when connecting.');
12120
- }
12121
- this.logger.debug('Streaming sync iteration started');
12122
- this.options.adapter.startSession();
12123
- let [req, bucketMap] = await this.collectLocalBucketState();
12124
- let targetCheckpoint = null;
12125
- // A checkpoint that has been validated but not applied (e.g. due to pending local writes)
12126
- let pendingValidatedCheckpoint = null;
12127
- const clientId = await this.options.adapter.getClientId();
12128
- const usingFixedKeyFormat = await this.requireKeyFormat(false);
12129
- this.logger.debug('Requesting stream from server');
12130
- const syncOptions = {
12131
- path: '/sync/stream',
12132
- abortSignal: signal,
12133
- data: {
12134
- buckets: req,
12135
- include_checksum: true,
12136
- raw_data: true,
12137
- parameters: resolvedOptions.params,
12138
- app_metadata: resolvedOptions.appMetadata,
12139
- client_id: clientId
12140
- }
12141
- };
12142
- const bson = await this.options.remote.getBSON();
12143
- const source = await this.receiveSyncLines({
12144
- options: syncOptions,
12145
- connection: resolvedOptions,
12146
- bson
12147
- });
12148
- const stream = injectable(map(source, (line) => {
12149
- if (typeof line == 'string') {
12150
- return JSON.parse(line);
12151
- }
12152
- else {
12153
- return bson.deserialize(line);
12154
- }
12155
- }));
12156
- this.logger.debug('Stream established. Processing events');
12157
- this.notifyCompletedUploads = () => {
12158
- stream.inject({ crud_upload_completed: null });
12159
- };
12160
- while (true) {
12161
- const { value: line, done } = await stream.next();
12162
- if (done) {
12163
- // The stream has closed while waiting
12164
- return;
12165
- }
12166
- if ('crud_upload_completed' in line) {
12167
- if (pendingValidatedCheckpoint != null) {
12168
- const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
12169
- if (applied) {
12170
- pendingValidatedCheckpoint = null;
12171
- }
12172
- else if (endIteration) {
12173
- break;
12174
- }
12175
- }
12176
- continue;
12177
- }
12178
- // A connection is active and messages are being received
12179
- if (!this.syncStatus.connected) {
12180
- // There is a connection now
12181
- Promise.resolve().then(() => this.triggerCrudUpload());
12182
- this.updateSyncStatus({
12183
- connected: true
12184
- });
12185
- }
12186
- if (isStreamingSyncCheckpoint(line)) {
12187
- targetCheckpoint = line.checkpoint;
12188
- // New checkpoint - existing validated checkpoint is no longer valid
12189
- pendingValidatedCheckpoint = null;
12190
- const bucketsToDelete = new Set(bucketMap.keys());
12191
- const newBuckets = new Map();
12192
- for (const checksum of line.checkpoint.buckets) {
12193
- newBuckets.set(checksum.bucket, {
12194
- name: checksum.bucket,
12195
- priority: checksum.priority ?? FALLBACK_PRIORITY
12196
- });
12197
- bucketsToDelete.delete(checksum.bucket);
12198
- }
12199
- if (bucketsToDelete.size > 0) {
12200
- this.logger.debug('Removing buckets', [...bucketsToDelete]);
12201
- }
12202
- bucketMap = newBuckets;
12203
- await this.options.adapter.removeBuckets([...bucketsToDelete]);
12204
- await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
12205
- await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
12206
- }
12207
- else if (isStreamingSyncCheckpointComplete(line)) {
12208
- const result = await this.applyCheckpoint(targetCheckpoint);
12209
- if (result.endIteration) {
12210
- return;
12211
- }
12212
- else if (!result.applied) {
12213
- // "Could not apply checkpoint due to local data". We need to retry after
12214
- // finishing uploads.
12215
- pendingValidatedCheckpoint = targetCheckpoint;
12216
- }
12217
- else {
12218
- // Nothing to retry later. This would likely already be null from the last
12219
- // checksum or checksum_diff operation, but we make sure.
12220
- pendingValidatedCheckpoint = null;
12221
- }
12222
- }
12223
- else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
12224
- const priority = line.partial_checkpoint_complete.priority;
12225
- this.logger.debug('Partial checkpoint complete', priority);
12226
- const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint, priority);
12227
- if (!result.checkpointValid) {
12228
- // This means checksums failed. Start again with a new checkpoint.
12229
- // TODO: better back-off
12230
- await new Promise((resolve) => setTimeout(resolve, 50));
12231
- return;
12232
- }
12233
- else if (!result.ready) ;
12234
- else {
12235
- // We'll keep on downloading, but can report that this priority is synced now.
12236
- this.logger.debug('partial checkpoint validation succeeded');
12237
- // All states with a higher priority can be deleted since this partial sync includes them.
12238
- const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
12239
- priorityStates.push({
12240
- priority,
12241
- lastSyncedAt: new Date(),
12242
- hasSynced: true
12243
- });
12244
- this.updateSyncStatus({
12245
- connected: true,
12246
- priorityStatusEntries: priorityStates
12247
- });
12248
- }
12249
- }
12250
- else if (isStreamingSyncCheckpointDiff(line)) {
12251
- // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
12252
- if (targetCheckpoint == null) {
12253
- throw new Error('Checkpoint diff without previous checkpoint');
12254
- }
12255
- // New checkpoint - existing validated checkpoint is no longer valid
12256
- pendingValidatedCheckpoint = null;
12257
- const diff = line.checkpoint_diff;
12258
- const newBuckets = new Map();
12259
- for (const checksum of targetCheckpoint.buckets) {
12260
- newBuckets.set(checksum.bucket, checksum);
12261
- }
12262
- for (const checksum of diff.updated_buckets) {
12263
- newBuckets.set(checksum.bucket, checksum);
12264
- }
12265
- for (const bucket of diff.removed_buckets) {
12266
- newBuckets.delete(bucket);
12267
- }
12268
- const newCheckpoint = {
12269
- last_op_id: diff.last_op_id,
12270
- buckets: [...newBuckets.values()],
12271
- write_checkpoint: diff.write_checkpoint
12272
- };
12273
- targetCheckpoint = newCheckpoint;
12274
- await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
12275
- bucketMap = new Map();
12276
- newBuckets.forEach((checksum, name) => bucketMap.set(name, {
12277
- name: checksum.bucket,
12278
- priority: checksum.priority ?? FALLBACK_PRIORITY
12279
- }));
12280
- const bucketsToDelete = diff.removed_buckets;
12281
- if (bucketsToDelete.length > 0) {
12282
- this.logger.debug('Remove buckets', bucketsToDelete);
12283
- }
12284
- await this.options.adapter.removeBuckets(bucketsToDelete);
12285
- await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
12286
- }
12287
- else if (isStreamingSyncData(line)) {
12288
- const { data } = line;
12289
- const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
12290
- let updatedProgress = null;
12291
- if (previousProgress) {
12292
- updatedProgress = { ...previousProgress };
12293
- const progressForBucket = updatedProgress[data.bucket];
12294
- if (progressForBucket) {
12295
- updatedProgress[data.bucket] = {
12296
- ...progressForBucket,
12297
- since_last: progressForBucket.since_last + data.data.length
12298
- };
12299
- }
12300
- }
12301
- this.updateSyncStatus({
12302
- dataFlow: {
12303
- downloading: true,
12304
- downloadProgress: updatedProgress
12305
- }
12306
- });
12307
- await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] }, usingFixedKeyFormat);
12308
- }
12309
- else if (isStreamingKeepalive(line)) {
12310
- const remaining_seconds = line.token_expires_in;
12311
- if (remaining_seconds == 0) {
12312
- // Connection would be closed automatically right after this
12313
- this.logger.debug('Token expiring; reconnect');
12314
- /**
12315
- * For a rare case where the backend connector does not update the token
12316
- * (uses the same one), this should have some delay.
12317
- */
12318
- await this.delayRetry();
12319
- return;
12320
- }
12321
- else if (remaining_seconds < 30) {
12322
- this.logger.debug('Token will expire soon; reconnect');
12323
- // Pre-emptively refresh the token
12324
- this.options.remote.invalidateCredentials();
12325
- return;
12326
- }
12327
- this.triggerCrudUpload();
12328
- }
12329
- else {
12330
- this.logger.debug('Received unknown sync line', line);
12331
- }
11947
+ });
12332
11948
  }
12333
- this.logger.debug('Stream input empty');
12334
- // Connection closed. Likely due to auth issue.
12335
- return;
12336
11949
  }
12337
11950
  async rustSyncIteration(signal, resolvedOptions) {
12338
11951
  const syncImplementation = this;
@@ -12405,6 +12018,10 @@ The next upload iteration will be delayed.`);
12405
12018
  const rawResponse = await adapter.control(op, payload ?? null);
12406
12019
  const logger = syncImplementation.logger;
12407
12020
  logger.trace('powersync_control', op, payload == null || typeof payload == 'string' ? payload : '<bytes>', rawResponse);
12021
+ if (op != exports.PowerSyncControlCommand.STOP) {
12022
+ // Evidently we have a working connection here, otherwise powersync_control would have failed.
12023
+ syncImplementation.connectionMayHaveChanged = false;
12024
+ }
12408
12025
  await handleInstructions(JSON.parse(rawResponse));
12409
12026
  }
12410
12027
  async function handleInstruction(instruction) {
@@ -12491,68 +12108,6 @@ The next upload iteration will be delayed.`);
12491
12108
  }
12492
12109
  return { immediateRestart: hideDisconnectOnRestart };
12493
12110
  }
12494
- async updateSyncStatusForStartingCheckpoint(checkpoint) {
12495
- const localProgress = await this.options.adapter.getBucketOperationProgress();
12496
- const progress = {};
12497
- let invalidated = false;
12498
- for (const bucket of checkpoint.buckets) {
12499
- const savedProgress = localProgress[bucket.bucket];
12500
- const atLast = savedProgress?.atLast ?? 0;
12501
- const sinceLast = savedProgress?.sinceLast ?? 0;
12502
- progress[bucket.bucket] = {
12503
- // The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
12504
- // will use by default.
12505
- priority: bucket.priority ?? 3,
12506
- at_last: atLast,
12507
- since_last: sinceLast,
12508
- target_count: bucket.count ?? 0
12509
- };
12510
- if (bucket.count != null && bucket.count < atLast + sinceLast) {
12511
- // Either due to a defrag / sync rule deploy or a compaction operation, the size
12512
- // of the bucket shrank so much that the local ops exceed the ops in the updated
12513
- // bucket. We can't prossibly report progress in this case (it would overshoot 100%).
12514
- invalidated = true;
12515
- }
12516
- }
12517
- if (invalidated) {
12518
- for (const bucket in progress) {
12519
- const bucketProgress = progress[bucket];
12520
- bucketProgress.at_last = 0;
12521
- bucketProgress.since_last = 0;
12522
- }
12523
- }
12524
- this.updateSyncStatus({
12525
- dataFlow: {
12526
- downloading: true,
12527
- downloadProgress: progress
12528
- }
12529
- });
12530
- }
12531
- async applyCheckpoint(checkpoint) {
12532
- let result = await this.options.adapter.syncLocalDatabase(checkpoint);
12533
- if (!result.checkpointValid) {
12534
- this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
12535
- // This means checksums failed. Start again with a new checkpoint.
12536
- // TODO: better back-off
12537
- await new Promise((resolve) => setTimeout(resolve, 50));
12538
- return { applied: false, endIteration: true };
12539
- }
12540
- else if (!result.ready) {
12541
- this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`);
12542
- return { applied: false, endIteration: false };
12543
- }
12544
- this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
12545
- this.updateSyncStatus({
12546
- connected: true,
12547
- lastSyncedAt: new Date(),
12548
- dataFlow: {
12549
- downloading: false,
12550
- downloadProgress: null,
12551
- downloadError: undefined
12552
- }
12553
- });
12554
- return { applied: true, endIteration: false };
12555
- }
12556
12111
  updateSyncStatus(options) {
12557
12112
  const updatedStatus = new SyncStatus({
12558
12113
  connected: options.connected ?? this.syncStatus.connected,
@@ -14091,14 +13646,12 @@ class SqliteBucketStorage extends BaseObserver {
14091
13646
  db;
14092
13647
  logger;
14093
13648
  tableNames;
14094
- _hasCompletedSync;
14095
13649
  updateListener;
14096
13650
  _clientId;
14097
13651
  constructor(db, logger = Logger.get('SqliteBucketStorage')) {
14098
13652
  super();
14099
13653
  this.db = db;
14100
13654
  this.logger = logger;
14101
- this._hasCompletedSync = false;
14102
13655
  this.tableNames = new Set();
14103
13656
  this.updateListener = db.registerListener({
14104
13657
  tablesUpdated: (update) => {
@@ -14110,7 +13663,6 @@ class SqliteBucketStorage extends BaseObserver {
14110
13663
  });
14111
13664
  }
14112
13665
  async init() {
14113
- this._hasCompletedSync = false;
14114
13666
  const existingTableRows = await this.db.getAll(`SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'`);
14115
13667
  for (const row of existingTableRows ?? []) {
14116
13668
  this.tableNames.add(row.name);
@@ -14132,156 +13684,6 @@ class SqliteBucketStorage extends BaseObserver {
14132
13684
  getMaxOpId() {
14133
13685
  return MAX_OP_ID;
14134
13686
  }
14135
- /**
14136
- * Reset any caches.
14137
- */
14138
- startSession() { }
14139
- async getBucketStates() {
14140
- const result = await this.db.getAll("SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != '$local'");
14141
- return result;
14142
- }
14143
- async getBucketOperationProgress() {
14144
- const rows = await this.db.getAll('SELECT name, count_at_last, count_since_last FROM ps_buckets');
14145
- return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
14146
- }
14147
- async saveSyncData(batch, fixedKeyFormat = false) {
14148
- await this.writeTransaction(async (tx) => {
14149
- for (const b of batch.buckets) {
14150
- await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
14151
- 'save',
14152
- JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
14153
- ]);
14154
- this.logger.debug(`Saved batch of data for bucket: ${b.bucket}, operations: ${b.data.length}`);
14155
- }
14156
- });
14157
- }
14158
- async removeBuckets(buckets) {
14159
- for (const bucket of buckets) {
14160
- await this.deleteBucket(bucket);
14161
- }
14162
- }
14163
- /**
14164
- * Mark a bucket for deletion.
14165
- */
14166
- async deleteBucket(bucket) {
14167
- await this.writeTransaction(async (tx) => {
14168
- await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
14169
- });
14170
- this.logger.debug(`Done deleting bucket ${bucket}`);
14171
- }
14172
- async hasCompletedSync() {
14173
- if (this._hasCompletedSync) {
14174
- return true;
14175
- }
14176
- const r = await this.db.get(`SELECT powersync_last_synced_at() as synced_at`);
14177
- const completed = r.synced_at != null;
14178
- if (completed) {
14179
- this._hasCompletedSync = true;
14180
- }
14181
- return completed;
14182
- }
14183
- async syncLocalDatabase(checkpoint, priority) {
14184
- const r = await this.validateChecksums(checkpoint, priority);
14185
- if (!r.checkpointValid) {
14186
- this.logger.error('Checksums failed for', r.checkpointFailures);
14187
- for (const b of r.checkpointFailures ?? []) {
14188
- await this.deleteBucket(b);
14189
- }
14190
- return { ready: false, checkpointValid: false, checkpointFailures: r.checkpointFailures };
14191
- }
14192
- if (priority == null) {
14193
- this.logger.debug(`Validated checksums checkpoint ${checkpoint.last_op_id}`);
14194
- }
14195
- else {
14196
- this.logger.debug(`Validated checksums for partial checkpoint ${checkpoint.last_op_id}, priority ${priority}`);
14197
- }
14198
- let buckets = checkpoint.buckets;
14199
- if (priority !== undefined) {
14200
- buckets = buckets.filter((b) => hasMatchingPriority(priority, b));
14201
- }
14202
- const bucketNames = buckets.map((b) => b.bucket);
14203
- await this.writeTransaction(async (tx) => {
14204
- await tx.execute(`UPDATE ps_buckets SET last_op = ? WHERE name IN (SELECT json_each.value FROM json_each(?))`, [
14205
- checkpoint.last_op_id,
14206
- JSON.stringify(bucketNames)
14207
- ]);
14208
- if (priority == null && checkpoint.write_checkpoint) {
14209
- await tx.execute("UPDATE ps_buckets SET last_op = ? WHERE name = '$local'", [checkpoint.write_checkpoint]);
14210
- }
14211
- });
14212
- const valid = await this.updateObjectsFromBuckets(checkpoint, priority);
14213
- if (!valid) {
14214
- return { ready: false, checkpointValid: true };
14215
- }
14216
- return {
14217
- ready: true,
14218
- checkpointValid: true
14219
- };
14220
- }
14221
- /**
14222
- * Atomically update the local state to the current checkpoint.
14223
- *
14224
- * This includes creating new tables, dropping old tables, and copying data over from the oplog.
14225
- */
14226
- async updateObjectsFromBuckets(checkpoint, priority) {
14227
- let arg = '';
14228
- if (priority !== undefined) {
14229
- const affectedBuckets = [];
14230
- for (const desc of checkpoint.buckets) {
14231
- if (hasMatchingPriority(priority, desc)) {
14232
- affectedBuckets.push(desc.bucket);
14233
- }
14234
- }
14235
- arg = JSON.stringify({ priority, buckets: affectedBuckets });
14236
- }
14237
- return this.writeTransaction(async (tx) => {
14238
- const { insertId: result } = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
14239
- 'sync_local',
14240
- arg
14241
- ]);
14242
- if (result == 1) {
14243
- if (priority == null) {
14244
- const bucketToCount = Object.fromEntries(checkpoint.buckets.map((b) => [b.bucket, b.count]));
14245
- // The two parameters could be replaced with one, but: https://github.com/powersync-ja/better-sqlite3/pull/6
14246
- const jsonBucketCount = JSON.stringify(bucketToCount);
14247
- await tx.execute("UPDATE ps_buckets SET count_since_last = 0, count_at_last = ?->name WHERE name != '$local' AND ?->name IS NOT NULL", [jsonBucketCount, jsonBucketCount]);
14248
- }
14249
- return true;
14250
- }
14251
- else {
14252
- return false;
14253
- }
14254
- });
14255
- }
14256
- async validateChecksums(checkpoint, priority) {
14257
- if (priority !== undefined) {
14258
- // Only validate the buckets within the priority we care about
14259
- const newBuckets = checkpoint.buckets.filter((cs) => hasMatchingPriority(priority, cs));
14260
- checkpoint = { ...checkpoint, buckets: newBuckets };
14261
- }
14262
- const rs = await this.db.execute('SELECT powersync_validate_checkpoint(?) as result', [
14263
- JSON.stringify({ ...checkpoint })
14264
- ]);
14265
- const resultItem = rs.rows?.item(0);
14266
- if (!resultItem) {
14267
- return {
14268
- checkpointValid: false,
14269
- ready: false,
14270
- checkpointFailures: []
14271
- };
14272
- }
14273
- const result = JSON.parse(resultItem['result']);
14274
- if (result['valid']) {
14275
- return { ready: true, checkpointValid: true };
14276
- }
14277
- else {
14278
- return {
14279
- checkpointValid: false,
14280
- ready: false,
14281
- checkpointFailures: result['failed_buckets']
14282
- };
14283
- }
14284
- }
14285
13687
  async updateLocalTarget(cb) {
14286
13688
  const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", [MAX_OP_ID]);
14287
13689
  if (!rs1.length) {
@@ -14372,12 +13774,6 @@ class SqliteBucketStorage extends BaseObserver {
14372
13774
  async writeTransaction(callback, options) {
14373
13775
  return this.db.writeTransaction(callback, options);
14374
13776
  }
14375
- /**
14376
- * Set a target checkpoint.
14377
- */
14378
- async setTargetCheckpoint(checkpoint) {
14379
- // No-op for now
14380
- }
14381
13777
  async control(op, payload) {
14382
13778
  return await this.writeTransaction(async (tx) => {
14383
13779
  const [[raw]] = await tx.executeRaw('SELECT powersync_control(?, ?)', [op, payload]);
@@ -14401,20 +13797,6 @@ class SqliteBucketStorage extends BaseObserver {
14401
13797
  }
14402
13798
  static _subkeyMigrationKey = 'powersync_js_migrated_subkeys';
14403
13799
  }
14404
- function hasMatchingPriority(priority, bucket) {
14405
- return bucket.priority != null && bucket.priority <= priority;
14406
- }
14407
-
14408
- // TODO JSON
14409
- class SyncDataBatch {
14410
- buckets;
14411
- static fromJSON(json) {
14412
- return new SyncDataBatch(json.buckets.map((bucket) => SyncDataBucket.fromRow(bucket)));
14413
- }
14414
- constructor(buckets) {
14415
- this.buckets = buckets;
14416
- }
14417
- }
14418
13800
 
14419
13801
  /**
14420
13802
  * Thrown when an underlying database connection is closed.
@@ -14474,10 +13856,8 @@ class Schema {
14474
13856
  * developer instead of automatically by PowerSync.
14475
13857
  * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
14476
13858
  * using client-side table and column constraints.
14477
- * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
14478
13859
  *
14479
13860
  * @param tables An object of (table name, raw table definition) entries.
14480
- * @experimental Note that the raw tables API is still experimental and may change in the future.
14481
13861
  */
14482
13862
  withRawTables(tables) {
14483
13863
  for (const [name, rawTableDefinition] of Object.entries(tables)) {
@@ -14727,13 +14107,9 @@ exports.MAX_OP_ID = MAX_OP_ID;
14727
14107
  exports.MEMORY_TRIGGER_CLAIM_MANAGER = MEMORY_TRIGGER_CLAIM_MANAGER;
14728
14108
  exports.Mutex = Mutex;
14729
14109
  exports.OnChangeQueryProcessor = OnChangeQueryProcessor;
14730
- exports.OpType = OpType;
14731
- exports.OplogEntry = OplogEntry;
14732
14110
  exports.Schema = Schema;
14733
14111
  exports.Semaphore = Semaphore;
14734
14112
  exports.SqliteBucketStorage = SqliteBucketStorage;
14735
- exports.SyncDataBatch = SyncDataBatch;
14736
- exports.SyncDataBucket = SyncDataBucket;
14737
14113
  exports.SyncProgress = SyncProgress;
14738
14114
  exports.SyncStatus = SyncStatus;
14739
14115
  exports.SyncingService = SyncingService;
@@ -14748,18 +14124,10 @@ exports.createBaseLogger = createBaseLogger;
14748
14124
  exports.createLogger = createLogger;
14749
14125
  exports.extractTableUpdates = extractTableUpdates;
14750
14126
  exports.isBatchedUpdateNotification = isBatchedUpdateNotification;
14751
- exports.isContinueCheckpointRequest = isContinueCheckpointRequest;
14752
14127
  exports.isDBAdapter = isDBAdapter;
14753
14128
  exports.isPowerSyncDatabaseOptionsWithSettings = isPowerSyncDatabaseOptionsWithSettings;
14754
14129
  exports.isSQLOpenFactory = isSQLOpenFactory;
14755
14130
  exports.isSQLOpenOptions = isSQLOpenOptions;
14756
- exports.isStreamingKeepalive = isStreamingKeepalive;
14757
- exports.isStreamingSyncCheckpoint = isStreamingSyncCheckpoint;
14758
- exports.isStreamingSyncCheckpointComplete = isStreamingSyncCheckpointComplete;
14759
- exports.isStreamingSyncCheckpointDiff = isStreamingSyncCheckpointDiff;
14760
- exports.isStreamingSyncCheckpointPartiallyComplete = isStreamingSyncCheckpointPartiallyComplete;
14761
- exports.isStreamingSyncData = isStreamingSyncData;
14762
- exports.isSyncNewCheckpointRequest = isSyncNewCheckpointRequest;
14763
14131
  exports.parseQuery = parseQuery;
14764
14132
  exports.runOnSchemaChange = runOnSchemaChange;
14765
14133
  exports.sanitizeSQL = sanitizeSQL;