@powersync/common 0.0.0-dev-20260503073249 → 0.0.0-dev-20260504100448

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/bundle.cjs +14 -659
  2. package/dist/bundle.cjs.map +1 -1
  3. package/dist/bundle.mjs +15 -648
  4. package/dist/bundle.mjs.map +1 -1
  5. package/dist/bundle.node.cjs +14 -659
  6. package/dist/bundle.node.cjs.map +1 -1
  7. package/dist/bundle.node.mjs +15 -648
  8. package/dist/bundle.node.mjs.map +1 -1
  9. package/dist/index.d.cts +19 -358
  10. package/legacy/sync_protocol.d.ts +103 -0
  11. package/lib/client/sync/bucket/BucketStorageAdapter.d.ts +1 -63
  12. package/lib/client/sync/bucket/BucketStorageAdapter.js.map +1 -1
  13. package/lib/client/sync/bucket/SqliteBucketStorage.d.ts +1 -28
  14. package/lib/client/sync/bucket/SqliteBucketStorage.js +0 -162
  15. package/lib/client/sync/bucket/SqliteBucketStorage.js.map +1 -1
  16. package/lib/client/sync/stream/AbstractRemote.d.ts +2 -12
  17. package/lib/client/sync/stream/AbstractRemote.js +3 -13
  18. package/lib/client/sync/stream/AbstractRemote.js.map +1 -1
  19. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +9 -35
  20. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +11 -338
  21. package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js.map +1 -1
  22. package/lib/client/sync/stream/JsonValue.d.ts +7 -0
  23. package/lib/client/sync/stream/JsonValue.js +2 -0
  24. package/lib/client/sync/stream/JsonValue.js.map +1 -0
  25. package/lib/client/sync/stream/core-instruction.d.ts +1 -2
  26. package/lib/client/sync/stream/core-instruction.js.map +1 -1
  27. package/lib/index.d.ts +1 -5
  28. package/lib/index.js +1 -5
  29. package/lib/index.js.map +1 -1
  30. package/package.json +7 -4
  31. package/src/client/sync/bucket/BucketStorageAdapter.ts +1 -70
  32. package/src/client/sync/bucket/SqliteBucketStorage.ts +1 -197
  33. package/src/client/sync/stream/AbstractRemote.ts +5 -27
  34. package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +22 -402
  35. package/src/client/sync/stream/JsonValue.ts +8 -0
  36. package/src/client/sync/stream/core-instruction.ts +1 -2
  37. package/src/index.ts +1 -5
  38. package/lib/client/sync/bucket/OpType.d.ts +0 -16
  39. package/lib/client/sync/bucket/OpType.js +0 -23
  40. package/lib/client/sync/bucket/OpType.js.map +0 -1
  41. package/lib/client/sync/bucket/OplogEntry.d.ts +0 -23
  42. package/lib/client/sync/bucket/OplogEntry.js +0 -36
  43. package/lib/client/sync/bucket/OplogEntry.js.map +0 -1
  44. package/lib/client/sync/bucket/SyncDataBatch.d.ts +0 -6
  45. package/lib/client/sync/bucket/SyncDataBatch.js +0 -12
  46. package/lib/client/sync/bucket/SyncDataBatch.js.map +0 -1
  47. package/lib/client/sync/bucket/SyncDataBucket.d.ts +0 -40
  48. package/lib/client/sync/bucket/SyncDataBucket.js +0 -40
  49. package/lib/client/sync/bucket/SyncDataBucket.js.map +0 -1
  50. package/lib/client/sync/stream/streaming-sync-types.d.ts +0 -143
  51. package/lib/client/sync/stream/streaming-sync-types.js +0 -26
  52. package/lib/client/sync/stream/streaming-sync-types.js.map +0 -1
  53. package/src/client/sync/bucket/OpType.ts +0 -23
  54. package/src/client/sync/bucket/OplogEntry.ts +0 -50
  55. package/src/client/sync/bucket/SyncDataBatch.ts +0 -11
  56. package/src/client/sync/bucket/SyncDataBucket.ts +0 -49
  57. package/src/client/sync/stream/streaming-sync-types.ts +0 -210
package/dist/bundle.cjs CHANGED
@@ -3489,103 +3489,6 @@ class AbortOperation extends Error {
3489
3489
  }
3490
3490
  }
3491
3491
 
3492
- exports.OpTypeEnum = void 0;
3493
- (function (OpTypeEnum) {
3494
- OpTypeEnum[OpTypeEnum["CLEAR"] = 1] = "CLEAR";
3495
- OpTypeEnum[OpTypeEnum["MOVE"] = 2] = "MOVE";
3496
- OpTypeEnum[OpTypeEnum["PUT"] = 3] = "PUT";
3497
- OpTypeEnum[OpTypeEnum["REMOVE"] = 4] = "REMOVE";
3498
- })(exports.OpTypeEnum || (exports.OpTypeEnum = {}));
3499
- /**
3500
- * Used internally for sync buckets.
3501
- */
3502
- class OpType {
3503
- value;
3504
- static fromJSON(jsonValue) {
3505
- return new OpType(exports.OpTypeEnum[jsonValue]);
3506
- }
3507
- constructor(value) {
3508
- this.value = value;
3509
- }
3510
- toJSON() {
3511
- return Object.entries(exports.OpTypeEnum).find(([, value]) => value === this.value)[0];
3512
- }
3513
- }
3514
-
3515
- class OplogEntry {
3516
- op_id;
3517
- op;
3518
- checksum;
3519
- subkey;
3520
- object_type;
3521
- object_id;
3522
- data;
3523
- static fromRow(row) {
3524
- return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, row.subkey, row.object_type, row.object_id, row.data);
3525
- }
3526
- constructor(op_id, op, checksum, subkey, object_type, object_id, data) {
3527
- this.op_id = op_id;
3528
- this.op = op;
3529
- this.checksum = checksum;
3530
- this.subkey = subkey;
3531
- this.object_type = object_type;
3532
- this.object_id = object_id;
3533
- this.data = data;
3534
- }
3535
- toJSON(fixedKeyEncoding = false) {
3536
- return {
3537
- op_id: this.op_id,
3538
- op: this.op.toJSON(),
3539
- object_type: this.object_type,
3540
- object_id: this.object_id,
3541
- checksum: this.checksum,
3542
- data: this.data,
3543
- // Older versions of the JS SDK used to always JSON.stringify here. That has always been wrong,
3544
- // but we need to migrate gradually to not break existing databases.
3545
- subkey: fixedKeyEncoding ? this.subkey : JSON.stringify(this.subkey)
3546
- };
3547
- }
3548
- }
3549
-
3550
- class SyncDataBucket {
3551
- bucket;
3552
- data;
3553
- has_more;
3554
- after;
3555
- next_after;
3556
- static fromRow(row) {
3557
- return new SyncDataBucket(row.bucket, row.data.map((entry) => OplogEntry.fromRow(entry)), row.has_more ?? false, row.after, row.next_after);
3558
- }
3559
- constructor(bucket, data,
3560
- /**
3561
- * True if the response does not contain all the data for this bucket, and another request must be made.
3562
- */
3563
- has_more,
3564
- /**
3565
- * The `after` specified in the request.
3566
- */
3567
- after,
3568
- /**
3569
- * Use this for the next request.
3570
- */
3571
- next_after) {
3572
- this.bucket = bucket;
3573
- this.data = data;
3574
- this.has_more = has_more;
3575
- this.after = after;
3576
- this.next_after = next_after;
3577
- }
3578
- toJSON(fixedKeyEncoding = false) {
3579
- return {
3580
- bucket: this.bucket,
3581
- has_more: this.has_more,
3582
- after: this.after,
3583
- next_after: this.next_after,
3584
- data: this.data.map((entry) => entry.toJSON(fixedKeyEncoding))
3585
- };
3586
- }
3587
- }
3588
-
3589
3492
  var buffer = {};
3590
3493
 
3591
3494
  var base64Js = {};
@@ -11298,22 +11201,12 @@ class AbstractRemote {
11298
11201
  * Returns a data stream of sync line data, fetched via RSocket-over-WebSocket.
11299
11202
  *
11300
11203
  * The only mechanism to abort the returned stream is to use the abort signal in {@link SocketSyncStreamOptions}.
11301
- *
11302
- * @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
11303
- * (required for compatibility with older sync services).
11304
11204
  */
11305
- async socketStreamRaw(options, bson) {
11205
+ async socketStreamRaw(options) {
11306
11206
  const { path, fetchStrategy = exports.FetchStrategy.Buffered } = options;
11307
- const mimeType = bson == null ? 'application/json' : 'application/bson';
11207
+ const mimeType = 'application/json';
11308
11208
  function toBuffer(js) {
11309
- let contents;
11310
- if (bson != null) {
11311
- contents = bson.serialize(js);
11312
- }
11313
- else {
11314
- contents = JSON.stringify(js);
11315
- }
11316
- return bufferExports.Buffer.from(contents);
11209
+ return bufferExports.Buffer.from(JSON.stringify(js));
11317
11210
  }
11318
11211
  const syncQueueRequestSize = fetchStrategy == exports.FetchStrategy.Buffered ? 10 : 1;
11319
11212
  const request = await this.buildRequest(path);
@@ -11615,32 +11508,6 @@ function coreStatusToJs(status) {
11615
11508
  };
11616
11509
  }
11617
11510
 
11618
- function isStreamingSyncData(line) {
11619
- return line.data != null;
11620
- }
11621
- function isStreamingKeepalive(line) {
11622
- return line.token_expires_in != null;
11623
- }
11624
- function isStreamingSyncCheckpoint(line) {
11625
- return line.checkpoint != null;
11626
- }
11627
- function isStreamingSyncCheckpointComplete(line) {
11628
- return line.checkpoint_complete != null;
11629
- }
11630
- function isStreamingSyncCheckpointPartiallyComplete(line) {
11631
- return line.partial_checkpoint_complete != null;
11632
- }
11633
- function isStreamingSyncCheckpointDiff(line) {
11634
- return line.checkpoint_diff != null;
11635
- }
11636
- function isContinueCheckpointRequest(request) {
11637
- return (Array.isArray(request.buckets) &&
11638
- typeof request.checkpoint_token == 'string');
11639
- }
11640
- function isSyncNewCheckpointRequest(request) {
11641
- return typeof request.request_checkpoint == 'object';
11642
- }
11643
-
11644
11511
  exports.LockType = void 0;
11645
11512
  (function (LockType) {
11646
11513
  LockType["CRUD"] = "crud";
@@ -11653,35 +11520,21 @@ exports.SyncStreamConnectionMethod = void 0;
11653
11520
  })(exports.SyncStreamConnectionMethod || (exports.SyncStreamConnectionMethod = {}));
11654
11521
  exports.SyncClientImplementation = void 0;
11655
11522
  (function (SyncClientImplementation) {
11656
- /**
11657
- * Decodes and handles sync lines received from the sync service in JavaScript.
11658
- *
11659
- * This is the default option.
11660
- *
11661
- * @deprecated We recommend the {@link RUST} client implementation for all apps. If you have issues with
11662
- * the Rust client, please file an issue or reach out to us. The JavaScript client will be removed in a future
11663
- * version of the PowerSync SDK.
11664
- */
11665
- SyncClientImplementation["JAVASCRIPT"] = "js";
11666
11523
  /**
11667
11524
  * This implementation offloads the sync line decoding and handling into the PowerSync
11668
11525
  * core extension.
11669
11526
  *
11670
- * This option is more performant than the {@link JAVASCRIPT} client, enabled by default and the
11671
- * recommended client implementation for all apps.
11527
+ * This is the only option, as an older JavaScript client implementation has been removed from the SDK.
11672
11528
  *
11673
11529
  * ## Compatibility warning
11674
11530
  *
11675
11531
  * The Rust sync client stores sync data in a format that is slightly different than the one used
11676
- * by the old {@link JAVASCRIPT} implementation. When adopting the {@link RUST} client on existing
11677
- * databases, the PowerSync SDK will migrate the format automatically.
11678
- * Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
11679
- * the version introducing {@link RUST} as an option) also supports the new format, so you can switch
11680
- * back to {@link JAVASCRIPT} later.
11532
+ * by the old JavaScript client. When adopting the {@link RUST} client on existing databases, the PowerSync SDK will
11533
+ * migrate the format automatically.
11681
11534
  *
11682
- * __However__: Upgrading the SDK version, then adopting {@link RUST} as a sync client and later
11683
- * downgrading the SDK to an older version (necessarily using the JavaScript-based implementation then)
11684
- * can lead to sync issues.
11535
+ * SDK versions supporting both the JavaScript and the Rust client support both formats with the JavaScript client
11536
+ * implementaiton. However, downgrading to an SDK version that only supports the JavaScript client would not be
11537
+ * possible anymore. Problematic SDK versions have been released before 2025-06-09.
11685
11538
  */
11686
11539
  SyncClientImplementation["RUST"] = "rust";
11687
11540
  })(exports.SyncClientImplementation || (exports.SyncClientImplementation = {}));
@@ -11704,13 +11557,7 @@ const DEFAULT_STREAM_CONNECTION_OPTIONS = {
11704
11557
  serializedSchema: undefined,
11705
11558
  includeDefaultStreams: true
11706
11559
  };
11707
- // The priority we assume when we receive checkpoint lines where no priority is set.
11708
- // This is the default priority used by the sync service, but can be set to an arbitrary
11709
- // value since sync services without priorities also won't send partial sync completion
11710
- // messages.
11711
- const FALLBACK_PRIORITY = 3;
11712
11560
  class AbstractStreamingSyncImplementation extends BaseObserver {
11713
- _lastSyncedAt;
11714
11561
  options;
11715
11562
  abortController;
11716
11563
  // In rare cases, mostly for tests, uploads can be triggered without being properly connected.
@@ -11800,9 +11647,6 @@ class AbstractStreamingSyncImplementation extends BaseObserver {
11800
11647
  this.crudUpdateListener = undefined;
11801
11648
  this.uploadAbortController?.abort();
11802
11649
  }
11803
- async hasCompletedSync() {
11804
- return this.options.adapter.hasCompletedSync();
11805
- }
11806
11650
  async getWriteCheckpoint() {
11807
11651
  const clientId = await this.options.adapter.getClientId();
11808
11652
  let path = `/write-checkpoint2.json?client_id=${clientId}`;
@@ -11884,7 +11728,7 @@ The next upload iteration will be delayed.`);
11884
11728
  });
11885
11729
  }
11886
11730
  }
11887
- this.uploadAbortController = null;
11731
+ this.uploadAbortController = undefined;
11888
11732
  }
11889
11733
  });
11890
11734
  }
@@ -12044,18 +11888,6 @@ The next upload iteration will be delayed.`);
12044
11888
  // iteration is active. That allows us to reconnect ASAP, instead of having to wait for the next sync line.
12045
11889
  this.handleActiveStreamsChange?.();
12046
11890
  }
12047
- async collectLocalBucketState() {
12048
- const bucketEntries = await this.options.adapter.getBucketStates();
12049
- const req = bucketEntries.map((entry) => ({
12050
- name: entry.bucket,
12051
- after: entry.op_id
12052
- }));
12053
- const localDescriptions = new Map();
12054
- for (const entry of bucketEntries) {
12055
- localDescriptions.set(entry.bucket, null);
12056
- }
12057
- return [req, localDescriptions];
12058
- }
12059
11891
  /**
12060
11892
  * Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
12061
11893
  * Because subkeys are always strings, this leads to quotes being added around them in `ps_oplog`.
@@ -12097,19 +11929,13 @@ The next upload iteration will be delayed.`);
12097
11929
  }
12098
11930
  const clientImplementation = resolvedOptions.clientImplementation;
12099
11931
  this.updateSyncStatus({ clientImplementation });
12100
- if (clientImplementation == exports.SyncClientImplementation.JAVASCRIPT) {
12101
- await this.legacyStreamingSyncIteration(signal, resolvedOptions);
12102
- return null;
12103
- }
12104
- else {
12105
- await this.requireKeyFormat(true);
12106
- return await this.rustSyncIteration(signal, resolvedOptions);
12107
- }
11932
+ await this.requireKeyFormat(true);
11933
+ return await this.rustSyncIteration(signal, resolvedOptions);
12108
11934
  }
12109
11935
  });
12110
11936
  }
12111
11937
  async receiveSyncLines(data) {
12112
- const { options, connection, bson } = data;
11938
+ const { options, connection } = data;
12113
11939
  const remote = this.options.remote;
12114
11940
  if (connection.connectionMethod == exports.SyncStreamConnectionMethod.HTTP) {
12115
11941
  return await remote.fetchStream(options);
@@ -12118,232 +11944,8 @@ The next upload iteration will be delayed.`);
12118
11944
  return await this.options.remote.socketStreamRaw({
12119
11945
  ...options,
12120
11946
  ...{ fetchStrategy: connection.fetchStrategy }
12121
- }, bson);
12122
- }
12123
- }
12124
- async legacyStreamingSyncIteration(signal, resolvedOptions) {
12125
- const rawTables = resolvedOptions.serializedSchema?.raw_tables;
12126
- if (rawTables != null && rawTables.length) {
12127
- this.logger.warn('Raw tables require the Rust-based sync client. The JS client will ignore them.');
12128
- }
12129
- if (this.activeStreams.length) {
12130
- this.logger.error('Sync streams require `clientImplementation: SyncClientImplementation.RUST` when connecting.');
12131
- }
12132
- this.logger.debug('Streaming sync iteration started');
12133
- this.options.adapter.startSession();
12134
- let [req, bucketMap] = await this.collectLocalBucketState();
12135
- let targetCheckpoint = null;
12136
- // A checkpoint that has been validated but not applied (e.g. due to pending local writes)
12137
- let pendingValidatedCheckpoint = null;
12138
- const clientId = await this.options.adapter.getClientId();
12139
- const usingFixedKeyFormat = await this.requireKeyFormat(false);
12140
- this.logger.debug('Requesting stream from server');
12141
- const syncOptions = {
12142
- path: '/sync/stream',
12143
- abortSignal: signal,
12144
- data: {
12145
- buckets: req,
12146
- include_checksum: true,
12147
- raw_data: true,
12148
- parameters: resolvedOptions.params,
12149
- app_metadata: resolvedOptions.appMetadata,
12150
- client_id: clientId
12151
- }
12152
- };
12153
- const bson = await this.options.remote.getBSON();
12154
- const source = await this.receiveSyncLines({
12155
- options: syncOptions,
12156
- connection: resolvedOptions,
12157
- bson
12158
- });
12159
- const stream = injectable(map(source, (line) => {
12160
- if (typeof line == 'string') {
12161
- return JSON.parse(line);
12162
- }
12163
- else {
12164
- return bson.deserialize(line);
12165
- }
12166
- }));
12167
- this.logger.debug('Stream established. Processing events');
12168
- this.notifyCompletedUploads = () => {
12169
- stream.inject({ crud_upload_completed: null });
12170
- };
12171
- while (true) {
12172
- const { value: line, done } = await stream.next();
12173
- if (done) {
12174
- // The stream has closed while waiting
12175
- return;
12176
- }
12177
- if ('crud_upload_completed' in line) {
12178
- if (pendingValidatedCheckpoint != null) {
12179
- const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
12180
- if (applied) {
12181
- pendingValidatedCheckpoint = null;
12182
- }
12183
- else if (endIteration) {
12184
- break;
12185
- }
12186
- }
12187
- continue;
12188
- }
12189
- // A connection is active and messages are being received
12190
- if (!this.syncStatus.connected) {
12191
- // There is a connection now
12192
- Promise.resolve().then(() => this.triggerCrudUpload());
12193
- this.updateSyncStatus({
12194
- connected: true
12195
- });
12196
- }
12197
- if (isStreamingSyncCheckpoint(line)) {
12198
- targetCheckpoint = line.checkpoint;
12199
- // New checkpoint - existing validated checkpoint is no longer valid
12200
- pendingValidatedCheckpoint = null;
12201
- const bucketsToDelete = new Set(bucketMap.keys());
12202
- const newBuckets = new Map();
12203
- for (const checksum of line.checkpoint.buckets) {
12204
- newBuckets.set(checksum.bucket, {
12205
- name: checksum.bucket,
12206
- priority: checksum.priority ?? FALLBACK_PRIORITY
12207
- });
12208
- bucketsToDelete.delete(checksum.bucket);
12209
- }
12210
- if (bucketsToDelete.size > 0) {
12211
- this.logger.debug('Removing buckets', [...bucketsToDelete]);
12212
- }
12213
- bucketMap = newBuckets;
12214
- await this.options.adapter.removeBuckets([...bucketsToDelete]);
12215
- await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
12216
- await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
12217
- }
12218
- else if (isStreamingSyncCheckpointComplete(line)) {
12219
- const result = await this.applyCheckpoint(targetCheckpoint);
12220
- if (result.endIteration) {
12221
- return;
12222
- }
12223
- else if (!result.applied) {
12224
- // "Could not apply checkpoint due to local data". We need to retry after
12225
- // finishing uploads.
12226
- pendingValidatedCheckpoint = targetCheckpoint;
12227
- }
12228
- else {
12229
- // Nothing to retry later. This would likely already be null from the last
12230
- // checksum or checksum_diff operation, but we make sure.
12231
- pendingValidatedCheckpoint = null;
12232
- }
12233
- }
12234
- else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
12235
- const priority = line.partial_checkpoint_complete.priority;
12236
- this.logger.debug('Partial checkpoint complete', priority);
12237
- const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint, priority);
12238
- if (!result.checkpointValid) {
12239
- // This means checksums failed. Start again with a new checkpoint.
12240
- // TODO: better back-off
12241
- await new Promise((resolve) => setTimeout(resolve, 50));
12242
- return;
12243
- }
12244
- else if (!result.ready) ;
12245
- else {
12246
- // We'll keep on downloading, but can report that this priority is synced now.
12247
- this.logger.debug('partial checkpoint validation succeeded');
12248
- // All states with a higher priority can be deleted since this partial sync includes them.
12249
- const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
12250
- priorityStates.push({
12251
- priority,
12252
- lastSyncedAt: new Date(),
12253
- hasSynced: true
12254
- });
12255
- this.updateSyncStatus({
12256
- connected: true,
12257
- priorityStatusEntries: priorityStates
12258
- });
12259
- }
12260
- }
12261
- else if (isStreamingSyncCheckpointDiff(line)) {
12262
- // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
12263
- if (targetCheckpoint == null) {
12264
- throw new Error('Checkpoint diff without previous checkpoint');
12265
- }
12266
- // New checkpoint - existing validated checkpoint is no longer valid
12267
- pendingValidatedCheckpoint = null;
12268
- const diff = line.checkpoint_diff;
12269
- const newBuckets = new Map();
12270
- for (const checksum of targetCheckpoint.buckets) {
12271
- newBuckets.set(checksum.bucket, checksum);
12272
- }
12273
- for (const checksum of diff.updated_buckets) {
12274
- newBuckets.set(checksum.bucket, checksum);
12275
- }
12276
- for (const bucket of diff.removed_buckets) {
12277
- newBuckets.delete(bucket);
12278
- }
12279
- const newCheckpoint = {
12280
- last_op_id: diff.last_op_id,
12281
- buckets: [...newBuckets.values()],
12282
- write_checkpoint: diff.write_checkpoint
12283
- };
12284
- targetCheckpoint = newCheckpoint;
12285
- await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
12286
- bucketMap = new Map();
12287
- newBuckets.forEach((checksum, name) => bucketMap.set(name, {
12288
- name: checksum.bucket,
12289
- priority: checksum.priority ?? FALLBACK_PRIORITY
12290
- }));
12291
- const bucketsToDelete = diff.removed_buckets;
12292
- if (bucketsToDelete.length > 0) {
12293
- this.logger.debug('Remove buckets', bucketsToDelete);
12294
- }
12295
- await this.options.adapter.removeBuckets(bucketsToDelete);
12296
- await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
12297
- }
12298
- else if (isStreamingSyncData(line)) {
12299
- const { data } = line;
12300
- const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
12301
- let updatedProgress = null;
12302
- if (previousProgress) {
12303
- updatedProgress = { ...previousProgress };
12304
- const progressForBucket = updatedProgress[data.bucket];
12305
- if (progressForBucket) {
12306
- updatedProgress[data.bucket] = {
12307
- ...progressForBucket,
12308
- since_last: progressForBucket.since_last + data.data.length
12309
- };
12310
- }
12311
- }
12312
- this.updateSyncStatus({
12313
- dataFlow: {
12314
- downloading: true,
12315
- downloadProgress: updatedProgress
12316
- }
12317
- });
12318
- await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] }, usingFixedKeyFormat);
12319
- }
12320
- else if (isStreamingKeepalive(line)) {
12321
- const remaining_seconds = line.token_expires_in;
12322
- if (remaining_seconds == 0) {
12323
- // Connection would be closed automatically right after this
12324
- this.logger.debug('Token expiring; reconnect');
12325
- /**
12326
- * For a rare case where the backend connector does not update the token
12327
- * (uses the same one), this should have some delay.
12328
- */
12329
- await this.delayRetry();
12330
- return;
12331
- }
12332
- else if (remaining_seconds < 30) {
12333
- this.logger.debug('Token will expire soon; reconnect');
12334
- // Pre-emptively refresh the token
12335
- this.options.remote.invalidateCredentials();
12336
- return;
12337
- }
12338
- this.triggerCrudUpload();
12339
- }
12340
- else {
12341
- this.logger.debug('Received unknown sync line', line);
12342
- }
11947
+ });
12343
11948
  }
12344
- this.logger.debug('Stream input empty');
12345
- // Connection closed. Likely due to auth issue.
12346
- return;
12347
11949
  }
12348
11950
  async rustSyncIteration(signal, resolvedOptions) {
12349
11951
  const syncImplementation = this;
@@ -12506,68 +12108,6 @@ The next upload iteration will be delayed.`);
12506
12108
  }
12507
12109
  return { immediateRestart: hideDisconnectOnRestart };
12508
12110
  }
12509
- async updateSyncStatusForStartingCheckpoint(checkpoint) {
12510
- const localProgress = await this.options.adapter.getBucketOperationProgress();
12511
- const progress = {};
12512
- let invalidated = false;
12513
- for (const bucket of checkpoint.buckets) {
12514
- const savedProgress = localProgress[bucket.bucket];
12515
- const atLast = savedProgress?.atLast ?? 0;
12516
- const sinceLast = savedProgress?.sinceLast ?? 0;
12517
- progress[bucket.bucket] = {
12518
- // The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
12519
- // will use by default.
12520
- priority: bucket.priority ?? 3,
12521
- at_last: atLast,
12522
- since_last: sinceLast,
12523
- target_count: bucket.count ?? 0
12524
- };
12525
- if (bucket.count != null && bucket.count < atLast + sinceLast) {
12526
- // Either due to a defrag / sync rule deploy or a compaction operation, the size
12527
- // of the bucket shrank so much that the local ops exceed the ops in the updated
12528
- // bucket. We can't prossibly report progress in this case (it would overshoot 100%).
12529
- invalidated = true;
12530
- }
12531
- }
12532
- if (invalidated) {
12533
- for (const bucket in progress) {
12534
- const bucketProgress = progress[bucket];
12535
- bucketProgress.at_last = 0;
12536
- bucketProgress.since_last = 0;
12537
- }
12538
- }
12539
- this.updateSyncStatus({
12540
- dataFlow: {
12541
- downloading: true,
12542
- downloadProgress: progress
12543
- }
12544
- });
12545
- }
12546
- async applyCheckpoint(checkpoint) {
12547
- let result = await this.options.adapter.syncLocalDatabase(checkpoint);
12548
- if (!result.checkpointValid) {
12549
- this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
12550
- // This means checksums failed. Start again with a new checkpoint.
12551
- // TODO: better back-off
12552
- await new Promise((resolve) => setTimeout(resolve, 50));
12553
- return { applied: false, endIteration: true };
12554
- }
12555
- else if (!result.ready) {
12556
- this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`);
12557
- return { applied: false, endIteration: false };
12558
- }
12559
- this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
12560
- this.updateSyncStatus({
12561
- connected: true,
12562
- lastSyncedAt: new Date(),
12563
- dataFlow: {
12564
- downloading: false,
12565
- downloadProgress: null,
12566
- downloadError: undefined
12567
- }
12568
- });
12569
- return { applied: true, endIteration: false };
12570
- }
12571
12111
  updateSyncStatus(options) {
12572
12112
  const updatedStatus = new SyncStatus({
12573
12113
  connected: options.connected ?? this.syncStatus.connected,
@@ -14106,14 +13646,12 @@ class SqliteBucketStorage extends BaseObserver {
14106
13646
  db;
14107
13647
  logger;
14108
13648
  tableNames;
14109
- _hasCompletedSync;
14110
13649
  updateListener;
14111
13650
  _clientId;
14112
13651
  constructor(db, logger = Logger.get('SqliteBucketStorage')) {
14113
13652
  super();
14114
13653
  this.db = db;
14115
13654
  this.logger = logger;
14116
- this._hasCompletedSync = false;
14117
13655
  this.tableNames = new Set();
14118
13656
  this.updateListener = db.registerListener({
14119
13657
  tablesUpdated: (update) => {
@@ -14125,7 +13663,6 @@ class SqliteBucketStorage extends BaseObserver {
14125
13663
  });
14126
13664
  }
14127
13665
  async init() {
14128
- this._hasCompletedSync = false;
14129
13666
  const existingTableRows = await this.db.getAll(`SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'`);
14130
13667
  for (const row of existingTableRows ?? []) {
14131
13668
  this.tableNames.add(row.name);
@@ -14147,156 +13684,6 @@ class SqliteBucketStorage extends BaseObserver {
14147
13684
  getMaxOpId() {
14148
13685
  return MAX_OP_ID;
14149
13686
  }
14150
- /**
14151
- * Reset any caches.
14152
- */
14153
- startSession() { }
14154
- async getBucketStates() {
14155
- const result = await this.db.getAll("SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != '$local'");
14156
- return result;
14157
- }
14158
- async getBucketOperationProgress() {
14159
- const rows = await this.db.getAll('SELECT name, count_at_last, count_since_last FROM ps_buckets');
14160
- return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
14161
- }
14162
- async saveSyncData(batch, fixedKeyFormat = false) {
14163
- await this.writeTransaction(async (tx) => {
14164
- for (const b of batch.buckets) {
14165
- await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
14166
- 'save',
14167
- JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
14168
- ]);
14169
- this.logger.debug(`Saved batch of data for bucket: ${b.bucket}, operations: ${b.data.length}`);
14170
- }
14171
- });
14172
- }
14173
- async removeBuckets(buckets) {
14174
- for (const bucket of buckets) {
14175
- await this.deleteBucket(bucket);
14176
- }
14177
- }
14178
- /**
14179
- * Mark a bucket for deletion.
14180
- */
14181
- async deleteBucket(bucket) {
14182
- await this.writeTransaction(async (tx) => {
14183
- await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
14184
- });
14185
- this.logger.debug(`Done deleting bucket ${bucket}`);
14186
- }
14187
- async hasCompletedSync() {
14188
- if (this._hasCompletedSync) {
14189
- return true;
14190
- }
14191
- const r = await this.db.get(`SELECT powersync_last_synced_at() as synced_at`);
14192
- const completed = r.synced_at != null;
14193
- if (completed) {
14194
- this._hasCompletedSync = true;
14195
- }
14196
- return completed;
14197
- }
14198
- async syncLocalDatabase(checkpoint, priority) {
14199
- const r = await this.validateChecksums(checkpoint, priority);
14200
- if (!r.checkpointValid) {
14201
- this.logger.error('Checksums failed for', r.checkpointFailures);
14202
- for (const b of r.checkpointFailures ?? []) {
14203
- await this.deleteBucket(b);
14204
- }
14205
- return { ready: false, checkpointValid: false, checkpointFailures: r.checkpointFailures };
14206
- }
14207
- if (priority == null) {
14208
- this.logger.debug(`Validated checksums checkpoint ${checkpoint.last_op_id}`);
14209
- }
14210
- else {
14211
- this.logger.debug(`Validated checksums for partial checkpoint ${checkpoint.last_op_id}, priority ${priority}`);
14212
- }
14213
- let buckets = checkpoint.buckets;
14214
- if (priority !== undefined) {
14215
- buckets = buckets.filter((b) => hasMatchingPriority(priority, b));
14216
- }
14217
- const bucketNames = buckets.map((b) => b.bucket);
14218
- await this.writeTransaction(async (tx) => {
14219
- await tx.execute(`UPDATE ps_buckets SET last_op = ? WHERE name IN (SELECT json_each.value FROM json_each(?))`, [
14220
- checkpoint.last_op_id,
14221
- JSON.stringify(bucketNames)
14222
- ]);
14223
- if (priority == null && checkpoint.write_checkpoint) {
14224
- await tx.execute("UPDATE ps_buckets SET last_op = ? WHERE name = '$local'", [checkpoint.write_checkpoint]);
14225
- }
14226
- });
14227
- const valid = await this.updateObjectsFromBuckets(checkpoint, priority);
14228
- if (!valid) {
14229
- return { ready: false, checkpointValid: true };
14230
- }
14231
- return {
14232
- ready: true,
14233
- checkpointValid: true
14234
- };
14235
- }
14236
- /**
14237
- * Atomically update the local state to the current checkpoint.
14238
- *
14239
- * This includes creating new tables, dropping old tables, and copying data over from the oplog.
14240
- */
14241
- async updateObjectsFromBuckets(checkpoint, priority) {
14242
- let arg = '';
14243
- if (priority !== undefined) {
14244
- const affectedBuckets = [];
14245
- for (const desc of checkpoint.buckets) {
14246
- if (hasMatchingPriority(priority, desc)) {
14247
- affectedBuckets.push(desc.bucket);
14248
- }
14249
- }
14250
- arg = JSON.stringify({ priority, buckets: affectedBuckets });
14251
- }
14252
- return this.writeTransaction(async (tx) => {
14253
- const { insertId: result } = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
14254
- 'sync_local',
14255
- arg
14256
- ]);
14257
- if (result == 1) {
14258
- if (priority == null) {
14259
- const bucketToCount = Object.fromEntries(checkpoint.buckets.map((b) => [b.bucket, b.count]));
14260
- // The two parameters could be replaced with one, but: https://github.com/powersync-ja/better-sqlite3/pull/6
14261
- const jsonBucketCount = JSON.stringify(bucketToCount);
14262
- await tx.execute("UPDATE ps_buckets SET count_since_last = 0, count_at_last = ?->name WHERE name != '$local' AND ?->name IS NOT NULL", [jsonBucketCount, jsonBucketCount]);
14263
- }
14264
- return true;
14265
- }
14266
- else {
14267
- return false;
14268
- }
14269
- });
14270
- }
14271
- async validateChecksums(checkpoint, priority) {
14272
- if (priority !== undefined) {
14273
- // Only validate the buckets within the priority we care about
14274
- const newBuckets = checkpoint.buckets.filter((cs) => hasMatchingPriority(priority, cs));
14275
- checkpoint = { ...checkpoint, buckets: newBuckets };
14276
- }
14277
- const rs = await this.db.execute('SELECT powersync_validate_checkpoint(?) as result', [
14278
- JSON.stringify({ ...checkpoint })
14279
- ]);
14280
- const resultItem = rs.rows?.item(0);
14281
- if (!resultItem) {
14282
- return {
14283
- checkpointValid: false,
14284
- ready: false,
14285
- checkpointFailures: []
14286
- };
14287
- }
14288
- const result = JSON.parse(resultItem['result']);
14289
- if (result['valid']) {
14290
- return { ready: true, checkpointValid: true };
14291
- }
14292
- else {
14293
- return {
14294
- checkpointValid: false,
14295
- ready: false,
14296
- checkpointFailures: result['failed_buckets']
14297
- };
14298
- }
14299
- }
14300
13687
  async updateLocalTarget(cb) {
14301
13688
  const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", [MAX_OP_ID]);
14302
13689
  if (!rs1.length) {
@@ -14387,12 +13774,6 @@ class SqliteBucketStorage extends BaseObserver {
14387
13774
  async writeTransaction(callback, options) {
14388
13775
  return this.db.writeTransaction(callback, options);
14389
13776
  }
14390
- /**
14391
- * Set a target checkpoint.
14392
- */
14393
- async setTargetCheckpoint(checkpoint) {
14394
- // No-op for now
14395
- }
14396
13777
  async control(op, payload) {
14397
13778
  return await this.writeTransaction(async (tx) => {
14398
13779
  const [[raw]] = await tx.executeRaw('SELECT powersync_control(?, ?)', [op, payload]);
@@ -14416,20 +13797,6 @@ class SqliteBucketStorage extends BaseObserver {
14416
13797
  }
14417
13798
  static _subkeyMigrationKey = 'powersync_js_migrated_subkeys';
14418
13799
  }
14419
- function hasMatchingPriority(priority, bucket) {
14420
- return bucket.priority != null && bucket.priority <= priority;
14421
- }
14422
-
14423
- // TODO JSON
14424
- class SyncDataBatch {
14425
- buckets;
14426
- static fromJSON(json) {
14427
- return new SyncDataBatch(json.buckets.map((bucket) => SyncDataBucket.fromRow(bucket)));
14428
- }
14429
- constructor(buckets) {
14430
- this.buckets = buckets;
14431
- }
14432
- }
14433
13800
 
14434
13801
  /**
14435
13802
  * Thrown when an underlying database connection is closed.
@@ -14740,13 +14107,9 @@ exports.MAX_OP_ID = MAX_OP_ID;
14740
14107
  exports.MEMORY_TRIGGER_CLAIM_MANAGER = MEMORY_TRIGGER_CLAIM_MANAGER;
14741
14108
  exports.Mutex = Mutex;
14742
14109
  exports.OnChangeQueryProcessor = OnChangeQueryProcessor;
14743
- exports.OpType = OpType;
14744
- exports.OplogEntry = OplogEntry;
14745
14110
  exports.Schema = Schema;
14746
14111
  exports.Semaphore = Semaphore;
14747
14112
  exports.SqliteBucketStorage = SqliteBucketStorage;
14748
- exports.SyncDataBatch = SyncDataBatch;
14749
- exports.SyncDataBucket = SyncDataBucket;
14750
14113
  exports.SyncProgress = SyncProgress;
14751
14114
  exports.SyncStatus = SyncStatus;
14752
14115
  exports.SyncingService = SyncingService;
@@ -14761,18 +14124,10 @@ exports.createBaseLogger = createBaseLogger;
14761
14124
  exports.createLogger = createLogger;
14762
14125
  exports.extractTableUpdates = extractTableUpdates;
14763
14126
  exports.isBatchedUpdateNotification = isBatchedUpdateNotification;
14764
- exports.isContinueCheckpointRequest = isContinueCheckpointRequest;
14765
14127
  exports.isDBAdapter = isDBAdapter;
14766
14128
  exports.isPowerSyncDatabaseOptionsWithSettings = isPowerSyncDatabaseOptionsWithSettings;
14767
14129
  exports.isSQLOpenFactory = isSQLOpenFactory;
14768
14130
  exports.isSQLOpenOptions = isSQLOpenOptions;
14769
- exports.isStreamingKeepalive = isStreamingKeepalive;
14770
- exports.isStreamingSyncCheckpoint = isStreamingSyncCheckpoint;
14771
- exports.isStreamingSyncCheckpointComplete = isStreamingSyncCheckpointComplete;
14772
- exports.isStreamingSyncCheckpointDiff = isStreamingSyncCheckpointDiff;
14773
- exports.isStreamingSyncCheckpointPartiallyComplete = isStreamingSyncCheckpointPartiallyComplete;
14774
- exports.isStreamingSyncData = isStreamingSyncData;
14775
- exports.isSyncNewCheckpointRequest = isSyncNewCheckpointRequest;
14776
14131
  exports.parseQuery = parseQuery;
14777
14132
  exports.runOnSchemaChange = runOnSchemaChange;
14778
14133
  exports.sanitizeSQL = sanitizeSQL;