@powersync/common 0.0.0-dev-20260414110516 → 0.0.0-dev-20260504100448
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.cjs +33 -665
- package/dist/bundle.cjs.map +1 -1
- package/dist/bundle.mjs +34 -654
- package/dist/bundle.mjs.map +1 -1
- package/dist/bundle.node.cjs +33 -665
- package/dist/bundle.node.cjs.map +1 -1
- package/dist/bundle.node.mjs +34 -654
- package/dist/bundle.node.mjs.map +1 -1
- package/dist/index.d.cts +22 -369
- package/legacy/sync_protocol.d.ts +103 -0
- package/lib/client/sync/bucket/BucketStorageAdapter.d.ts +1 -63
- package/lib/client/sync/bucket/BucketStorageAdapter.js.map +1 -1
- package/lib/client/sync/bucket/SqliteBucketStorage.d.ts +1 -28
- package/lib/client/sync/bucket/SqliteBucketStorage.js +0 -162
- package/lib/client/sync/bucket/SqliteBucketStorage.js.map +1 -1
- package/lib/client/sync/stream/AbstractRemote.d.ts +2 -12
- package/lib/client/sync/stream/AbstractRemote.js +3 -13
- package/lib/client/sync/stream/AbstractRemote.js.map +1 -1
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +12 -35
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +29 -337
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js.map +1 -1
- package/lib/client/sync/stream/JsonValue.d.ts +7 -0
- package/lib/client/sync/stream/JsonValue.js +2 -0
- package/lib/client/sync/stream/JsonValue.js.map +1 -0
- package/lib/client/sync/stream/core-instruction.d.ts +1 -2
- package/lib/client/sync/stream/core-instruction.js.map +1 -1
- package/lib/db/crud/SyncStatus.d.ts +0 -4
- package/lib/db/crud/SyncStatus.js +0 -4
- package/lib/db/crud/SyncStatus.js.map +1 -1
- package/lib/db/schema/RawTable.d.ts +0 -5
- package/lib/db/schema/Schema.d.ts +0 -2
- package/lib/db/schema/Schema.js +0 -2
- package/lib/db/schema/Schema.js.map +1 -1
- package/lib/index.d.ts +1 -5
- package/lib/index.js +1 -5
- package/lib/index.js.map +1 -1
- package/package.json +7 -4
- package/src/client/sync/bucket/BucketStorageAdapter.ts +1 -70
- package/src/client/sync/bucket/SqliteBucketStorage.ts +1 -197
- package/src/client/sync/stream/AbstractRemote.ts +5 -27
- package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +41 -407
- package/src/client/sync/stream/JsonValue.ts +8 -0
- package/src/client/sync/stream/core-instruction.ts +1 -2
- package/src/db/crud/SyncStatus.ts +0 -4
- package/src/db/schema/RawTable.ts +0 -5
- package/src/db/schema/Schema.ts +0 -2
- package/src/index.ts +1 -5
- package/lib/client/sync/bucket/OpType.d.ts +0 -16
- package/lib/client/sync/bucket/OpType.js +0 -23
- package/lib/client/sync/bucket/OpType.js.map +0 -1
- package/lib/client/sync/bucket/OplogEntry.d.ts +0 -23
- package/lib/client/sync/bucket/OplogEntry.js +0 -36
- package/lib/client/sync/bucket/OplogEntry.js.map +0 -1
- package/lib/client/sync/bucket/SyncDataBatch.d.ts +0 -6
- package/lib/client/sync/bucket/SyncDataBatch.js +0 -12
- package/lib/client/sync/bucket/SyncDataBatch.js.map +0 -1
- package/lib/client/sync/bucket/SyncDataBucket.d.ts +0 -40
- package/lib/client/sync/bucket/SyncDataBucket.js +0 -40
- package/lib/client/sync/bucket/SyncDataBucket.js.map +0 -1
- package/lib/client/sync/stream/streaming-sync-types.d.ts +0 -143
- package/lib/client/sync/stream/streaming-sync-types.js +0 -26
- package/lib/client/sync/stream/streaming-sync-types.js.map +0 -1
- package/src/client/sync/bucket/OpType.ts +0 -23
- package/src/client/sync/bucket/OplogEntry.ts +0 -50
- package/src/client/sync/bucket/SyncDataBatch.ts +0 -11
- package/src/client/sync/bucket/SyncDataBucket.ts +0 -49
- package/src/client/sync/stream/streaming-sync-types.ts +0 -210
|
@@ -3,7 +3,7 @@ import { SyncStatus, SyncStatusOptions } from '../../../db/crud/SyncStatus.js';
|
|
|
3
3
|
import { BaseListener, BaseObserver, BaseObserverInterface, Disposable } from '../../../utils/BaseObserver.js';
|
|
4
4
|
import { BucketStorageAdapter } from '../bucket/BucketStorageAdapter.js';
|
|
5
5
|
import { AbstractRemote, FetchStrategy } from './AbstractRemote.js';
|
|
6
|
-
import { StreamingSyncRequestParameterType } from './
|
|
6
|
+
import { StreamingSyncRequestParameterType } from './JsonValue.js';
|
|
7
7
|
export declare enum LockType {
|
|
8
8
|
CRUD = "crud",
|
|
9
9
|
SYNC = "sync"
|
|
@@ -13,35 +13,21 @@ export declare enum SyncStreamConnectionMethod {
|
|
|
13
13
|
WEB_SOCKET = "web-socket"
|
|
14
14
|
}
|
|
15
15
|
export declare enum SyncClientImplementation {
|
|
16
|
-
/**
|
|
17
|
-
* Decodes and handles sync lines received from the sync service in JavaScript.
|
|
18
|
-
*
|
|
19
|
-
* This is the default option.
|
|
20
|
-
*
|
|
21
|
-
* @deprecated We recommend the {@link RUST} client implementation for all apps. If you have issues with
|
|
22
|
-
* the Rust client, please file an issue or reach out to us. The JavaScript client will be removed in a future
|
|
23
|
-
* version of the PowerSync SDK.
|
|
24
|
-
*/
|
|
25
|
-
JAVASCRIPT = "js",
|
|
26
16
|
/**
|
|
27
17
|
* This implementation offloads the sync line decoding and handling into the PowerSync
|
|
28
18
|
* core extension.
|
|
29
19
|
*
|
|
30
|
-
* This
|
|
31
|
-
* recommended client implementation for all apps.
|
|
20
|
+
* This is the only option, as an older JavaScript client implementation has been removed from the SDK.
|
|
32
21
|
*
|
|
33
22
|
* ## Compatibility warning
|
|
34
23
|
*
|
|
35
24
|
* The Rust sync client stores sync data in a format that is slightly different than the one used
|
|
36
|
-
* by the old
|
|
37
|
-
*
|
|
38
|
-
* Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
|
|
39
|
-
* the version introducing {@link RUST} as an option) also supports the new format, so you can switch
|
|
40
|
-
* back to {@link JAVASCRIPT} later.
|
|
25
|
+
* by the old JavaScript client. When adopting the {@link RUST} client on existing databases, the PowerSync SDK will
|
|
26
|
+
* migrate the format automatically.
|
|
41
27
|
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
28
|
+
* SDK versions supporting both the JavaScript and the Rust client support both formats with the JavaScript client
|
|
29
|
+
* implementaiton. However, downgrading to an SDK version that only supports the JavaScript client would not be
|
|
30
|
+
* possible anymore. Problematic SDK versions have been released before 2025-06-09.
|
|
45
31
|
*/
|
|
46
32
|
RUST = "rust"
|
|
47
33
|
}
|
|
@@ -94,11 +80,7 @@ export interface BaseConnectionOptions {
|
|
|
94
80
|
*/
|
|
95
81
|
appMetadata?: Record<string, string>;
|
|
96
82
|
/**
|
|
97
|
-
*
|
|
98
|
-
* service, or whether this work should be offloaded to the PowerSync core extension.
|
|
99
|
-
*
|
|
100
|
-
* This defaults to the JavaScript implementation ({@link SyncClientImplementation.JAVASCRIPT})
|
|
101
|
-
* since the ({@link SyncClientImplementation.RUST}) implementation is experimental at the moment.
|
|
83
|
+
* @deprecated The Rust sync client is used unconditionally, so this option can't be configured.
|
|
102
84
|
*/
|
|
103
85
|
clientImplementation?: SyncClientImplementation;
|
|
104
86
|
/**
|
|
@@ -155,15 +137,14 @@ export interface StreamingSyncImplementation extends BaseObserverInterface<Strea
|
|
|
155
137
|
*/
|
|
156
138
|
disconnect(): Promise<void>;
|
|
157
139
|
getWriteCheckpoint: () => Promise<string>;
|
|
158
|
-
hasCompletedSync: () => Promise<boolean>;
|
|
159
140
|
isConnected: boolean;
|
|
160
|
-
lastSyncedAt?: Date;
|
|
161
141
|
syncStatus: SyncStatus;
|
|
162
142
|
triggerCrudUpload: () => void;
|
|
163
143
|
waitForReady(): Promise<void>;
|
|
164
144
|
waitForStatus(status: SyncStatusOptions): Promise<void>;
|
|
165
145
|
waitUntilStatusMatches(predicate: (status: SyncStatus) => boolean): Promise<void>;
|
|
166
146
|
updateSubscriptions(subscriptions: SubscribedStream[]): void;
|
|
147
|
+
markConnectionMayHaveChanged(): void;
|
|
167
148
|
}
|
|
168
149
|
export declare const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
|
|
169
150
|
export declare const DEFAULT_RETRY_DELAY_MS = 5000;
|
|
@@ -178,14 +159,14 @@ export type SubscribedStream = {
|
|
|
178
159
|
params: Record<string, any> | null;
|
|
179
160
|
};
|
|
180
161
|
export declare abstract class AbstractStreamingSyncImplementation extends BaseObserver<StreamingSyncImplementationListener> implements StreamingSyncImplementation {
|
|
181
|
-
protected _lastSyncedAt: Date | null;
|
|
182
162
|
protected options: AbstractStreamingSyncImplementationOptions;
|
|
183
163
|
protected abortController: AbortController | null;
|
|
184
|
-
protected uploadAbortController: AbortController |
|
|
164
|
+
protected uploadAbortController: AbortController | undefined;
|
|
185
165
|
protected crudUpdateListener?: () => void;
|
|
186
166
|
protected streamingSyncPromise?: Promise<void>;
|
|
187
167
|
protected logger: ILogger;
|
|
188
168
|
private activeStreams;
|
|
169
|
+
private connectionMayHaveChanged;
|
|
189
170
|
private isUploadingCrud;
|
|
190
171
|
private notifyCompletedUploads?;
|
|
191
172
|
private handleActiveStreamsChange?;
|
|
@@ -199,7 +180,6 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
|
|
|
199
180
|
get isConnected(): boolean;
|
|
200
181
|
dispose(): Promise<void>;
|
|
201
182
|
abstract obtainLock<T>(lockOptions: LockOptions<T>): Promise<T>;
|
|
202
|
-
hasCompletedSync(): Promise<boolean>;
|
|
203
183
|
getWriteCheckpoint(): Promise<string>;
|
|
204
184
|
protected _uploadAllCrud(): Promise<void>;
|
|
205
185
|
connect(options?: PowerSyncConnectionOptions): Promise<void>;
|
|
@@ -208,7 +188,7 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
|
|
|
208
188
|
* @deprecated use [connect instead]
|
|
209
189
|
*/
|
|
210
190
|
streamingSync(signal?: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
|
|
211
|
-
|
|
191
|
+
markConnectionMayHaveChanged(): void;
|
|
212
192
|
/**
|
|
213
193
|
* Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
|
|
214
194
|
* Because subkeys are always strings, this leads to quotes being added around them in `ps_oplog`.
|
|
@@ -227,10 +207,7 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
|
|
|
227
207
|
private requireKeyFormat;
|
|
228
208
|
protected streamingSyncIteration(signal: AbortSignal, options?: PowerSyncConnectionOptions): Promise<RustIterationResult | null>;
|
|
229
209
|
private receiveSyncLines;
|
|
230
|
-
private legacyStreamingSyncIteration;
|
|
231
210
|
private rustSyncIteration;
|
|
232
|
-
private updateSyncStatusForStartingCheckpoint;
|
|
233
|
-
private applyCheckpoint;
|
|
234
211
|
protected updateSyncStatus(options: SyncStatusOptions): void;
|
|
235
212
|
private delayRetry;
|
|
236
213
|
updateSubscriptions(subscriptions: SubscribedStream[]): void;
|
|
@@ -4,10 +4,8 @@ import { AbortOperation } from '../../../utils/AbortOperation.js';
|
|
|
4
4
|
import { BaseObserver } from '../../../utils/BaseObserver.js';
|
|
5
5
|
import { throttleLeadingTrailing } from '../../../utils/async.js';
|
|
6
6
|
import { PowerSyncControlCommand } from '../bucket/BucketStorageAdapter.js';
|
|
7
|
-
import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
|
|
8
7
|
import { FetchStrategy } from './AbstractRemote.js';
|
|
9
8
|
import { coreStatusToJs } from './core-instruction.js';
|
|
10
|
-
import { isStreamingKeepalive, isStreamingSyncCheckpoint, isStreamingSyncCheckpointComplete, isStreamingSyncCheckpointDiff, isStreamingSyncCheckpointPartiallyComplete, isStreamingSyncData } from './streaming-sync-types.js';
|
|
11
9
|
import { injectable, map } from '../../../utils/stream_transform.js';
|
|
12
10
|
export var LockType;
|
|
13
11
|
(function (LockType) {
|
|
@@ -21,35 +19,21 @@ export var SyncStreamConnectionMethod;
|
|
|
21
19
|
})(SyncStreamConnectionMethod || (SyncStreamConnectionMethod = {}));
|
|
22
20
|
export var SyncClientImplementation;
|
|
23
21
|
(function (SyncClientImplementation) {
|
|
24
|
-
/**
|
|
25
|
-
* Decodes and handles sync lines received from the sync service in JavaScript.
|
|
26
|
-
*
|
|
27
|
-
* This is the default option.
|
|
28
|
-
*
|
|
29
|
-
* @deprecated We recommend the {@link RUST} client implementation for all apps. If you have issues with
|
|
30
|
-
* the Rust client, please file an issue or reach out to us. The JavaScript client will be removed in a future
|
|
31
|
-
* version of the PowerSync SDK.
|
|
32
|
-
*/
|
|
33
|
-
SyncClientImplementation["JAVASCRIPT"] = "js";
|
|
34
22
|
/**
|
|
35
23
|
* This implementation offloads the sync line decoding and handling into the PowerSync
|
|
36
24
|
* core extension.
|
|
37
25
|
*
|
|
38
|
-
* This
|
|
39
|
-
* recommended client implementation for all apps.
|
|
26
|
+
* This is the only option, as an older JavaScript client implementation has been removed from the SDK.
|
|
40
27
|
*
|
|
41
28
|
* ## Compatibility warning
|
|
42
29
|
*
|
|
43
30
|
* The Rust sync client stores sync data in a format that is slightly different than the one used
|
|
44
|
-
* by the old
|
|
45
|
-
*
|
|
46
|
-
* Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
|
|
47
|
-
* the version introducing {@link RUST} as an option) also supports the new format, so you can switch
|
|
48
|
-
* back to {@link JAVASCRIPT} later.
|
|
31
|
+
* by the old JavaScript client. When adopting the {@link RUST} client on existing databases, the PowerSync SDK will
|
|
32
|
+
* migrate the format automatically.
|
|
49
33
|
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
34
|
+
* SDK versions supporting both the JavaScript and the Rust client support both formats with the JavaScript client
|
|
35
|
+
* implementaiton. However, downgrading to an SDK version that only supports the JavaScript client would not be
|
|
36
|
+
* possible anymore. Problematic SDK versions have been released before 2025-06-09.
|
|
53
37
|
*/
|
|
54
38
|
SyncClientImplementation["RUST"] = "rust";
|
|
55
39
|
})(SyncClientImplementation || (SyncClientImplementation = {}));
|
|
@@ -78,7 +62,6 @@ export const DEFAULT_STREAM_CONNECTION_OPTIONS = {
|
|
|
78
62
|
// messages.
|
|
79
63
|
const FALLBACK_PRIORITY = 3;
|
|
80
64
|
export class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
81
|
-
_lastSyncedAt;
|
|
82
65
|
options;
|
|
83
66
|
abortController;
|
|
84
67
|
// In rare cases, mostly for tests, uploads can be triggered without being properly connected.
|
|
@@ -88,6 +71,7 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
|
88
71
|
streamingSyncPromise;
|
|
89
72
|
logger;
|
|
90
73
|
activeStreams;
|
|
74
|
+
connectionMayHaveChanged = false;
|
|
91
75
|
isUploadingCrud = false;
|
|
92
76
|
notifyCompletedUploads;
|
|
93
77
|
handleActiveStreamsChange;
|
|
@@ -167,9 +151,6 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
|
167
151
|
this.crudUpdateListener = undefined;
|
|
168
152
|
this.uploadAbortController?.abort();
|
|
169
153
|
}
|
|
170
|
-
async hasCompletedSync() {
|
|
171
|
-
return this.options.adapter.hasCompletedSync();
|
|
172
|
-
}
|
|
173
154
|
async getWriteCheckpoint() {
|
|
174
155
|
const clientId = await this.options.adapter.getClientId();
|
|
175
156
|
let path = `/write-checkpoint2.json?client_id=${clientId}`;
|
|
@@ -251,7 +232,7 @@ The next upload iteration will be delayed.`);
|
|
|
251
232
|
});
|
|
252
233
|
}
|
|
253
234
|
}
|
|
254
|
-
this.uploadAbortController =
|
|
235
|
+
this.uploadAbortController = undefined;
|
|
255
236
|
}
|
|
256
237
|
});
|
|
257
238
|
}
|
|
@@ -367,6 +348,11 @@ The next upload iteration will be delayed.`);
|
|
|
367
348
|
shouldDelayRetry = false;
|
|
368
349
|
// A disconnect was requested, we should not delay since there is no explicit retry
|
|
369
350
|
}
|
|
351
|
+
else if (this.connectionMayHaveChanged && ex.message?.indexOf('No iteration is active') >= 0) {
|
|
352
|
+
this.connectionMayHaveChanged = false;
|
|
353
|
+
this.logger.info('Sync error after changed connection, retrying immediately');
|
|
354
|
+
shouldDelayRetry = false;
|
|
355
|
+
}
|
|
370
356
|
else {
|
|
371
357
|
this.logger.error(ex);
|
|
372
358
|
}
|
|
@@ -397,17 +383,14 @@ The next upload iteration will be delayed.`);
|
|
|
397
383
|
// Mark as disconnected if here
|
|
398
384
|
this.updateSyncStatus({ connected: false, connecting: false });
|
|
399
385
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
localDescriptions.set(entry.bucket, null);
|
|
409
|
-
}
|
|
410
|
-
return [req, localDescriptions];
|
|
386
|
+
markConnectionMayHaveChanged() {
|
|
387
|
+
// By setting this field, we'll immediately retry if the next sync event causes an error triggered by us not having
|
|
388
|
+
// an active sync iteration on the connection in use.
|
|
389
|
+
this.connectionMayHaveChanged = true;
|
|
390
|
+
// This triggers a `powersync_control` invocation if a sync iteration is currently active. This is a cheap call to
|
|
391
|
+
// make when no subscriptions have actually changed, we're mainly interested in this immediately throwing if no
|
|
392
|
+
// iteration is active. That allows us to reconnect ASAP, instead of having to wait for the next sync line.
|
|
393
|
+
this.handleActiveStreamsChange?.();
|
|
411
394
|
}
|
|
412
395
|
/**
|
|
413
396
|
* Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
|
|
@@ -450,19 +433,13 @@ The next upload iteration will be delayed.`);
|
|
|
450
433
|
}
|
|
451
434
|
const clientImplementation = resolvedOptions.clientImplementation;
|
|
452
435
|
this.updateSyncStatus({ clientImplementation });
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
return null;
|
|
456
|
-
}
|
|
457
|
-
else {
|
|
458
|
-
await this.requireKeyFormat(true);
|
|
459
|
-
return await this.rustSyncIteration(signal, resolvedOptions);
|
|
460
|
-
}
|
|
436
|
+
await this.requireKeyFormat(true);
|
|
437
|
+
return await this.rustSyncIteration(signal, resolvedOptions);
|
|
461
438
|
}
|
|
462
439
|
});
|
|
463
440
|
}
|
|
464
441
|
async receiveSyncLines(data) {
|
|
465
|
-
const { options, connection
|
|
442
|
+
const { options, connection } = data;
|
|
466
443
|
const remote = this.options.remote;
|
|
467
444
|
if (connection.connectionMethod == SyncStreamConnectionMethod.HTTP) {
|
|
468
445
|
return await remote.fetchStream(options);
|
|
@@ -471,235 +448,8 @@ The next upload iteration will be delayed.`);
|
|
|
471
448
|
return await this.options.remote.socketStreamRaw({
|
|
472
449
|
...options,
|
|
473
450
|
...{ fetchStrategy: connection.fetchStrategy }
|
|
474
|
-
}
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
async legacyStreamingSyncIteration(signal, resolvedOptions) {
|
|
478
|
-
const rawTables = resolvedOptions.serializedSchema?.raw_tables;
|
|
479
|
-
if (rawTables != null && rawTables.length) {
|
|
480
|
-
this.logger.warn('Raw tables require the Rust-based sync client. The JS client will ignore them.');
|
|
481
|
-
}
|
|
482
|
-
if (this.activeStreams.length) {
|
|
483
|
-
this.logger.error('Sync streams require `clientImplementation: SyncClientImplementation.RUST` when connecting.');
|
|
484
|
-
}
|
|
485
|
-
this.logger.debug('Streaming sync iteration started');
|
|
486
|
-
this.options.adapter.startSession();
|
|
487
|
-
let [req, bucketMap] = await this.collectLocalBucketState();
|
|
488
|
-
let targetCheckpoint = null;
|
|
489
|
-
// A checkpoint that has been validated but not applied (e.g. due to pending local writes)
|
|
490
|
-
let pendingValidatedCheckpoint = null;
|
|
491
|
-
const clientId = await this.options.adapter.getClientId();
|
|
492
|
-
const usingFixedKeyFormat = await this.requireKeyFormat(false);
|
|
493
|
-
this.logger.debug('Requesting stream from server');
|
|
494
|
-
const syncOptions = {
|
|
495
|
-
path: '/sync/stream',
|
|
496
|
-
abortSignal: signal,
|
|
497
|
-
data: {
|
|
498
|
-
buckets: req,
|
|
499
|
-
include_checksum: true,
|
|
500
|
-
raw_data: true,
|
|
501
|
-
parameters: resolvedOptions.params,
|
|
502
|
-
app_metadata: resolvedOptions.appMetadata,
|
|
503
|
-
client_id: clientId
|
|
504
|
-
}
|
|
505
|
-
};
|
|
506
|
-
const bson = await this.options.remote.getBSON();
|
|
507
|
-
const source = await this.receiveSyncLines({
|
|
508
|
-
options: syncOptions,
|
|
509
|
-
connection: resolvedOptions,
|
|
510
|
-
bson
|
|
511
|
-
});
|
|
512
|
-
const stream = injectable(map(source, (line) => {
|
|
513
|
-
if (typeof line == 'string') {
|
|
514
|
-
return JSON.parse(line);
|
|
515
|
-
}
|
|
516
|
-
else {
|
|
517
|
-
return bson.deserialize(line);
|
|
518
|
-
}
|
|
519
|
-
}));
|
|
520
|
-
this.logger.debug('Stream established. Processing events');
|
|
521
|
-
this.notifyCompletedUploads = () => {
|
|
522
|
-
stream.inject({ crud_upload_completed: null });
|
|
523
|
-
};
|
|
524
|
-
while (true) {
|
|
525
|
-
const { value: line, done } = await stream.next();
|
|
526
|
-
if (done) {
|
|
527
|
-
// The stream has closed while waiting
|
|
528
|
-
return;
|
|
529
|
-
}
|
|
530
|
-
if ('crud_upload_completed' in line) {
|
|
531
|
-
if (pendingValidatedCheckpoint != null) {
|
|
532
|
-
const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
|
|
533
|
-
if (applied) {
|
|
534
|
-
pendingValidatedCheckpoint = null;
|
|
535
|
-
}
|
|
536
|
-
else if (endIteration) {
|
|
537
|
-
break;
|
|
538
|
-
}
|
|
539
|
-
}
|
|
540
|
-
continue;
|
|
541
|
-
}
|
|
542
|
-
// A connection is active and messages are being received
|
|
543
|
-
if (!this.syncStatus.connected) {
|
|
544
|
-
// There is a connection now
|
|
545
|
-
Promise.resolve().then(() => this.triggerCrudUpload());
|
|
546
|
-
this.updateSyncStatus({
|
|
547
|
-
connected: true
|
|
548
|
-
});
|
|
549
|
-
}
|
|
550
|
-
if (isStreamingSyncCheckpoint(line)) {
|
|
551
|
-
targetCheckpoint = line.checkpoint;
|
|
552
|
-
// New checkpoint - existing validated checkpoint is no longer valid
|
|
553
|
-
pendingValidatedCheckpoint = null;
|
|
554
|
-
const bucketsToDelete = new Set(bucketMap.keys());
|
|
555
|
-
const newBuckets = new Map();
|
|
556
|
-
for (const checksum of line.checkpoint.buckets) {
|
|
557
|
-
newBuckets.set(checksum.bucket, {
|
|
558
|
-
name: checksum.bucket,
|
|
559
|
-
priority: checksum.priority ?? FALLBACK_PRIORITY
|
|
560
|
-
});
|
|
561
|
-
bucketsToDelete.delete(checksum.bucket);
|
|
562
|
-
}
|
|
563
|
-
if (bucketsToDelete.size > 0) {
|
|
564
|
-
this.logger.debug('Removing buckets', [...bucketsToDelete]);
|
|
565
|
-
}
|
|
566
|
-
bucketMap = newBuckets;
|
|
567
|
-
await this.options.adapter.removeBuckets([...bucketsToDelete]);
|
|
568
|
-
await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
|
|
569
|
-
await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
|
|
570
|
-
}
|
|
571
|
-
else if (isStreamingSyncCheckpointComplete(line)) {
|
|
572
|
-
const result = await this.applyCheckpoint(targetCheckpoint);
|
|
573
|
-
if (result.endIteration) {
|
|
574
|
-
return;
|
|
575
|
-
}
|
|
576
|
-
else if (!result.applied) {
|
|
577
|
-
// "Could not apply checkpoint due to local data". We need to retry after
|
|
578
|
-
// finishing uploads.
|
|
579
|
-
pendingValidatedCheckpoint = targetCheckpoint;
|
|
580
|
-
}
|
|
581
|
-
else {
|
|
582
|
-
// Nothing to retry later. This would likely already be null from the last
|
|
583
|
-
// checksum or checksum_diff operation, but we make sure.
|
|
584
|
-
pendingValidatedCheckpoint = null;
|
|
585
|
-
}
|
|
586
|
-
}
|
|
587
|
-
else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
|
|
588
|
-
const priority = line.partial_checkpoint_complete.priority;
|
|
589
|
-
this.logger.debug('Partial checkpoint complete', priority);
|
|
590
|
-
const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint, priority);
|
|
591
|
-
if (!result.checkpointValid) {
|
|
592
|
-
// This means checksums failed. Start again with a new checkpoint.
|
|
593
|
-
// TODO: better back-off
|
|
594
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
595
|
-
return;
|
|
596
|
-
}
|
|
597
|
-
else if (!result.ready) {
|
|
598
|
-
// If we have pending uploads, we can't complete new checkpoints outside of priority 0.
|
|
599
|
-
// We'll resolve this for a complete checkpoint.
|
|
600
|
-
}
|
|
601
|
-
else {
|
|
602
|
-
// We'll keep on downloading, but can report that this priority is synced now.
|
|
603
|
-
this.logger.debug('partial checkpoint validation succeeded');
|
|
604
|
-
// All states with a higher priority can be deleted since this partial sync includes them.
|
|
605
|
-
const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
|
|
606
|
-
priorityStates.push({
|
|
607
|
-
priority,
|
|
608
|
-
lastSyncedAt: new Date(),
|
|
609
|
-
hasSynced: true
|
|
610
|
-
});
|
|
611
|
-
this.updateSyncStatus({
|
|
612
|
-
connected: true,
|
|
613
|
-
priorityStatusEntries: priorityStates
|
|
614
|
-
});
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
else if (isStreamingSyncCheckpointDiff(line)) {
|
|
618
|
-
// TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
|
|
619
|
-
if (targetCheckpoint == null) {
|
|
620
|
-
throw new Error('Checkpoint diff without previous checkpoint');
|
|
621
|
-
}
|
|
622
|
-
// New checkpoint - existing validated checkpoint is no longer valid
|
|
623
|
-
pendingValidatedCheckpoint = null;
|
|
624
|
-
const diff = line.checkpoint_diff;
|
|
625
|
-
const newBuckets = new Map();
|
|
626
|
-
for (const checksum of targetCheckpoint.buckets) {
|
|
627
|
-
newBuckets.set(checksum.bucket, checksum);
|
|
628
|
-
}
|
|
629
|
-
for (const checksum of diff.updated_buckets) {
|
|
630
|
-
newBuckets.set(checksum.bucket, checksum);
|
|
631
|
-
}
|
|
632
|
-
for (const bucket of diff.removed_buckets) {
|
|
633
|
-
newBuckets.delete(bucket);
|
|
634
|
-
}
|
|
635
|
-
const newCheckpoint = {
|
|
636
|
-
last_op_id: diff.last_op_id,
|
|
637
|
-
buckets: [...newBuckets.values()],
|
|
638
|
-
write_checkpoint: diff.write_checkpoint
|
|
639
|
-
};
|
|
640
|
-
targetCheckpoint = newCheckpoint;
|
|
641
|
-
await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
|
|
642
|
-
bucketMap = new Map();
|
|
643
|
-
newBuckets.forEach((checksum, name) => bucketMap.set(name, {
|
|
644
|
-
name: checksum.bucket,
|
|
645
|
-
priority: checksum.priority ?? FALLBACK_PRIORITY
|
|
646
|
-
}));
|
|
647
|
-
const bucketsToDelete = diff.removed_buckets;
|
|
648
|
-
if (bucketsToDelete.length > 0) {
|
|
649
|
-
this.logger.debug('Remove buckets', bucketsToDelete);
|
|
650
|
-
}
|
|
651
|
-
await this.options.adapter.removeBuckets(bucketsToDelete);
|
|
652
|
-
await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
|
|
653
|
-
}
|
|
654
|
-
else if (isStreamingSyncData(line)) {
|
|
655
|
-
const { data } = line;
|
|
656
|
-
const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
|
|
657
|
-
let updatedProgress = null;
|
|
658
|
-
if (previousProgress) {
|
|
659
|
-
updatedProgress = { ...previousProgress };
|
|
660
|
-
const progressForBucket = updatedProgress[data.bucket];
|
|
661
|
-
if (progressForBucket) {
|
|
662
|
-
updatedProgress[data.bucket] = {
|
|
663
|
-
...progressForBucket,
|
|
664
|
-
since_last: progressForBucket.since_last + data.data.length
|
|
665
|
-
};
|
|
666
|
-
}
|
|
667
|
-
}
|
|
668
|
-
this.updateSyncStatus({
|
|
669
|
-
dataFlow: {
|
|
670
|
-
downloading: true,
|
|
671
|
-
downloadProgress: updatedProgress
|
|
672
|
-
}
|
|
673
|
-
});
|
|
674
|
-
await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] }, usingFixedKeyFormat);
|
|
675
|
-
}
|
|
676
|
-
else if (isStreamingKeepalive(line)) {
|
|
677
|
-
const remaining_seconds = line.token_expires_in;
|
|
678
|
-
if (remaining_seconds == 0) {
|
|
679
|
-
// Connection would be closed automatically right after this
|
|
680
|
-
this.logger.debug('Token expiring; reconnect');
|
|
681
|
-
/**
|
|
682
|
-
* For a rare case where the backend connector does not update the token
|
|
683
|
-
* (uses the same one), this should have some delay.
|
|
684
|
-
*/
|
|
685
|
-
await this.delayRetry();
|
|
686
|
-
return;
|
|
687
|
-
}
|
|
688
|
-
else if (remaining_seconds < 30) {
|
|
689
|
-
this.logger.debug('Token will expire soon; reconnect');
|
|
690
|
-
// Pre-emptively refresh the token
|
|
691
|
-
this.options.remote.invalidateCredentials();
|
|
692
|
-
return;
|
|
693
|
-
}
|
|
694
|
-
this.triggerCrudUpload();
|
|
695
|
-
}
|
|
696
|
-
else {
|
|
697
|
-
this.logger.debug('Received unknown sync line', line);
|
|
698
|
-
}
|
|
451
|
+
});
|
|
699
452
|
}
|
|
700
|
-
this.logger.debug('Stream input empty');
|
|
701
|
-
// Connection closed. Likely due to auth issue.
|
|
702
|
-
return;
|
|
703
453
|
}
|
|
704
454
|
async rustSyncIteration(signal, resolvedOptions) {
|
|
705
455
|
const syncImplementation = this;
|
|
@@ -772,6 +522,10 @@ The next upload iteration will be delayed.`);
|
|
|
772
522
|
const rawResponse = await adapter.control(op, payload ?? null);
|
|
773
523
|
const logger = syncImplementation.logger;
|
|
774
524
|
logger.trace('powersync_control', op, payload == null || typeof payload == 'string' ? payload : '<bytes>', rawResponse);
|
|
525
|
+
if (op != PowerSyncControlCommand.STOP) {
|
|
526
|
+
// Evidently we have a working connection here, otherwise powersync_control would have failed.
|
|
527
|
+
syncImplementation.connectionMayHaveChanged = false;
|
|
528
|
+
}
|
|
775
529
|
await handleInstructions(JSON.parse(rawResponse));
|
|
776
530
|
}
|
|
777
531
|
async function handleInstruction(instruction) {
|
|
@@ -860,68 +614,6 @@ The next upload iteration will be delayed.`);
|
|
|
860
614
|
}
|
|
861
615
|
return { immediateRestart: hideDisconnectOnRestart };
|
|
862
616
|
}
|
|
863
|
-
async updateSyncStatusForStartingCheckpoint(checkpoint) {
|
|
864
|
-
const localProgress = await this.options.adapter.getBucketOperationProgress();
|
|
865
|
-
const progress = {};
|
|
866
|
-
let invalidated = false;
|
|
867
|
-
for (const bucket of checkpoint.buckets) {
|
|
868
|
-
const savedProgress = localProgress[bucket.bucket];
|
|
869
|
-
const atLast = savedProgress?.atLast ?? 0;
|
|
870
|
-
const sinceLast = savedProgress?.sinceLast ?? 0;
|
|
871
|
-
progress[bucket.bucket] = {
|
|
872
|
-
// The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
|
|
873
|
-
// will use by default.
|
|
874
|
-
priority: bucket.priority ?? 3,
|
|
875
|
-
at_last: atLast,
|
|
876
|
-
since_last: sinceLast,
|
|
877
|
-
target_count: bucket.count ?? 0
|
|
878
|
-
};
|
|
879
|
-
if (bucket.count != null && bucket.count < atLast + sinceLast) {
|
|
880
|
-
// Either due to a defrag / sync rule deploy or a compaction operation, the size
|
|
881
|
-
// of the bucket shrank so much that the local ops exceed the ops in the updated
|
|
882
|
-
// bucket. We can't prossibly report progress in this case (it would overshoot 100%).
|
|
883
|
-
invalidated = true;
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
if (invalidated) {
|
|
887
|
-
for (const bucket in progress) {
|
|
888
|
-
const bucketProgress = progress[bucket];
|
|
889
|
-
bucketProgress.at_last = 0;
|
|
890
|
-
bucketProgress.since_last = 0;
|
|
891
|
-
}
|
|
892
|
-
}
|
|
893
|
-
this.updateSyncStatus({
|
|
894
|
-
dataFlow: {
|
|
895
|
-
downloading: true,
|
|
896
|
-
downloadProgress: progress
|
|
897
|
-
}
|
|
898
|
-
});
|
|
899
|
-
}
|
|
900
|
-
async applyCheckpoint(checkpoint) {
|
|
901
|
-
let result = await this.options.adapter.syncLocalDatabase(checkpoint);
|
|
902
|
-
if (!result.checkpointValid) {
|
|
903
|
-
this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
|
|
904
|
-
// This means checksums failed. Start again with a new checkpoint.
|
|
905
|
-
// TODO: better back-off
|
|
906
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
907
|
-
return { applied: false, endIteration: true };
|
|
908
|
-
}
|
|
909
|
-
else if (!result.ready) {
|
|
910
|
-
this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`);
|
|
911
|
-
return { applied: false, endIteration: false };
|
|
912
|
-
}
|
|
913
|
-
this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
|
|
914
|
-
this.updateSyncStatus({
|
|
915
|
-
connected: true,
|
|
916
|
-
lastSyncedAt: new Date(),
|
|
917
|
-
dataFlow: {
|
|
918
|
-
downloading: false,
|
|
919
|
-
downloadProgress: null,
|
|
920
|
-
downloadError: undefined
|
|
921
|
-
}
|
|
922
|
-
});
|
|
923
|
-
return { applied: true, endIteration: false };
|
|
924
|
-
}
|
|
925
617
|
updateSyncStatus(options) {
|
|
926
618
|
const updatedStatus = new SyncStatus({
|
|
927
619
|
connected: options.connected ?? this.syncStatus.connected,
|