@powersync/common 0.0.0-dev-20250710154318 → 0.0.0-dev-20250714144421
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.cjs +3 -3
- package/dist/bundle.mjs +3 -3
- package/lib/client/AbstractPowerSyncDatabase.d.ts +0 -5
- package/lib/client/AbstractPowerSyncDatabase.js +3 -13
- package/lib/client/sync/bucket/SqliteBucketStorage.d.ts +1 -3
- package/lib/client/sync/bucket/SqliteBucketStorage.js +15 -17
- package/lib/client/sync/stream/AbstractRemote.d.ts +1 -10
- package/lib/client/sync/stream/AbstractRemote.js +0 -16
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +1 -1
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +81 -75
- package/lib/client/sync/stream/streaming-sync-types.d.ts +4 -0
- package/lib/utils/async.d.ts +0 -1
- package/lib/utils/async.js +0 -10
- package/package.json +1 -1
|
@@ -100,11 +100,6 @@ export declare const DEFAULT_LOCK_TIMEOUT_MS = 120000;
|
|
|
100
100
|
export declare const isPowerSyncDatabaseOptionsWithSettings: (test: any) => test is PowerSyncDatabaseOptionsWithSettings;
|
|
101
101
|
export declare abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDBListener> {
|
|
102
102
|
protected options: PowerSyncDatabaseOptions;
|
|
103
|
-
/**
|
|
104
|
-
* Transactions should be queued in the DBAdapter, but we also want to prevent
|
|
105
|
-
* calls to `.execute` while an async transaction is running.
|
|
106
|
-
*/
|
|
107
|
-
protected static transactionMutex: Mutex;
|
|
108
103
|
/**
|
|
109
104
|
* Returns true if the connection is closed.
|
|
110
105
|
*/
|
|
@@ -8,7 +8,6 @@ import { UploadQueueStats } from '../db/crud/UploadQueueStatus.js';
|
|
|
8
8
|
import { BaseObserver } from '../utils/BaseObserver.js';
|
|
9
9
|
import { ControlledExecutor } from '../utils/ControlledExecutor.js';
|
|
10
10
|
import { throttleTrailing } from '../utils/async.js';
|
|
11
|
-
import { mutexRunExclusive } from '../utils/mutex.js';
|
|
12
11
|
import { ConnectionManager } from './ConnectionManager.js';
|
|
13
12
|
import { isDBAdapter, isSQLOpenFactory, isSQLOpenOptions } from './SQLOpenFactory.js';
|
|
14
13
|
import { runOnSchemaChange } from './runOnSchemaChange.js';
|
|
@@ -45,11 +44,6 @@ export const isPowerSyncDatabaseOptionsWithSettings = (test) => {
|
|
|
45
44
|
};
|
|
46
45
|
export class AbstractPowerSyncDatabase extends BaseObserver {
|
|
47
46
|
options;
|
|
48
|
-
/**
|
|
49
|
-
* Transactions should be queued in the DBAdapter, but we also want to prevent
|
|
50
|
-
* calls to `.execute` while an async transaction is running.
|
|
51
|
-
*/
|
|
52
|
-
static transactionMutex = new Mutex();
|
|
53
47
|
/**
|
|
54
48
|
* Returns true if the connection is closed.
|
|
55
49
|
*/
|
|
@@ -475,8 +469,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
|
|
|
475
469
|
* @returns The query result as an object with structured key-value pairs
|
|
476
470
|
*/
|
|
477
471
|
async execute(sql, parameters) {
|
|
478
|
-
|
|
479
|
-
return this.database.execute(sql, parameters);
|
|
472
|
+
return this.writeLock((tx) => tx.execute(sql, parameters));
|
|
480
473
|
}
|
|
481
474
|
/**
|
|
482
475
|
* Execute a SQL write (INSERT/UPDATE/DELETE) query directly on the database without any PowerSync processing.
|
|
@@ -544,7 +537,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
|
|
|
544
537
|
*/
|
|
545
538
|
async readLock(callback) {
|
|
546
539
|
await this.waitForReady();
|
|
547
|
-
return
|
|
540
|
+
return this.database.readLock(callback);
|
|
548
541
|
}
|
|
549
542
|
/**
|
|
550
543
|
* Takes a global lock, without starting a transaction.
|
|
@@ -552,10 +545,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
|
|
|
552
545
|
*/
|
|
553
546
|
async writeLock(callback) {
|
|
554
547
|
await this.waitForReady();
|
|
555
|
-
return
|
|
556
|
-
const res = await callback(this.database);
|
|
557
|
-
return res;
|
|
558
|
-
});
|
|
548
|
+
return this.database.writeLock(callback);
|
|
559
549
|
}
|
|
560
550
|
/**
|
|
561
551
|
* Open a read-only transaction.
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { Mutex } from 'async-mutex';
|
|
2
1
|
import { ILogger } from 'js-logger';
|
|
3
2
|
import { DBAdapter, Transaction } from '../../../db/DBAdapter.js';
|
|
4
3
|
import { BaseObserver } from '../../../utils/BaseObserver.js';
|
|
@@ -8,13 +7,12 @@ import { CrudEntry } from './CrudEntry.js';
|
|
|
8
7
|
import { SyncDataBatch } from './SyncDataBatch.js';
|
|
9
8
|
export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListener> implements BucketStorageAdapter {
|
|
10
9
|
private db;
|
|
11
|
-
private mutex;
|
|
12
10
|
private logger;
|
|
13
11
|
tableNames: Set<string>;
|
|
14
12
|
private _hasCompletedSync;
|
|
15
13
|
private updateListener;
|
|
16
14
|
private _clientId?;
|
|
17
|
-
constructor(db: DBAdapter,
|
|
15
|
+
constructor(db: DBAdapter, logger?: ILogger);
|
|
18
16
|
init(): Promise<void>;
|
|
19
17
|
dispose(): Promise<void>;
|
|
20
18
|
_getClientId(): Promise<string>;
|
|
@@ -6,16 +6,14 @@ import { PSInternalTable } from './BucketStorageAdapter.js';
|
|
|
6
6
|
import { CrudEntry } from './CrudEntry.js';
|
|
7
7
|
export class SqliteBucketStorage extends BaseObserver {
|
|
8
8
|
db;
|
|
9
|
-
mutex;
|
|
10
9
|
logger;
|
|
11
10
|
tableNames;
|
|
12
11
|
_hasCompletedSync;
|
|
13
12
|
updateListener;
|
|
14
13
|
_clientId;
|
|
15
|
-
constructor(db,
|
|
14
|
+
constructor(db, logger = Logger.get('SqliteBucketStorage')) {
|
|
16
15
|
super();
|
|
17
16
|
this.db = db;
|
|
18
|
-
this.mutex = mutex;
|
|
19
17
|
this.logger = logger;
|
|
20
18
|
this._hasCompletedSync = false;
|
|
21
19
|
this.tableNames = new Set();
|
|
@@ -66,11 +64,11 @@ export class SqliteBucketStorage extends BaseObserver {
|
|
|
66
64
|
async saveSyncData(batch, fixedKeyFormat = false) {
|
|
67
65
|
await this.writeTransaction(async (tx) => {
|
|
68
66
|
for (const b of batch.buckets) {
|
|
69
|
-
|
|
67
|
+
await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
|
|
70
68
|
'save',
|
|
71
69
|
JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
|
|
72
70
|
]);
|
|
73
|
-
this.logger.debug(
|
|
71
|
+
this.logger.debug(`Saved batch of data for bucket: ${b.bucket}, operations: ${b.data.length}`);
|
|
74
72
|
}
|
|
75
73
|
});
|
|
76
74
|
}
|
|
@@ -86,7 +84,7 @@ export class SqliteBucketStorage extends BaseObserver {
|
|
|
86
84
|
await this.writeTransaction(async (tx) => {
|
|
87
85
|
await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
|
|
88
86
|
});
|
|
89
|
-
this.logger.debug(
|
|
87
|
+
this.logger.debug(`Done deleting bucket ${bucket}`);
|
|
90
88
|
}
|
|
91
89
|
async hasCompletedSync() {
|
|
92
90
|
if (this._hasCompletedSync) {
|
|
@@ -108,6 +106,12 @@ export class SqliteBucketStorage extends BaseObserver {
|
|
|
108
106
|
}
|
|
109
107
|
return { ready: false, checkpointValid: false, checkpointFailures: r.checkpointFailures };
|
|
110
108
|
}
|
|
109
|
+
if (priority == null) {
|
|
110
|
+
this.logger.debug(`Validated checksums checkpoint ${checkpoint.last_op_id}`);
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
this.logger.debug(`Validated checksums for partial checkpoint ${checkpoint.last_op_id}, priority ${priority}`);
|
|
114
|
+
}
|
|
111
115
|
let buckets = checkpoint.buckets;
|
|
112
116
|
if (priority !== undefined) {
|
|
113
117
|
buckets = buckets.filter((b) => hasMatchingPriority(priority, b));
|
|
@@ -124,7 +128,6 @@ export class SqliteBucketStorage extends BaseObserver {
|
|
|
124
128
|
});
|
|
125
129
|
const valid = await this.updateObjectsFromBuckets(checkpoint, priority);
|
|
126
130
|
if (!valid) {
|
|
127
|
-
this.logger.debug('Not at a consistent checkpoint - cannot update local db');
|
|
128
131
|
return { ready: false, checkpointValid: true };
|
|
129
132
|
}
|
|
130
133
|
return {
|
|
@@ -177,7 +180,6 @@ export class SqliteBucketStorage extends BaseObserver {
|
|
|
177
180
|
JSON.stringify({ ...checkpoint })
|
|
178
181
|
]);
|
|
179
182
|
const resultItem = rs.rows?.item(0);
|
|
180
|
-
this.logger.debug('validateChecksums priority, checkpoint, result item', priority, checkpoint, resultItem);
|
|
181
183
|
if (!resultItem) {
|
|
182
184
|
return {
|
|
183
185
|
checkpointValid: false,
|
|
@@ -210,30 +212,26 @@ export class SqliteBucketStorage extends BaseObserver {
|
|
|
210
212
|
}
|
|
211
213
|
const seqBefore = rs[0]['seq'];
|
|
212
214
|
const opId = await cb();
|
|
213
|
-
this.logger.debug(`[updateLocalTarget] Updating target to checkpoint ${opId}`);
|
|
214
215
|
return this.writeTransaction(async (tx) => {
|
|
215
216
|
const anyData = await tx.execute('SELECT 1 FROM ps_crud LIMIT 1');
|
|
216
217
|
if (anyData.rows?.length) {
|
|
217
218
|
// if isNotEmpty
|
|
218
|
-
this.logger.debug(
|
|
219
|
+
this.logger.debug(`New data uploaded since write checkpoint ${opId} - need new write checkpoint`);
|
|
219
220
|
return false;
|
|
220
221
|
}
|
|
221
222
|
const rs = await tx.execute("SELECT seq FROM sqlite_sequence WHERE name = 'ps_crud'");
|
|
222
223
|
if (!rs.rows?.length) {
|
|
223
224
|
// assert isNotEmpty
|
|
224
|
-
throw new Error('
|
|
225
|
+
throw new Error('SQLite Sequence should not be empty');
|
|
225
226
|
}
|
|
226
227
|
const seqAfter = rs.rows?.item(0)['seq'];
|
|
227
|
-
this.logger.debug('seqAfter', JSON.stringify(rs.rows?.item(0)));
|
|
228
228
|
if (seqAfter != seqBefore) {
|
|
229
|
-
this.logger.debug(
|
|
229
|
+
this.logger.debug(`New data uploaded since write checpoint ${opId} - need new write checkpoint (sequence updated)`);
|
|
230
230
|
// New crud data may have been uploaded since we got the checkpoint. Abort.
|
|
231
231
|
return false;
|
|
232
232
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
]);
|
|
236
|
-
this.logger.debug(['[updateLocalTarget] Response from updating target_op ', JSON.stringify(response)]);
|
|
233
|
+
this.logger.debug(`Updating target write checkpoint to ${opId}`);
|
|
234
|
+
await tx.execute("UPDATE ps_buckets SET target_op = CAST(? as INTEGER) WHERE name='$local'", [opId]);
|
|
237
235
|
return true;
|
|
238
236
|
});
|
|
239
237
|
}
|
|
@@ -3,7 +3,7 @@ import { type fetch } from 'cross-fetch';
|
|
|
3
3
|
import Logger, { ILogger } from 'js-logger';
|
|
4
4
|
import { DataStream } from '../../../utils/DataStream.js';
|
|
5
5
|
import { PowerSyncCredentials } from '../../connection/PowerSyncCredentials.js';
|
|
6
|
-
import {
|
|
6
|
+
import { StreamingSyncRequest } from './streaming-sync-types.js';
|
|
7
7
|
export type BSONImplementation = typeof BSON;
|
|
8
8
|
export type RemoteConnector = {
|
|
9
9
|
fetchCredentials: () => Promise<PowerSyncCredentials | null>;
|
|
@@ -120,11 +120,6 @@ export declare abstract class AbstractRemote {
|
|
|
120
120
|
*/
|
|
121
121
|
abstract getBSON(): Promise<BSONImplementation>;
|
|
122
122
|
protected createSocket(url: string): WebSocket;
|
|
123
|
-
/**
|
|
124
|
-
* Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
|
|
125
|
-
* sent by the server.
|
|
126
|
-
*/
|
|
127
|
-
socketStream(options: SocketSyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
|
|
128
123
|
/**
|
|
129
124
|
* Returns a data stream of sync line data.
|
|
130
125
|
*
|
|
@@ -133,10 +128,6 @@ export declare abstract class AbstractRemote {
|
|
|
133
128
|
* (required for compatibility with older sync services).
|
|
134
129
|
*/
|
|
135
130
|
socketStreamRaw<T>(options: SocketSyncStreamOptions, map: (buffer: Uint8Array) => T, bson?: typeof BSON): Promise<DataStream<T>>;
|
|
136
|
-
/**
|
|
137
|
-
* Connects to the sync/stream http endpoint, parsing lines as JSON.
|
|
138
|
-
*/
|
|
139
|
-
postStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
|
|
140
131
|
/**
|
|
141
132
|
* Connects to the sync/stream http endpoint, mapping and emitting each received string line.
|
|
142
133
|
*/
|
|
@@ -178,14 +178,6 @@ export class AbstractRemote {
|
|
|
178
178
|
createSocket(url) {
|
|
179
179
|
return new WebSocket(url);
|
|
180
180
|
}
|
|
181
|
-
/**
|
|
182
|
-
* Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
|
|
183
|
-
* sent by the server.
|
|
184
|
-
*/
|
|
185
|
-
async socketStream(options) {
|
|
186
|
-
const bson = await this.getBSON();
|
|
187
|
-
return await this.socketStreamRaw(options, (data) => bson.deserialize(data), bson);
|
|
188
|
-
}
|
|
189
181
|
/**
|
|
190
182
|
* Returns a data stream of sync line data.
|
|
191
183
|
*
|
|
@@ -363,14 +355,6 @@ export class AbstractRemote {
|
|
|
363
355
|
}
|
|
364
356
|
return stream;
|
|
365
357
|
}
|
|
366
|
-
/**
|
|
367
|
-
* Connects to the sync/stream http endpoint, parsing lines as JSON.
|
|
368
|
-
*/
|
|
369
|
-
async postStream(options) {
|
|
370
|
-
return await this.postStreamRaw(options, (line) => {
|
|
371
|
-
return JSON.parse(line);
|
|
372
|
-
});
|
|
373
|
-
}
|
|
374
358
|
/**
|
|
375
359
|
* Connects to the sync/stream http endpoint, mapping and emitting each received string line.
|
|
376
360
|
*/
|
|
@@ -166,7 +166,7 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
|
|
|
166
166
|
protected abortController: AbortController | null;
|
|
167
167
|
protected crudUpdateListener?: () => void;
|
|
168
168
|
protected streamingSyncPromise?: Promise<void>;
|
|
169
|
-
private
|
|
169
|
+
private isUploadingCrud;
|
|
170
170
|
private notifyCompletedUploads?;
|
|
171
171
|
syncStatus: SyncStatus;
|
|
172
172
|
triggerCrudUpload: () => void;
|
|
@@ -3,7 +3,7 @@ import { SyncStatus } from '../../../db/crud/SyncStatus.js';
|
|
|
3
3
|
import { FULL_SYNC_PRIORITY } from '../../../db/crud/SyncProgress.js';
|
|
4
4
|
import { AbortOperation } from '../../../utils/AbortOperation.js';
|
|
5
5
|
import { BaseObserver } from '../../../utils/BaseObserver.js';
|
|
6
|
-
import {
|
|
6
|
+
import { throttleLeadingTrailing } from '../../../utils/async.js';
|
|
7
7
|
import { PowerSyncControlCommand } from '../bucket/BucketStorageAdapter.js';
|
|
8
8
|
import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
|
|
9
9
|
import { FetchStrategy } from './AbstractRemote.js';
|
|
@@ -85,7 +85,7 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
|
85
85
|
abortController;
|
|
86
86
|
crudUpdateListener;
|
|
87
87
|
streamingSyncPromise;
|
|
88
|
-
|
|
88
|
+
isUploadingCrud = false;
|
|
89
89
|
notifyCompletedUploads;
|
|
90
90
|
syncStatus;
|
|
91
91
|
triggerCrudUpload;
|
|
@@ -103,15 +103,13 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
|
103
103
|
});
|
|
104
104
|
this.abortController = null;
|
|
105
105
|
this.triggerCrudUpload = throttleLeadingTrailing(() => {
|
|
106
|
-
if (!this.syncStatus.connected || this.
|
|
106
|
+
if (!this.syncStatus.connected || this.isUploadingCrud) {
|
|
107
107
|
return;
|
|
108
108
|
}
|
|
109
|
-
this.
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
resolve();
|
|
114
|
-
});
|
|
109
|
+
this.isUploadingCrud = true;
|
|
110
|
+
this._uploadAllCrud().finally(() => {
|
|
111
|
+
this.notifyCompletedUploads?.();
|
|
112
|
+
this.isUploadingCrud = false;
|
|
115
113
|
});
|
|
116
114
|
}, this.options.crudUploadThrottleMs);
|
|
117
115
|
}
|
|
@@ -169,11 +167,10 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
|
|
|
169
167
|
}
|
|
170
168
|
async getWriteCheckpoint() {
|
|
171
169
|
const clientId = await this.options.adapter.getClientId();
|
|
172
|
-
this.logger.debug(`Creating write checkpoint for ${clientId}`);
|
|
173
170
|
let path = `/write-checkpoint2.json?client_id=${clientId}`;
|
|
174
171
|
const response = await this.options.remote.get(path);
|
|
175
172
|
const checkpoint = response['data']['write_checkpoint'];
|
|
176
|
-
this.logger.debug(`
|
|
173
|
+
this.logger.debug(`Created write checkpoint: ${checkpoint}`);
|
|
177
174
|
return checkpoint;
|
|
178
175
|
}
|
|
179
176
|
async _uploadAllCrud() {
|
|
@@ -213,10 +210,10 @@ The next upload iteration will be delayed.`);
|
|
|
213
210
|
}
|
|
214
211
|
else {
|
|
215
212
|
// Uploading is completed
|
|
216
|
-
this.logger.debug('Upload complete, creating write checkpoint');
|
|
217
213
|
const neededUpdate = await this.options.adapter.updateLocalTarget(() => this.getWriteCheckpoint());
|
|
218
|
-
if (neededUpdate == false) {
|
|
219
|
-
this
|
|
214
|
+
if (neededUpdate == false && checkedCrudItem != null) {
|
|
215
|
+
// Only log this if there was something to upload
|
|
216
|
+
this.logger.debug('Upload complete, no write checkpoint needed.');
|
|
220
217
|
}
|
|
221
218
|
break;
|
|
222
219
|
}
|
|
@@ -368,6 +365,7 @@ The next upload iteration will be delayed.`);
|
|
|
368
365
|
});
|
|
369
366
|
}
|
|
370
367
|
finally {
|
|
368
|
+
this.notifyCompletedUploads = undefined;
|
|
371
369
|
if (!signal.aborted) {
|
|
372
370
|
nestedAbortController.abort(new AbortOperation('Closing sync stream network requests before retry.'));
|
|
373
371
|
nestedAbortController = new AbortController();
|
|
@@ -445,10 +443,9 @@ The next upload iteration will be delayed.`);
|
|
|
445
443
|
this.logger.debug('Streaming sync iteration started');
|
|
446
444
|
this.options.adapter.startSession();
|
|
447
445
|
let [req, bucketMap] = await this.collectLocalBucketState();
|
|
448
|
-
// These are compared by reference
|
|
449
446
|
let targetCheckpoint = null;
|
|
450
|
-
|
|
451
|
-
let
|
|
447
|
+
// A checkpoint that has been validated but not applied (e.g. due to pending local writes)
|
|
448
|
+
let pendingValidatedCheckpoint = null;
|
|
452
449
|
const clientId = await this.options.adapter.getClientId();
|
|
453
450
|
const usingFixedKeyFormat = await this.requireKeyFormat(false);
|
|
454
451
|
this.logger.debug('Requesting stream from server');
|
|
@@ -465,21 +462,55 @@ The next upload iteration will be delayed.`);
|
|
|
465
462
|
};
|
|
466
463
|
let stream;
|
|
467
464
|
if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
|
|
468
|
-
stream = await this.options.remote.
|
|
465
|
+
stream = await this.options.remote.postStreamRaw(syncOptions, (line) => {
|
|
466
|
+
if (typeof line == 'string') {
|
|
467
|
+
return JSON.parse(line);
|
|
468
|
+
}
|
|
469
|
+
else {
|
|
470
|
+
// Directly enqueued by us
|
|
471
|
+
return line;
|
|
472
|
+
}
|
|
473
|
+
});
|
|
469
474
|
}
|
|
470
475
|
else {
|
|
471
|
-
|
|
476
|
+
const bson = await this.options.remote.getBSON();
|
|
477
|
+
stream = await this.options.remote.socketStreamRaw({
|
|
472
478
|
...syncOptions,
|
|
473
479
|
...{ fetchStrategy: resolvedOptions.fetchStrategy }
|
|
474
|
-
})
|
|
480
|
+
}, (payload) => {
|
|
481
|
+
if (payload instanceof Uint8Array) {
|
|
482
|
+
return bson.deserialize(payload);
|
|
483
|
+
}
|
|
484
|
+
else {
|
|
485
|
+
// Directly enqueued by us
|
|
486
|
+
return payload;
|
|
487
|
+
}
|
|
488
|
+
}, bson);
|
|
475
489
|
}
|
|
476
490
|
this.logger.debug('Stream established. Processing events');
|
|
491
|
+
this.notifyCompletedUploads = () => {
|
|
492
|
+
if (!stream.closed) {
|
|
493
|
+
stream.enqueueData({ crud_upload_completed: null });
|
|
494
|
+
}
|
|
495
|
+
};
|
|
477
496
|
while (!stream.closed) {
|
|
478
497
|
const line = await stream.read();
|
|
479
498
|
if (!line) {
|
|
480
499
|
// The stream has closed while waiting
|
|
481
500
|
return;
|
|
482
501
|
}
|
|
502
|
+
if ('crud_upload_completed' in line) {
|
|
503
|
+
if (pendingValidatedCheckpoint != null) {
|
|
504
|
+
const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
|
|
505
|
+
if (applied) {
|
|
506
|
+
pendingValidatedCheckpoint = null;
|
|
507
|
+
}
|
|
508
|
+
else if (endIteration) {
|
|
509
|
+
break;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
continue;
|
|
513
|
+
}
|
|
483
514
|
// A connection is active and messages are being received
|
|
484
515
|
if (!this.syncStatus.connected) {
|
|
485
516
|
// There is a connection now
|
|
@@ -490,6 +521,8 @@ The next upload iteration will be delayed.`);
|
|
|
490
521
|
}
|
|
491
522
|
if (isStreamingSyncCheckpoint(line)) {
|
|
492
523
|
targetCheckpoint = line.checkpoint;
|
|
524
|
+
// New checkpoint - existing validated checkpoint is no longer valid
|
|
525
|
+
pendingValidatedCheckpoint = null;
|
|
493
526
|
const bucketsToDelete = new Set(bucketMap.keys());
|
|
494
527
|
const newBuckets = new Map();
|
|
495
528
|
for (const checksum of line.checkpoint.buckets) {
|
|
@@ -508,14 +541,20 @@ The next upload iteration will be delayed.`);
|
|
|
508
541
|
await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
|
|
509
542
|
}
|
|
510
543
|
else if (isStreamingSyncCheckpointComplete(line)) {
|
|
511
|
-
const result = await this.applyCheckpoint(targetCheckpoint
|
|
544
|
+
const result = await this.applyCheckpoint(targetCheckpoint);
|
|
512
545
|
if (result.endIteration) {
|
|
513
546
|
return;
|
|
514
547
|
}
|
|
515
|
-
else if (result.applied) {
|
|
516
|
-
|
|
548
|
+
else if (!result.applied) {
|
|
549
|
+
// "Could not apply checkpoint due to local data". We need to retry after
|
|
550
|
+
// finishing uploads.
|
|
551
|
+
pendingValidatedCheckpoint = targetCheckpoint;
|
|
552
|
+
}
|
|
553
|
+
else {
|
|
554
|
+
// Nothing to retry later. This would likely already be null from the last
|
|
555
|
+
// checksum or checksum_diff operation, but we make sure.
|
|
556
|
+
pendingValidatedCheckpoint = null;
|
|
517
557
|
}
|
|
518
|
-
validatedCheckpoint = targetCheckpoint;
|
|
519
558
|
}
|
|
520
559
|
else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
|
|
521
560
|
const priority = line.partial_checkpoint_complete.priority;
|
|
@@ -552,6 +591,8 @@ The next upload iteration will be delayed.`);
|
|
|
552
591
|
if (targetCheckpoint == null) {
|
|
553
592
|
throw new Error('Checkpoint diff without previous checkpoint');
|
|
554
593
|
}
|
|
594
|
+
// New checkpoint - existing validated checkpoint is no longer valid
|
|
595
|
+
pendingValidatedCheckpoint = null;
|
|
555
596
|
const diff = line.checkpoint_diff;
|
|
556
597
|
const newBuckets = new Map();
|
|
557
598
|
for (const checksum of targetCheckpoint.buckets) {
|
|
@@ -625,26 +666,7 @@ The next upload iteration will be delayed.`);
|
|
|
625
666
|
this.triggerCrudUpload();
|
|
626
667
|
}
|
|
627
668
|
else {
|
|
628
|
-
this.logger.debug('
|
|
629
|
-
if (targetCheckpoint === appliedCheckpoint) {
|
|
630
|
-
this.updateSyncStatus({
|
|
631
|
-
connected: true,
|
|
632
|
-
lastSyncedAt: new Date(),
|
|
633
|
-
priorityStatusEntries: [],
|
|
634
|
-
dataFlow: {
|
|
635
|
-
downloadError: undefined
|
|
636
|
-
}
|
|
637
|
-
});
|
|
638
|
-
}
|
|
639
|
-
else if (validatedCheckpoint === targetCheckpoint) {
|
|
640
|
-
const result = await this.applyCheckpoint(targetCheckpoint, signal);
|
|
641
|
-
if (result.endIteration) {
|
|
642
|
-
return;
|
|
643
|
-
}
|
|
644
|
-
else if (result.applied) {
|
|
645
|
-
appliedCheckpoint = targetCheckpoint;
|
|
646
|
-
}
|
|
647
|
-
}
|
|
669
|
+
this.logger.debug('Received unknown sync line', line);
|
|
648
670
|
}
|
|
649
671
|
}
|
|
650
672
|
this.logger.debug('Stream input empty');
|
|
@@ -860,46 +882,30 @@ The next upload iteration will be delayed.`);
|
|
|
860
882
|
}
|
|
861
883
|
});
|
|
862
884
|
}
|
|
863
|
-
async applyCheckpoint(checkpoint
|
|
885
|
+
async applyCheckpoint(checkpoint) {
|
|
864
886
|
let result = await this.options.adapter.syncLocalDatabase(checkpoint);
|
|
865
|
-
const pending = this.pendingCrudUpload;
|
|
866
887
|
if (!result.checkpointValid) {
|
|
867
|
-
this.logger.debug(
|
|
888
|
+
this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
|
|
868
889
|
// This means checksums failed. Start again with a new checkpoint.
|
|
869
890
|
// TODO: better back-off
|
|
870
891
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
871
892
|
return { applied: false, endIteration: true };
|
|
872
893
|
}
|
|
873
|
-
else if (!result.ready
|
|
874
|
-
|
|
875
|
-
// checkpoint, which prevented this checkpoint from applying. Wait for that to complete and
|
|
876
|
-
// try again.
|
|
877
|
-
this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. Waiting for in-progress upload before retrying.`);
|
|
878
|
-
await Promise.race([pending, onAbortPromise(abort)]);
|
|
879
|
-
this.logger.debug(`Pending uploads complete, retrying local checkpoint at ${checkpoint.last_op_id}`);
|
|
880
|
-
if (abort.aborted) {
|
|
881
|
-
return { applied: false, endIteration: true };
|
|
882
|
-
}
|
|
883
|
-
// Try again now that uploads have completed.
|
|
884
|
-
result = await this.options.adapter.syncLocalDatabase(checkpoint);
|
|
885
|
-
}
|
|
886
|
-
if (result.checkpointValid && result.ready) {
|
|
887
|
-
this.logger.debug('validated checkpoint', checkpoint);
|
|
888
|
-
this.updateSyncStatus({
|
|
889
|
-
connected: true,
|
|
890
|
-
lastSyncedAt: new Date(),
|
|
891
|
-
dataFlow: {
|
|
892
|
-
downloading: false,
|
|
893
|
-
downloadProgress: null,
|
|
894
|
-
downloadError: undefined
|
|
895
|
-
}
|
|
896
|
-
});
|
|
897
|
-
return { applied: true, endIteration: false };
|
|
898
|
-
}
|
|
899
|
-
else {
|
|
900
|
-
this.logger.debug('Could not apply checkpoint. Waiting for next sync complete line.');
|
|
894
|
+
else if (!result.ready) {
|
|
895
|
+
this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`);
|
|
901
896
|
return { applied: false, endIteration: false };
|
|
902
897
|
}
|
|
898
|
+
this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
|
|
899
|
+
this.updateSyncStatus({
|
|
900
|
+
connected: true,
|
|
901
|
+
lastSyncedAt: new Date(),
|
|
902
|
+
dataFlow: {
|
|
903
|
+
downloading: false,
|
|
904
|
+
downloadProgress: null,
|
|
905
|
+
downloadError: undefined
|
|
906
|
+
}
|
|
907
|
+
});
|
|
908
|
+
return { applied: true, endIteration: false };
|
|
903
909
|
}
|
|
904
910
|
updateSyncStatus(options) {
|
|
905
911
|
const updatedStatus = new SyncStatus({
|
|
@@ -101,6 +101,10 @@ export interface StreamingSyncKeepalive {
|
|
|
101
101
|
token_expires_in: number;
|
|
102
102
|
}
|
|
103
103
|
export type StreamingSyncLine = StreamingSyncDataJSON | StreamingSyncCheckpoint | StreamingSyncCheckpointDiff | StreamingSyncCheckpointComplete | StreamingSyncCheckpointPartiallyComplete | StreamingSyncKeepalive;
|
|
104
|
+
export type CrudUploadNotification = {
|
|
105
|
+
crud_upload_completed: null;
|
|
106
|
+
};
|
|
107
|
+
export type StreamingSyncLineOrCrudUploadComplete = StreamingSyncLine | CrudUploadNotification;
|
|
104
108
|
export interface BucketRequest {
|
|
105
109
|
name: string;
|
|
106
110
|
/**
|
package/lib/utils/async.d.ts
CHANGED
|
@@ -12,4 +12,3 @@ export declare function throttleTrailing(func: () => void, wait: number): () =>
|
|
|
12
12
|
* Roughly equivalent to lodash/throttle with {leading: true, trailing: true}
|
|
13
13
|
*/
|
|
14
14
|
export declare function throttleLeadingTrailing(func: () => void, wait: number): () => void;
|
|
15
|
-
export declare function onAbortPromise(signal: AbortSignal): Promise<void>;
|
package/lib/utils/async.js
CHANGED
|
@@ -43,13 +43,3 @@ export function throttleLeadingTrailing(func, wait) {
|
|
|
43
43
|
}
|
|
44
44
|
};
|
|
45
45
|
}
|
|
46
|
-
export function onAbortPromise(signal) {
|
|
47
|
-
return new Promise((resolve) => {
|
|
48
|
-
if (signal.aborted) {
|
|
49
|
-
resolve();
|
|
50
|
-
}
|
|
51
|
-
else {
|
|
52
|
-
signal.onabort = () => resolve();
|
|
53
|
-
}
|
|
54
|
-
});
|
|
55
|
-
}
|