@powersync/common 0.0.0-dev-20260414110516 → 0.0.0-dev-20260504100448
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.cjs +33 -665
- package/dist/bundle.cjs.map +1 -1
- package/dist/bundle.mjs +34 -654
- package/dist/bundle.mjs.map +1 -1
- package/dist/bundle.node.cjs +33 -665
- package/dist/bundle.node.cjs.map +1 -1
- package/dist/bundle.node.mjs +34 -654
- package/dist/bundle.node.mjs.map +1 -1
- package/dist/index.d.cts +22 -369
- package/legacy/sync_protocol.d.ts +103 -0
- package/lib/client/sync/bucket/BucketStorageAdapter.d.ts +1 -63
- package/lib/client/sync/bucket/BucketStorageAdapter.js.map +1 -1
- package/lib/client/sync/bucket/SqliteBucketStorage.d.ts +1 -28
- package/lib/client/sync/bucket/SqliteBucketStorage.js +0 -162
- package/lib/client/sync/bucket/SqliteBucketStorage.js.map +1 -1
- package/lib/client/sync/stream/AbstractRemote.d.ts +2 -12
- package/lib/client/sync/stream/AbstractRemote.js +3 -13
- package/lib/client/sync/stream/AbstractRemote.js.map +1 -1
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +12 -35
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +29 -337
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js.map +1 -1
- package/lib/client/sync/stream/JsonValue.d.ts +7 -0
- package/lib/client/sync/stream/JsonValue.js +2 -0
- package/lib/client/sync/stream/JsonValue.js.map +1 -0
- package/lib/client/sync/stream/core-instruction.d.ts +1 -2
- package/lib/client/sync/stream/core-instruction.js.map +1 -1
- package/lib/db/crud/SyncStatus.d.ts +0 -4
- package/lib/db/crud/SyncStatus.js +0 -4
- package/lib/db/crud/SyncStatus.js.map +1 -1
- package/lib/db/schema/RawTable.d.ts +0 -5
- package/lib/db/schema/Schema.d.ts +0 -2
- package/lib/db/schema/Schema.js +0 -2
- package/lib/db/schema/Schema.js.map +1 -1
- package/lib/index.d.ts +1 -5
- package/lib/index.js +1 -5
- package/lib/index.js.map +1 -1
- package/package.json +7 -4
- package/src/client/sync/bucket/BucketStorageAdapter.ts +1 -70
- package/src/client/sync/bucket/SqliteBucketStorage.ts +1 -197
- package/src/client/sync/stream/AbstractRemote.ts +5 -27
- package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +41 -407
- package/src/client/sync/stream/JsonValue.ts +8 -0
- package/src/client/sync/stream/core-instruction.ts +1 -2
- package/src/db/crud/SyncStatus.ts +0 -4
- package/src/db/schema/RawTable.ts +0 -5
- package/src/db/schema/Schema.ts +0 -2
- package/src/index.ts +1 -5
- package/lib/client/sync/bucket/OpType.d.ts +0 -16
- package/lib/client/sync/bucket/OpType.js +0 -23
- package/lib/client/sync/bucket/OpType.js.map +0 -1
- package/lib/client/sync/bucket/OplogEntry.d.ts +0 -23
- package/lib/client/sync/bucket/OplogEntry.js +0 -36
- package/lib/client/sync/bucket/OplogEntry.js.map +0 -1
- package/lib/client/sync/bucket/SyncDataBatch.d.ts +0 -6
- package/lib/client/sync/bucket/SyncDataBatch.js +0 -12
- package/lib/client/sync/bucket/SyncDataBatch.js.map +0 -1
- package/lib/client/sync/bucket/SyncDataBucket.d.ts +0 -40
- package/lib/client/sync/bucket/SyncDataBucket.js +0 -40
- package/lib/client/sync/bucket/SyncDataBucket.js.map +0 -1
- package/lib/client/sync/stream/streaming-sync-types.d.ts +0 -143
- package/lib/client/sync/stream/streaming-sync-types.js +0 -26
- package/lib/client/sync/stream/streaming-sync-types.js.map +0 -1
- package/src/client/sync/bucket/OpType.ts +0 -23
- package/src/client/sync/bucket/OplogEntry.ts +0 -50
- package/src/client/sync/bucket/SyncDataBatch.ts +0 -11
- package/src/client/sync/bucket/SyncDataBucket.ts +0 -49
- package/src/client/sync/stream/streaming-sync-types.ts +0 -210
|
@@ -1,42 +1,15 @@
|
|
|
1
1
|
import Logger, { ILogger } from 'js-logger';
|
|
2
2
|
|
|
3
|
-
import { InternalProgressInformation } from '../../../db/crud/SyncProgress.js';
|
|
4
3
|
import { SyncStatus, SyncStatusOptions } from '../../../db/crud/SyncStatus.js';
|
|
5
4
|
import { AbortOperation } from '../../../utils/AbortOperation.js';
|
|
6
5
|
import { BaseListener, BaseObserver, BaseObserverInterface, Disposable } from '../../../utils/BaseObserver.js';
|
|
7
6
|
import { throttleLeadingTrailing } from '../../../utils/async.js';
|
|
8
|
-
import {
|
|
9
|
-
BucketChecksum,
|
|
10
|
-
BucketDescription,
|
|
11
|
-
BucketStorageAdapter,
|
|
12
|
-
Checkpoint,
|
|
13
|
-
PowerSyncControlCommand
|
|
14
|
-
} from '../bucket/BucketStorageAdapter.js';
|
|
7
|
+
import { BucketStorageAdapter, PowerSyncControlCommand } from '../bucket/BucketStorageAdapter.js';
|
|
15
8
|
import { CrudEntry } from '../bucket/CrudEntry.js';
|
|
16
|
-
import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
|
|
17
9
|
import { AbstractRemote, FetchStrategy, SyncStreamOptions } from './AbstractRemote.js';
|
|
18
10
|
import { EstablishSyncStream, Instruction, coreStatusToJs } from './core-instruction.js';
|
|
19
|
-
import {
|
|
20
|
-
|
|
21
|
-
StreamingSyncLine,
|
|
22
|
-
StreamingSyncLineOrCrudUploadComplete,
|
|
23
|
-
StreamingSyncRequestParameterType,
|
|
24
|
-
isStreamingKeepalive,
|
|
25
|
-
isStreamingSyncCheckpoint,
|
|
26
|
-
isStreamingSyncCheckpointComplete,
|
|
27
|
-
isStreamingSyncCheckpointDiff,
|
|
28
|
-
isStreamingSyncCheckpointPartiallyComplete,
|
|
29
|
-
isStreamingSyncData
|
|
30
|
-
} from './streaming-sync-types.js';
|
|
31
|
-
import {
|
|
32
|
-
extractBsonObjects,
|
|
33
|
-
extractJsonLines,
|
|
34
|
-
injectable,
|
|
35
|
-
InjectableIterator,
|
|
36
|
-
map,
|
|
37
|
-
SimpleAsyncIterator
|
|
38
|
-
} from '../../../utils/stream_transform.js';
|
|
39
|
-
import type { BSON } from 'bson';
|
|
11
|
+
import { injectable, InjectableIterator, map, SimpleAsyncIterator } from '../../../utils/stream_transform.js';
|
|
12
|
+
import { StreamingSyncRequestParameterType } from './JsonValue.js';
|
|
40
13
|
|
|
41
14
|
export enum LockType {
|
|
42
15
|
CRUD = 'crud',
|
|
@@ -49,35 +22,21 @@ export enum SyncStreamConnectionMethod {
|
|
|
49
22
|
}
|
|
50
23
|
|
|
51
24
|
export enum SyncClientImplementation {
|
|
52
|
-
/**
|
|
53
|
-
* Decodes and handles sync lines received from the sync service in JavaScript.
|
|
54
|
-
*
|
|
55
|
-
* This is the default option.
|
|
56
|
-
*
|
|
57
|
-
* @deprecated We recommend the {@link RUST} client implementation for all apps. If you have issues with
|
|
58
|
-
* the Rust client, please file an issue or reach out to us. The JavaScript client will be removed in a future
|
|
59
|
-
* version of the PowerSync SDK.
|
|
60
|
-
*/
|
|
61
|
-
JAVASCRIPT = 'js',
|
|
62
25
|
/**
|
|
63
26
|
* This implementation offloads the sync line decoding and handling into the PowerSync
|
|
64
27
|
* core extension.
|
|
65
28
|
*
|
|
66
|
-
* This
|
|
67
|
-
* recommended client implementation for all apps.
|
|
29
|
+
* This is the only option, as an older JavaScript client implementation has been removed from the SDK.
|
|
68
30
|
*
|
|
69
31
|
* ## Compatibility warning
|
|
70
32
|
*
|
|
71
33
|
* The Rust sync client stores sync data in a format that is slightly different than the one used
|
|
72
|
-
* by the old
|
|
73
|
-
*
|
|
74
|
-
* Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
|
|
75
|
-
* the version introducing {@link RUST} as an option) also supports the new format, so you can switch
|
|
76
|
-
* back to {@link JAVASCRIPT} later.
|
|
34
|
+
* by the old JavaScript client. When adopting the {@link RUST} client on existing databases, the PowerSync SDK will
|
|
35
|
+
* migrate the format automatically.
|
|
77
36
|
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
37
|
+
* SDK versions supporting both the JavaScript and the Rust client support both formats with the JavaScript client
|
|
38
|
+
* implementaiton. However, downgrading to an SDK version that only supports the JavaScript client would not be
|
|
39
|
+
* possible anymore. Problematic SDK versions have been released before 2025-06-09.
|
|
81
40
|
*/
|
|
82
41
|
RUST = 'rust'
|
|
83
42
|
}
|
|
@@ -137,11 +96,7 @@ export interface BaseConnectionOptions {
|
|
|
137
96
|
appMetadata?: Record<string, string>;
|
|
138
97
|
|
|
139
98
|
/**
|
|
140
|
-
*
|
|
141
|
-
* service, or whether this work should be offloaded to the PowerSync core extension.
|
|
142
|
-
*
|
|
143
|
-
* This defaults to the JavaScript implementation ({@link SyncClientImplementation.JAVASCRIPT})
|
|
144
|
-
* since the ({@link SyncClientImplementation.RUST}) implementation is experimental at the moment.
|
|
99
|
+
* @deprecated The Rust sync client is used unconditionally, so this option can't be configured.
|
|
145
100
|
*/
|
|
146
101
|
clientImplementation?: SyncClientImplementation;
|
|
147
102
|
|
|
@@ -207,15 +162,14 @@ export interface StreamingSyncImplementation
|
|
|
207
162
|
*/
|
|
208
163
|
disconnect(): Promise<void>;
|
|
209
164
|
getWriteCheckpoint: () => Promise<string>;
|
|
210
|
-
hasCompletedSync: () => Promise<boolean>;
|
|
211
165
|
isConnected: boolean;
|
|
212
|
-
lastSyncedAt?: Date;
|
|
213
166
|
syncStatus: SyncStatus;
|
|
214
167
|
triggerCrudUpload: () => void;
|
|
215
168
|
waitForReady(): Promise<void>;
|
|
216
169
|
waitForStatus(status: SyncStatusOptions): Promise<void>;
|
|
217
170
|
waitUntilStatusMatches(predicate: (status: SyncStatus) => boolean): Promise<void>;
|
|
218
171
|
updateSubscriptions(subscriptions: SubscribedStream[]): void;
|
|
172
|
+
markConnectionMayHaveChanged(): void;
|
|
219
173
|
}
|
|
220
174
|
|
|
221
175
|
export const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
|
|
@@ -253,16 +207,16 @@ export abstract class AbstractStreamingSyncImplementation
|
|
|
253
207
|
extends BaseObserver<StreamingSyncImplementationListener>
|
|
254
208
|
implements StreamingSyncImplementation
|
|
255
209
|
{
|
|
256
|
-
protected _lastSyncedAt: Date | null;
|
|
257
210
|
protected options: AbstractStreamingSyncImplementationOptions;
|
|
258
211
|
protected abortController: AbortController | null;
|
|
259
212
|
// In rare cases, mostly for tests, uploads can be triggered without being properly connected.
|
|
260
213
|
// This allows ensuring that all upload processes can be aborted.
|
|
261
|
-
protected uploadAbortController: AbortController |
|
|
214
|
+
protected uploadAbortController: AbortController | undefined;
|
|
262
215
|
protected crudUpdateListener?: () => void;
|
|
263
216
|
protected streamingSyncPromise?: Promise<void>;
|
|
264
217
|
protected logger: ILogger;
|
|
265
218
|
private activeStreams: SubscribedStream[];
|
|
219
|
+
private connectionMayHaveChanged = false;
|
|
266
220
|
|
|
267
221
|
private isUploadingCrud: boolean = false;
|
|
268
222
|
private notifyCompletedUploads?: () => void;
|
|
@@ -309,7 +263,7 @@ export abstract class AbstractStreamingSyncImplementation
|
|
|
309
263
|
* Match only the partial status options provided in the
|
|
310
264
|
* matching status
|
|
311
265
|
*/
|
|
312
|
-
const matchPartialObject = (compA: object, compB:
|
|
266
|
+
const matchPartialObject = (compA: object, compB: any): any => {
|
|
313
267
|
return Object.entries(compA).every(([key, value]) => {
|
|
314
268
|
const comparisonBValue = compB[key];
|
|
315
269
|
if (typeof value == 'object' && typeof comparisonBValue == 'object') {
|
|
@@ -359,10 +313,6 @@ export abstract class AbstractStreamingSyncImplementation
|
|
|
359
313
|
|
|
360
314
|
abstract obtainLock<T>(lockOptions: LockOptions<T>): Promise<T>;
|
|
361
315
|
|
|
362
|
-
async hasCompletedSync() {
|
|
363
|
-
return this.options.adapter.hasCompletedSync();
|
|
364
|
-
}
|
|
365
|
-
|
|
366
316
|
async getWriteCheckpoint(): Promise<string> {
|
|
367
317
|
const clientId = await this.options.adapter.getClientId();
|
|
368
318
|
let path = `/write-checkpoint2.json?client_id=${clientId}`;
|
|
@@ -433,7 +383,7 @@ The next upload iteration will be delayed.`);
|
|
|
433
383
|
this.updateSyncStatus({
|
|
434
384
|
dataFlow: {
|
|
435
385
|
uploading: false,
|
|
436
|
-
uploadError: ex
|
|
386
|
+
uploadError: ex as Error
|
|
437
387
|
}
|
|
438
388
|
});
|
|
439
389
|
await this.delayRetry(controller.signal);
|
|
@@ -442,7 +392,7 @@ The next upload iteration will be delayed.`);
|
|
|
442
392
|
break;
|
|
443
393
|
}
|
|
444
394
|
this.logger.debug(
|
|
445
|
-
`Caught exception when uploading. Upload will retry after a delay. Exception: ${ex.message}`
|
|
395
|
+
`Caught exception when uploading. Upload will retry after a delay. Exception: ${(ex as Error).message}`
|
|
446
396
|
);
|
|
447
397
|
} finally {
|
|
448
398
|
this.updateSyncStatus({
|
|
@@ -452,7 +402,7 @@ The next upload iteration will be delayed.`);
|
|
|
452
402
|
});
|
|
453
403
|
}
|
|
454
404
|
}
|
|
455
|
-
this.uploadAbortController =
|
|
405
|
+
this.uploadAbortController = undefined;
|
|
456
406
|
}
|
|
457
407
|
});
|
|
458
408
|
}
|
|
@@ -579,13 +529,17 @@ The next upload iteration will be delayed.`);
|
|
|
579
529
|
this.logger.warn(ex);
|
|
580
530
|
shouldDelayRetry = false;
|
|
581
531
|
// A disconnect was requested, we should not delay since there is no explicit retry
|
|
532
|
+
} else if (this.connectionMayHaveChanged && (ex as Error).message?.indexOf('No iteration is active') >= 0) {
|
|
533
|
+
this.connectionMayHaveChanged = false;
|
|
534
|
+
this.logger.info('Sync error after changed connection, retrying immediately');
|
|
535
|
+
shouldDelayRetry = false;
|
|
582
536
|
} else {
|
|
583
537
|
this.logger.error(ex);
|
|
584
538
|
}
|
|
585
539
|
|
|
586
540
|
this.updateSyncStatus({
|
|
587
541
|
dataFlow: {
|
|
588
|
-
downloadError: ex
|
|
542
|
+
downloadError: ex as Error
|
|
589
543
|
}
|
|
590
544
|
});
|
|
591
545
|
} finally {
|
|
@@ -614,18 +568,15 @@ The next upload iteration will be delayed.`);
|
|
|
614
568
|
this.updateSyncStatus({ connected: false, connecting: false });
|
|
615
569
|
}
|
|
616
570
|
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
after: entry.op_id
|
|
622
|
-
}));
|
|
623
|
-
const localDescriptions = new Map<string, BucketDescription | null>();
|
|
624
|
-
for (const entry of bucketEntries) {
|
|
625
|
-
localDescriptions.set(entry.bucket, null);
|
|
626
|
-
}
|
|
571
|
+
markConnectionMayHaveChanged() {
|
|
572
|
+
// By setting this field, we'll immediately retry if the next sync event causes an error triggered by us not having
|
|
573
|
+
// an active sync iteration on the connection in use.
|
|
574
|
+
this.connectionMayHaveChanged = true;
|
|
627
575
|
|
|
628
|
-
|
|
576
|
+
// This triggers a `powersync_control` invocation if a sync iteration is currently active. This is a cheap call to
|
|
577
|
+
// make when no subscriptions have actually changed, we're mainly interested in this immediately throwing if no
|
|
578
|
+
// iteration is active. That allows us to reconnect ASAP, instead of having to wait for the next sync line.
|
|
579
|
+
this.handleActiveStreamsChange?.();
|
|
629
580
|
}
|
|
630
581
|
|
|
631
582
|
/**
|
|
@@ -678,13 +629,8 @@ The next upload iteration will be delayed.`);
|
|
|
678
629
|
const clientImplementation = resolvedOptions.clientImplementation;
|
|
679
630
|
this.updateSyncStatus({ clientImplementation });
|
|
680
631
|
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
return null;
|
|
684
|
-
} else {
|
|
685
|
-
await this.requireKeyFormat(true);
|
|
686
|
-
return await this.rustSyncIteration(signal, resolvedOptions);
|
|
687
|
-
}
|
|
632
|
+
await this.requireKeyFormat(true);
|
|
633
|
+
return await this.rustSyncIteration(signal, resolvedOptions);
|
|
688
634
|
}
|
|
689
635
|
});
|
|
690
636
|
}
|
|
@@ -692,260 +638,18 @@ The next upload iteration will be delayed.`);
|
|
|
692
638
|
private async receiveSyncLines(data: {
|
|
693
639
|
options: SyncStreamOptions;
|
|
694
640
|
connection: RequiredPowerSyncConnectionOptions;
|
|
695
|
-
bson?: typeof BSON;
|
|
696
641
|
}): Promise<SimpleAsyncIterator<Uint8Array | string>> {
|
|
697
|
-
const { options, connection
|
|
642
|
+
const { options, connection } = data;
|
|
698
643
|
const remote = this.options.remote;
|
|
699
644
|
|
|
700
645
|
if (connection.connectionMethod == SyncStreamConnectionMethod.HTTP) {
|
|
701
646
|
return await remote.fetchStream(options);
|
|
702
647
|
} else {
|
|
703
|
-
return await this.options.remote.socketStreamRaw(
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
},
|
|
708
|
-
bson
|
|
709
|
-
);
|
|
710
|
-
}
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
private async legacyStreamingSyncIteration(signal: AbortSignal, resolvedOptions: RequiredPowerSyncConnectionOptions) {
|
|
714
|
-
const rawTables = resolvedOptions.serializedSchema?.raw_tables;
|
|
715
|
-
if (rawTables != null && rawTables.length) {
|
|
716
|
-
this.logger.warn('Raw tables require the Rust-based sync client. The JS client will ignore them.');
|
|
717
|
-
}
|
|
718
|
-
if (this.activeStreams.length) {
|
|
719
|
-
this.logger.error('Sync streams require `clientImplementation: SyncClientImplementation.RUST` when connecting.');
|
|
720
|
-
}
|
|
721
|
-
|
|
722
|
-
this.logger.debug('Streaming sync iteration started');
|
|
723
|
-
this.options.adapter.startSession();
|
|
724
|
-
let [req, bucketMap] = await this.collectLocalBucketState();
|
|
725
|
-
|
|
726
|
-
let targetCheckpoint: Checkpoint | null = null;
|
|
727
|
-
// A checkpoint that has been validated but not applied (e.g. due to pending local writes)
|
|
728
|
-
let pendingValidatedCheckpoint: Checkpoint | null = null;
|
|
729
|
-
|
|
730
|
-
const clientId = await this.options.adapter.getClientId();
|
|
731
|
-
const usingFixedKeyFormat = await this.requireKeyFormat(false);
|
|
732
|
-
|
|
733
|
-
this.logger.debug('Requesting stream from server');
|
|
734
|
-
|
|
735
|
-
const syncOptions: SyncStreamOptions = {
|
|
736
|
-
path: '/sync/stream',
|
|
737
|
-
abortSignal: signal,
|
|
738
|
-
data: {
|
|
739
|
-
buckets: req,
|
|
740
|
-
include_checksum: true,
|
|
741
|
-
raw_data: true,
|
|
742
|
-
parameters: resolvedOptions.params,
|
|
743
|
-
app_metadata: resolvedOptions.appMetadata,
|
|
744
|
-
client_id: clientId
|
|
745
|
-
}
|
|
746
|
-
};
|
|
747
|
-
|
|
748
|
-
const bson = await this.options.remote.getBSON();
|
|
749
|
-
const source = await this.receiveSyncLines({
|
|
750
|
-
options: syncOptions,
|
|
751
|
-
connection: resolvedOptions,
|
|
752
|
-
bson
|
|
753
|
-
});
|
|
754
|
-
const stream: InjectableIterator<StreamingSyncLineOrCrudUploadComplete> = injectable(
|
|
755
|
-
map(source, (line) => {
|
|
756
|
-
if (typeof line == 'string') {
|
|
757
|
-
return JSON.parse(line) as StreamingSyncLine;
|
|
758
|
-
} else {
|
|
759
|
-
return bson.deserialize(line) as StreamingSyncLine;
|
|
760
|
-
}
|
|
761
|
-
})
|
|
762
|
-
);
|
|
763
|
-
|
|
764
|
-
this.logger.debug('Stream established. Processing events');
|
|
765
|
-
|
|
766
|
-
this.notifyCompletedUploads = () => {
|
|
767
|
-
stream.inject({ crud_upload_completed: null });
|
|
768
|
-
};
|
|
769
|
-
|
|
770
|
-
while (true) {
|
|
771
|
-
const { value: line, done } = await stream.next();
|
|
772
|
-
if (done) {
|
|
773
|
-
// The stream has closed while waiting
|
|
774
|
-
return;
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
if ('crud_upload_completed' in line) {
|
|
778
|
-
if (pendingValidatedCheckpoint != null) {
|
|
779
|
-
const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
|
|
780
|
-
if (applied) {
|
|
781
|
-
pendingValidatedCheckpoint = null;
|
|
782
|
-
} else if (endIteration) {
|
|
783
|
-
break;
|
|
784
|
-
}
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
continue;
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
// A connection is active and messages are being received
|
|
791
|
-
if (!this.syncStatus.connected) {
|
|
792
|
-
// There is a connection now
|
|
793
|
-
Promise.resolve().then(() => this.triggerCrudUpload());
|
|
794
|
-
this.updateSyncStatus({
|
|
795
|
-
connected: true
|
|
796
|
-
});
|
|
797
|
-
}
|
|
798
|
-
|
|
799
|
-
if (isStreamingSyncCheckpoint(line)) {
|
|
800
|
-
targetCheckpoint = line.checkpoint;
|
|
801
|
-
// New checkpoint - existing validated checkpoint is no longer valid
|
|
802
|
-
pendingValidatedCheckpoint = null;
|
|
803
|
-
const bucketsToDelete = new Set<string>(bucketMap.keys());
|
|
804
|
-
const newBuckets = new Map<string, BucketDescription>();
|
|
805
|
-
for (const checksum of line.checkpoint.buckets) {
|
|
806
|
-
newBuckets.set(checksum.bucket, {
|
|
807
|
-
name: checksum.bucket,
|
|
808
|
-
priority: checksum.priority ?? FALLBACK_PRIORITY
|
|
809
|
-
});
|
|
810
|
-
bucketsToDelete.delete(checksum.bucket);
|
|
811
|
-
}
|
|
812
|
-
if (bucketsToDelete.size > 0) {
|
|
813
|
-
this.logger.debug('Removing buckets', [...bucketsToDelete]);
|
|
814
|
-
}
|
|
815
|
-
bucketMap = newBuckets;
|
|
816
|
-
await this.options.adapter.removeBuckets([...bucketsToDelete]);
|
|
817
|
-
await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
|
|
818
|
-
await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
|
|
819
|
-
} else if (isStreamingSyncCheckpointComplete(line)) {
|
|
820
|
-
const result = await this.applyCheckpoint(targetCheckpoint!);
|
|
821
|
-
if (result.endIteration) {
|
|
822
|
-
return;
|
|
823
|
-
} else if (!result.applied) {
|
|
824
|
-
// "Could not apply checkpoint due to local data". We need to retry after
|
|
825
|
-
// finishing uploads.
|
|
826
|
-
pendingValidatedCheckpoint = targetCheckpoint;
|
|
827
|
-
} else {
|
|
828
|
-
// Nothing to retry later. This would likely already be null from the last
|
|
829
|
-
// checksum or checksum_diff operation, but we make sure.
|
|
830
|
-
pendingValidatedCheckpoint = null;
|
|
831
|
-
}
|
|
832
|
-
} else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
|
|
833
|
-
const priority = line.partial_checkpoint_complete.priority;
|
|
834
|
-
this.logger.debug('Partial checkpoint complete', priority);
|
|
835
|
-
const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint!, priority);
|
|
836
|
-
if (!result.checkpointValid) {
|
|
837
|
-
// This means checksums failed. Start again with a new checkpoint.
|
|
838
|
-
// TODO: better back-off
|
|
839
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
840
|
-
return;
|
|
841
|
-
} else if (!result.ready) {
|
|
842
|
-
// If we have pending uploads, we can't complete new checkpoints outside of priority 0.
|
|
843
|
-
// We'll resolve this for a complete checkpoint.
|
|
844
|
-
} else {
|
|
845
|
-
// We'll keep on downloading, but can report that this priority is synced now.
|
|
846
|
-
this.logger.debug('partial checkpoint validation succeeded');
|
|
847
|
-
|
|
848
|
-
// All states with a higher priority can be deleted since this partial sync includes them.
|
|
849
|
-
const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
|
|
850
|
-
priorityStates.push({
|
|
851
|
-
priority,
|
|
852
|
-
lastSyncedAt: new Date(),
|
|
853
|
-
hasSynced: true
|
|
854
|
-
});
|
|
855
|
-
|
|
856
|
-
this.updateSyncStatus({
|
|
857
|
-
connected: true,
|
|
858
|
-
priorityStatusEntries: priorityStates
|
|
859
|
-
});
|
|
860
|
-
}
|
|
861
|
-
} else if (isStreamingSyncCheckpointDiff(line)) {
|
|
862
|
-
// TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
|
|
863
|
-
if (targetCheckpoint == null) {
|
|
864
|
-
throw new Error('Checkpoint diff without previous checkpoint');
|
|
865
|
-
}
|
|
866
|
-
// New checkpoint - existing validated checkpoint is no longer valid
|
|
867
|
-
pendingValidatedCheckpoint = null;
|
|
868
|
-
const diff = line.checkpoint_diff;
|
|
869
|
-
const newBuckets = new Map<string, BucketChecksum>();
|
|
870
|
-
for (const checksum of targetCheckpoint.buckets) {
|
|
871
|
-
newBuckets.set(checksum.bucket, checksum);
|
|
872
|
-
}
|
|
873
|
-
for (const checksum of diff.updated_buckets) {
|
|
874
|
-
newBuckets.set(checksum.bucket, checksum);
|
|
875
|
-
}
|
|
876
|
-
for (const bucket of diff.removed_buckets) {
|
|
877
|
-
newBuckets.delete(bucket);
|
|
878
|
-
}
|
|
879
|
-
|
|
880
|
-
const newCheckpoint: Checkpoint = {
|
|
881
|
-
last_op_id: diff.last_op_id,
|
|
882
|
-
buckets: [...newBuckets.values()],
|
|
883
|
-
write_checkpoint: diff.write_checkpoint
|
|
884
|
-
};
|
|
885
|
-
targetCheckpoint = newCheckpoint;
|
|
886
|
-
await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
|
|
887
|
-
|
|
888
|
-
bucketMap = new Map();
|
|
889
|
-
newBuckets.forEach((checksum, name) =>
|
|
890
|
-
bucketMap.set(name, {
|
|
891
|
-
name: checksum.bucket,
|
|
892
|
-
priority: checksum.priority ?? FALLBACK_PRIORITY
|
|
893
|
-
})
|
|
894
|
-
);
|
|
895
|
-
|
|
896
|
-
const bucketsToDelete = diff.removed_buckets;
|
|
897
|
-
if (bucketsToDelete.length > 0) {
|
|
898
|
-
this.logger.debug('Remove buckets', bucketsToDelete);
|
|
899
|
-
}
|
|
900
|
-
await this.options.adapter.removeBuckets(bucketsToDelete);
|
|
901
|
-
await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
|
|
902
|
-
} else if (isStreamingSyncData(line)) {
|
|
903
|
-
const { data } = line;
|
|
904
|
-
const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
|
|
905
|
-
let updatedProgress: InternalProgressInformation | null = null;
|
|
906
|
-
if (previousProgress) {
|
|
907
|
-
updatedProgress = { ...previousProgress };
|
|
908
|
-
const progressForBucket = updatedProgress[data.bucket];
|
|
909
|
-
if (progressForBucket) {
|
|
910
|
-
updatedProgress[data.bucket] = {
|
|
911
|
-
...progressForBucket,
|
|
912
|
-
since_last: progressForBucket.since_last + data.data.length
|
|
913
|
-
};
|
|
914
|
-
}
|
|
915
|
-
}
|
|
916
|
-
|
|
917
|
-
this.updateSyncStatus({
|
|
918
|
-
dataFlow: {
|
|
919
|
-
downloading: true,
|
|
920
|
-
downloadProgress: updatedProgress
|
|
921
|
-
}
|
|
922
|
-
});
|
|
923
|
-
await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] }, usingFixedKeyFormat);
|
|
924
|
-
} else if (isStreamingKeepalive(line)) {
|
|
925
|
-
const remaining_seconds = line.token_expires_in;
|
|
926
|
-
if (remaining_seconds == 0) {
|
|
927
|
-
// Connection would be closed automatically right after this
|
|
928
|
-
this.logger.debug('Token expiring; reconnect');
|
|
929
|
-
/**
|
|
930
|
-
* For a rare case where the backend connector does not update the token
|
|
931
|
-
* (uses the same one), this should have some delay.
|
|
932
|
-
*/
|
|
933
|
-
await this.delayRetry();
|
|
934
|
-
return;
|
|
935
|
-
} else if (remaining_seconds < 30) {
|
|
936
|
-
this.logger.debug('Token will expire soon; reconnect');
|
|
937
|
-
// Pre-emptively refresh the token
|
|
938
|
-
this.options.remote.invalidateCredentials();
|
|
939
|
-
return;
|
|
940
|
-
}
|
|
941
|
-
this.triggerCrudUpload();
|
|
942
|
-
} else {
|
|
943
|
-
this.logger.debug('Received unknown sync line', line);
|
|
944
|
-
}
|
|
648
|
+
return await this.options.remote.socketStreamRaw({
|
|
649
|
+
...options,
|
|
650
|
+
...{ fetchStrategy: connection.fetchStrategy }
|
|
651
|
+
});
|
|
945
652
|
}
|
|
946
|
-
this.logger.debug('Stream input empty');
|
|
947
|
-
// Connection closed. Likely due to auth issue.
|
|
948
|
-
return;
|
|
949
653
|
}
|
|
950
654
|
|
|
951
655
|
private async rustSyncIteration(
|
|
@@ -1041,6 +745,10 @@ The next upload iteration will be delayed.`);
|
|
|
1041
745
|
rawResponse
|
|
1042
746
|
);
|
|
1043
747
|
|
|
748
|
+
if (op != PowerSyncControlCommand.STOP) {
|
|
749
|
+
// Evidently we have a working connection here, otherwise powersync_control would have failed.
|
|
750
|
+
syncImplementation.connectionMayHaveChanged = false;
|
|
751
|
+
}
|
|
1044
752
|
await handleInstructions(JSON.parse(rawResponse));
|
|
1045
753
|
}
|
|
1046
754
|
|
|
@@ -1133,80 +841,6 @@ The next upload iteration will be delayed.`);
|
|
|
1133
841
|
return { immediateRestart: hideDisconnectOnRestart };
|
|
1134
842
|
}
|
|
1135
843
|
|
|
1136
|
-
private async updateSyncStatusForStartingCheckpoint(checkpoint: Checkpoint) {
|
|
1137
|
-
const localProgress = await this.options.adapter.getBucketOperationProgress();
|
|
1138
|
-
const progress: InternalProgressInformation = {};
|
|
1139
|
-
let invalidated = false;
|
|
1140
|
-
|
|
1141
|
-
for (const bucket of checkpoint.buckets) {
|
|
1142
|
-
const savedProgress = localProgress[bucket.bucket];
|
|
1143
|
-
const atLast = savedProgress?.atLast ?? 0;
|
|
1144
|
-
const sinceLast = savedProgress?.sinceLast ?? 0;
|
|
1145
|
-
|
|
1146
|
-
progress[bucket.bucket] = {
|
|
1147
|
-
// The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
|
|
1148
|
-
// will use by default.
|
|
1149
|
-
priority: bucket.priority ?? 3,
|
|
1150
|
-
at_last: atLast,
|
|
1151
|
-
since_last: sinceLast,
|
|
1152
|
-
target_count: bucket.count ?? 0
|
|
1153
|
-
};
|
|
1154
|
-
|
|
1155
|
-
if (bucket.count != null && bucket.count < atLast + sinceLast) {
|
|
1156
|
-
// Either due to a defrag / sync rule deploy or a compaction operation, the size
|
|
1157
|
-
// of the bucket shrank so much that the local ops exceed the ops in the updated
|
|
1158
|
-
// bucket. We can't prossibly report progress in this case (it would overshoot 100%).
|
|
1159
|
-
invalidated = true;
|
|
1160
|
-
}
|
|
1161
|
-
}
|
|
1162
|
-
|
|
1163
|
-
if (invalidated) {
|
|
1164
|
-
for (const bucket in progress) {
|
|
1165
|
-
const bucketProgress = progress[bucket];
|
|
1166
|
-
bucketProgress.at_last = 0;
|
|
1167
|
-
bucketProgress.since_last = 0;
|
|
1168
|
-
}
|
|
1169
|
-
}
|
|
1170
|
-
|
|
1171
|
-
this.updateSyncStatus({
|
|
1172
|
-
dataFlow: {
|
|
1173
|
-
downloading: true,
|
|
1174
|
-
downloadProgress: progress
|
|
1175
|
-
}
|
|
1176
|
-
});
|
|
1177
|
-
}
|
|
1178
|
-
|
|
1179
|
-
private async applyCheckpoint(checkpoint: Checkpoint) {
|
|
1180
|
-
let result = await this.options.adapter.syncLocalDatabase(checkpoint);
|
|
1181
|
-
|
|
1182
|
-
if (!result.checkpointValid) {
|
|
1183
|
-
this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
|
|
1184
|
-
// This means checksums failed. Start again with a new checkpoint.
|
|
1185
|
-
// TODO: better back-off
|
|
1186
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
1187
|
-
return { applied: false, endIteration: true };
|
|
1188
|
-
} else if (!result.ready) {
|
|
1189
|
-
this.logger.debug(
|
|
1190
|
-
`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`
|
|
1191
|
-
);
|
|
1192
|
-
|
|
1193
|
-
return { applied: false, endIteration: false };
|
|
1194
|
-
}
|
|
1195
|
-
|
|
1196
|
-
this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
|
|
1197
|
-
this.updateSyncStatus({
|
|
1198
|
-
connected: true,
|
|
1199
|
-
lastSyncedAt: new Date(),
|
|
1200
|
-
dataFlow: {
|
|
1201
|
-
downloading: false,
|
|
1202
|
-
downloadProgress: null,
|
|
1203
|
-
downloadError: undefined
|
|
1204
|
-
}
|
|
1205
|
-
});
|
|
1206
|
-
|
|
1207
|
-
return { applied: true, endIteration: false };
|
|
1208
|
-
}
|
|
1209
|
-
|
|
1210
844
|
protected updateSyncStatus(options: SyncStatusOptions) {
|
|
1211
845
|
const updatedStatus = new SyncStatus({
|
|
1212
846
|
connected: options.connected ?? this.syncStatus.connected,
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { StreamingSyncRequest } from './streaming-sync-types.js';
|
|
2
1
|
import * as sync_status from '../../../db/crud/SyncStatus.js';
|
|
3
2
|
import { FULL_SYNC_PRIORITY } from '../../../db/crud/SyncProgress.js';
|
|
4
3
|
|
|
@@ -21,7 +20,7 @@ export interface LogLine {
|
|
|
21
20
|
}
|
|
22
21
|
|
|
23
22
|
export interface EstablishSyncStream {
|
|
24
|
-
request:
|
|
23
|
+
request: unknown;
|
|
25
24
|
}
|
|
26
25
|
|
|
27
26
|
export interface UpdateSyncStatus {
|
|
@@ -122,8 +122,6 @@ export class SyncStatus {
|
|
|
122
122
|
*
|
|
123
123
|
* This returns null when the database is currently being opened and we don't have reliable information about all
|
|
124
124
|
* included streams yet.
|
|
125
|
-
*
|
|
126
|
-
* @experimental Sync streams are currently in alpha.
|
|
127
125
|
*/
|
|
128
126
|
get syncStreams(): SyncStreamStatus[] | undefined {
|
|
129
127
|
return this.options.dataFlow?.internalStreamSubscriptions?.map((core) => new SyncStreamStatusView(this, core));
|
|
@@ -131,8 +129,6 @@ export class SyncStatus {
|
|
|
131
129
|
|
|
132
130
|
/**
|
|
133
131
|
* If the `stream` appears in {@link syncStreams}, returns the current status for that stream.
|
|
134
|
-
*
|
|
135
|
-
* @experimental Sync streams are currently in alpha.
|
|
136
132
|
*/
|
|
137
133
|
forStream(stream: SyncStreamDescription): SyncStreamStatus | undefined {
|
|
138
134
|
const asJson = JSON.stringify(stream.parameters);
|
|
@@ -9,11 +9,6 @@ import { TableOrRawTableOptions } from './Table.js';
|
|
|
9
9
|
* To collect local writes to raw tables with PowerSync, custom triggers are required. See
|
|
10
10
|
* {@link https://docs.powersync.com/usage/use-case-examples/raw-tables the documentation} for details and an example on
|
|
11
11
|
* using raw tables.
|
|
12
|
-
*
|
|
13
|
-
* Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
|
|
14
|
-
*
|
|
15
|
-
* @experimental Please note that this feature is experimental at the moment, and not covered by PowerSync semver or
|
|
16
|
-
* stability guarantees.
|
|
17
12
|
*/
|
|
18
13
|
export type RawTableType = RawTableTypeWithStatements | InferredRawTableType;
|
|
19
14
|
|
package/src/db/schema/Schema.ts
CHANGED
|
@@ -51,10 +51,8 @@ export class Schema<S extends SchemaType = SchemaType> {
|
|
|
51
51
|
* developer instead of automatically by PowerSync.
|
|
52
52
|
* Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
|
|
53
53
|
* using client-side table and column constraints.
|
|
54
|
-
* Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
|
|
55
54
|
*
|
|
56
55
|
* @param tables An object of (table name, raw table definition) entries.
|
|
57
|
-
* @experimental Note that the raw tables API is still experimental and may change in the future.
|
|
58
56
|
*/
|
|
59
57
|
withRawTables(tables: Record<string, RawTableType>) {
|
|
60
58
|
for (const [name, rawTableDefinition] of Object.entries(tables)) {
|
package/src/index.ts
CHANGED
|
@@ -20,14 +20,10 @@ export * from './client/sync/bucket/BucketStorageAdapter.js';
|
|
|
20
20
|
export * from './client/sync/bucket/CrudBatch.js';
|
|
21
21
|
export { CrudEntry, OpId, UpdateType } from './client/sync/bucket/CrudEntry.js';
|
|
22
22
|
export * from './client/sync/bucket/CrudTransaction.js';
|
|
23
|
-
export * from './client/sync/bucket/OplogEntry.js';
|
|
24
|
-
export * from './client/sync/bucket/OpType.js';
|
|
25
23
|
export * from './client/sync/bucket/SqliteBucketStorage.js';
|
|
26
|
-
export * from './client/sync/bucket/SyncDataBatch.js';
|
|
27
|
-
export * from './client/sync/bucket/SyncDataBucket.js';
|
|
28
24
|
export * from './client/sync/stream/AbstractRemote.js';
|
|
29
25
|
export * from './client/sync/stream/AbstractStreamingSyncImplementation.js';
|
|
30
|
-
export * from './client/sync/stream/
|
|
26
|
+
export * from './client/sync/stream/JsonValue.js';
|
|
31
27
|
export * from './client/sync/sync-streams.js';
|
|
32
28
|
|
|
33
29
|
export * from './client/ConnectionManager.js';
|