@powersync/common 1.32.0 → 1.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.mjs +5 -5
- package/lib/client/sync/bucket/BucketStorageAdapter.d.ts +15 -1
- package/lib/client/sync/bucket/BucketStorageAdapter.js +9 -0
- package/lib/client/sync/bucket/CrudEntry.d.ts +2 -0
- package/lib/client/sync/bucket/CrudEntry.js +13 -2
- package/lib/client/sync/bucket/OplogEntry.d.ts +4 -4
- package/lib/client/sync/bucket/OplogEntry.js +5 -3
- package/lib/client/sync/bucket/SqliteBucketStorage.d.ts +6 -2
- package/lib/client/sync/bucket/SqliteBucketStorage.js +24 -2
- package/lib/client/sync/bucket/SyncDataBucket.d.ts +1 -1
- package/lib/client/sync/bucket/SyncDataBucket.js +2 -2
- package/lib/client/sync/stream/AbstractRemote.d.ts +16 -3
- package/lib/client/sync/stream/AbstractRemote.js +68 -78
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +69 -0
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +431 -204
- package/lib/client/sync/stream/core-instruction.d.ts +53 -0
- package/lib/client/sync/stream/core-instruction.js +1 -0
- package/lib/db/crud/SyncProgress.d.ts +2 -6
- package/lib/db/crud/SyncProgress.js +2 -2
- package/package.json +1 -2
|
@@ -51,17 +51,27 @@ export declare enum PSInternalTable {
|
|
|
51
51
|
OPLOG = "ps_oplog",
|
|
52
52
|
UNTYPED = "ps_untyped"
|
|
53
53
|
}
|
|
54
|
+
export declare enum PowerSyncControlCommand {
|
|
55
|
+
PROCESS_TEXT_LINE = "line_text",
|
|
56
|
+
PROCESS_BSON_LINE = "line_binary",
|
|
57
|
+
STOP = "stop",
|
|
58
|
+
START = "start",
|
|
59
|
+
NOTIFY_TOKEN_REFRESHED = "refreshed_token",
|
|
60
|
+
NOTIFY_CRUD_UPLOAD_COMPLETED = "completed_upload"
|
|
61
|
+
}
|
|
54
62
|
export interface BucketStorageListener extends BaseListener {
|
|
55
63
|
crudUpdate: () => void;
|
|
56
64
|
}
|
|
57
65
|
export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener>, Disposable {
|
|
58
66
|
init(): Promise<void>;
|
|
59
|
-
saveSyncData(batch: SyncDataBatch): Promise<void>;
|
|
67
|
+
saveSyncData(batch: SyncDataBatch, fixedKeyFormat?: boolean): Promise<void>;
|
|
60
68
|
removeBuckets(buckets: string[]): Promise<void>;
|
|
61
69
|
setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
|
|
62
70
|
startSession(): void;
|
|
63
71
|
getBucketStates(): Promise<BucketState[]>;
|
|
64
72
|
getBucketOperationProgress(): Promise<BucketOperationProgress>;
|
|
73
|
+
hasMigratedSubkeys(): Promise<boolean>;
|
|
74
|
+
migrateToFixedSubkeys(): Promise<void>;
|
|
65
75
|
syncLocalDatabase(checkpoint: Checkpoint, priority?: number): Promise<{
|
|
66
76
|
checkpointValid: boolean;
|
|
67
77
|
ready: boolean;
|
|
@@ -85,4 +95,8 @@ export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener
|
|
|
85
95
|
* Get an unique client id.
|
|
86
96
|
*/
|
|
87
97
|
getClientId(): Promise<string>;
|
|
98
|
+
/**
|
|
99
|
+
* Invokes the `powersync_control` function for the sync client.
|
|
100
|
+
*/
|
|
101
|
+
control(op: PowerSyncControlCommand, payload: string | ArrayBuffer | null): Promise<string>;
|
|
88
102
|
}
|
|
@@ -6,3 +6,12 @@ export var PSInternalTable;
|
|
|
6
6
|
PSInternalTable["OPLOG"] = "ps_oplog";
|
|
7
7
|
PSInternalTable["UNTYPED"] = "ps_untyped";
|
|
8
8
|
})(PSInternalTable || (PSInternalTable = {}));
|
|
9
|
+
export var PowerSyncControlCommand;
|
|
10
|
+
(function (PowerSyncControlCommand) {
|
|
11
|
+
PowerSyncControlCommand["PROCESS_TEXT_LINE"] = "line_text";
|
|
12
|
+
PowerSyncControlCommand["PROCESS_BSON_LINE"] = "line_binary";
|
|
13
|
+
PowerSyncControlCommand["STOP"] = "stop";
|
|
14
|
+
PowerSyncControlCommand["START"] = "start";
|
|
15
|
+
PowerSyncControlCommand["NOTIFY_TOKEN_REFRESHED"] = "refreshed_token";
|
|
16
|
+
PowerSyncControlCommand["NOTIFY_CRUD_UPLOAD_COMPLETED"] = "completed_upload";
|
|
17
|
+
})(PowerSyncControlCommand || (PowerSyncControlCommand = {}));
|
|
@@ -74,7 +74,9 @@ export class CrudEntry {
|
|
|
74
74
|
type: this.table,
|
|
75
75
|
id: this.id,
|
|
76
76
|
tx_id: this.transactionId,
|
|
77
|
-
data: this.opData
|
|
77
|
+
data: this.opData,
|
|
78
|
+
old: this.previousValues,
|
|
79
|
+
metadata: this.metadata
|
|
78
80
|
};
|
|
79
81
|
}
|
|
80
82
|
equals(entry) {
|
|
@@ -93,6 +95,15 @@ export class CrudEntry {
|
|
|
93
95
|
* Generates an array for use in deep comparison operations
|
|
94
96
|
*/
|
|
95
97
|
toComparisonArray() {
|
|
96
|
-
return [
|
|
98
|
+
return [
|
|
99
|
+
this.transactionId,
|
|
100
|
+
this.clientId,
|
|
101
|
+
this.op,
|
|
102
|
+
this.table,
|
|
103
|
+
this.id,
|
|
104
|
+
this.opData,
|
|
105
|
+
this.previousValues,
|
|
106
|
+
this.metadata
|
|
107
|
+
];
|
|
97
108
|
}
|
|
98
109
|
}
|
|
@@ -7,17 +7,17 @@ export interface OplogEntryJSON {
|
|
|
7
7
|
object_type?: string;
|
|
8
8
|
op_id: string;
|
|
9
9
|
op: OpTypeJSON;
|
|
10
|
-
subkey?: string
|
|
10
|
+
subkey?: string;
|
|
11
11
|
}
|
|
12
12
|
export declare class OplogEntry {
|
|
13
13
|
op_id: OpId;
|
|
14
14
|
op: OpType;
|
|
15
15
|
checksum: number;
|
|
16
|
-
subkey
|
|
16
|
+
subkey?: string | undefined;
|
|
17
17
|
object_type?: string | undefined;
|
|
18
18
|
object_id?: string | undefined;
|
|
19
19
|
data?: string | undefined;
|
|
20
20
|
static fromRow(row: OplogEntryJSON): OplogEntry;
|
|
21
|
-
constructor(op_id: OpId, op: OpType, checksum: number, subkey
|
|
22
|
-
toJSON(): OplogEntryJSON;
|
|
21
|
+
constructor(op_id: OpId, op: OpType, checksum: number, subkey?: string | undefined, object_type?: string | undefined, object_id?: string | undefined, data?: string | undefined);
|
|
22
|
+
toJSON(fixedKeyEncoding?: boolean): OplogEntryJSON;
|
|
23
23
|
}
|
|
@@ -8,7 +8,7 @@ export class OplogEntry {
|
|
|
8
8
|
object_id;
|
|
9
9
|
data;
|
|
10
10
|
static fromRow(row) {
|
|
11
|
-
return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum,
|
|
11
|
+
return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, row.subkey, row.object_type, row.object_id, row.data);
|
|
12
12
|
}
|
|
13
13
|
constructor(op_id, op, checksum, subkey, object_type, object_id, data) {
|
|
14
14
|
this.op_id = op_id;
|
|
@@ -19,7 +19,7 @@ export class OplogEntry {
|
|
|
19
19
|
this.object_id = object_id;
|
|
20
20
|
this.data = data;
|
|
21
21
|
}
|
|
22
|
-
toJSON() {
|
|
22
|
+
toJSON(fixedKeyEncoding = false) {
|
|
23
23
|
return {
|
|
24
24
|
op_id: this.op_id,
|
|
25
25
|
op: this.op.toJSON(),
|
|
@@ -27,7 +27,9 @@ export class OplogEntry {
|
|
|
27
27
|
object_id: this.object_id,
|
|
28
28
|
checksum: this.checksum,
|
|
29
29
|
data: this.data,
|
|
30
|
-
|
|
30
|
+
// Older versions of the JS SDK used to always JSON.stringify here. That has always been wrong,
|
|
31
|
+
// but we need to migrate gradually to not break existing databases.
|
|
32
|
+
subkey: fixedKeyEncoding ? this.subkey : JSON.stringify(this.subkey)
|
|
31
33
|
};
|
|
32
34
|
}
|
|
33
35
|
}
|
|
@@ -2,7 +2,7 @@ import { Mutex } from 'async-mutex';
|
|
|
2
2
|
import { ILogger } from 'js-logger';
|
|
3
3
|
import { DBAdapter, Transaction } from '../../../db/DBAdapter.js';
|
|
4
4
|
import { BaseObserver } from '../../../utils/BaseObserver.js';
|
|
5
|
-
import { BucketOperationProgress, BucketState, BucketStorageAdapter, BucketStorageListener, Checkpoint, SyncLocalDatabaseResult } from './BucketStorageAdapter.js';
|
|
5
|
+
import { BucketOperationProgress, BucketState, BucketStorageAdapter, BucketStorageListener, Checkpoint, PowerSyncControlCommand, SyncLocalDatabaseResult } from './BucketStorageAdapter.js';
|
|
6
6
|
import { CrudBatch } from './CrudBatch.js';
|
|
7
7
|
import { CrudEntry } from './CrudEntry.js';
|
|
8
8
|
import { SyncDataBatch } from './SyncDataBatch.js';
|
|
@@ -31,7 +31,7 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
|
|
|
31
31
|
startSession(): void;
|
|
32
32
|
getBucketStates(): Promise<BucketState[]>;
|
|
33
33
|
getBucketOperationProgress(): Promise<BucketOperationProgress>;
|
|
34
|
-
saveSyncData(batch: SyncDataBatch): Promise<void>;
|
|
34
|
+
saveSyncData(batch: SyncDataBatch, fixedKeyFormat?: boolean): Promise<void>;
|
|
35
35
|
removeBuckets(buckets: string[]): Promise<void>;
|
|
36
36
|
/**
|
|
37
37
|
* Mark a bucket for deletion.
|
|
@@ -68,4 +68,8 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
|
|
|
68
68
|
* Set a target checkpoint.
|
|
69
69
|
*/
|
|
70
70
|
setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
|
|
71
|
+
control(op: PowerSyncControlCommand, payload: string | ArrayBuffer | null): Promise<string>;
|
|
72
|
+
hasMigratedSubkeys(): Promise<boolean>;
|
|
73
|
+
migrateToFixedSubkeys(): Promise<void>;
|
|
74
|
+
static _subkeyMigrationKey: string;
|
|
71
75
|
}
|
|
@@ -70,13 +70,13 @@ export class SqliteBucketStorage extends BaseObserver {
|
|
|
70
70
|
const rows = await this.db.getAll('SELECT name, count_at_last, count_since_last FROM ps_buckets');
|
|
71
71
|
return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
|
|
72
72
|
}
|
|
73
|
-
async saveSyncData(batch) {
|
|
73
|
+
async saveSyncData(batch, fixedKeyFormat = false) {
|
|
74
74
|
await this.writeTransaction(async (tx) => {
|
|
75
75
|
let count = 0;
|
|
76
76
|
for (const b of batch.buckets) {
|
|
77
77
|
const result = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
|
|
78
78
|
'save',
|
|
79
|
-
JSON.stringify({ buckets: [b.toJSON()] })
|
|
79
|
+
JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
|
|
80
80
|
]);
|
|
81
81
|
this.logger.debug('saveSyncData', JSON.stringify(result));
|
|
82
82
|
count += b.data.length;
|
|
@@ -339,6 +339,28 @@ export class SqliteBucketStorage extends BaseObserver {
|
|
|
339
339
|
async setTargetCheckpoint(checkpoint) {
|
|
340
340
|
// No-op for now
|
|
341
341
|
}
|
|
342
|
+
async control(op, payload) {
|
|
343
|
+
return await this.writeTransaction(async (tx) => {
|
|
344
|
+
const [[raw]] = await tx.executeRaw('SELECT powersync_control(?, ?)', [op, payload]);
|
|
345
|
+
return raw;
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
async hasMigratedSubkeys() {
|
|
349
|
+
const { r } = await this.db.get('SELECT EXISTS(SELECT * FROM ps_kv WHERE key = ?) as r', [
|
|
350
|
+
SqliteBucketStorage._subkeyMigrationKey
|
|
351
|
+
]);
|
|
352
|
+
return r != 0;
|
|
353
|
+
}
|
|
354
|
+
async migrateToFixedSubkeys() {
|
|
355
|
+
await this.writeTransaction(async (tx) => {
|
|
356
|
+
await tx.execute('UPDATE ps_oplog SET key = powersync_remove_duplicate_key_encoding(key);');
|
|
357
|
+
await tx.execute('INSERT OR REPLACE INTO ps_kv (key, value) VALUES (?, ?);', [
|
|
358
|
+
SqliteBucketStorage._subkeyMigrationKey,
|
|
359
|
+
'1'
|
|
360
|
+
]);
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
static _subkeyMigrationKey = 'powersync_js_migrated_subkeys';
|
|
342
364
|
}
|
|
343
365
|
function hasMatchingPriority(priority, bucket) {
|
|
344
366
|
return bucket.priority != null && bucket.priority <= priority;
|
|
@@ -27,13 +27,13 @@ export class SyncDataBucket {
|
|
|
27
27
|
this.after = after;
|
|
28
28
|
this.next_after = next_after;
|
|
29
29
|
}
|
|
30
|
-
toJSON() {
|
|
30
|
+
toJSON(fixedKeyEncoding = false) {
|
|
31
31
|
return {
|
|
32
32
|
bucket: this.bucket,
|
|
33
33
|
has_more: this.has_more,
|
|
34
34
|
after: this.after,
|
|
35
35
|
next_after: this.next_after,
|
|
36
|
-
data: this.data.map((entry) => entry.toJSON())
|
|
36
|
+
data: this.data.map((entry) => entry.toJSON(fixedKeyEncoding))
|
|
37
37
|
};
|
|
38
38
|
}
|
|
39
39
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { BSON } from 'bson';
|
|
2
|
+
import { Buffer } from 'buffer';
|
|
2
3
|
import { type fetch } from 'cross-fetch';
|
|
3
4
|
import Logger, { ILogger } from 'js-logger';
|
|
4
5
|
import { DataStream } from '../../../utils/DataStream.js';
|
|
@@ -115,18 +116,30 @@ export declare abstract class AbstractRemote {
|
|
|
115
116
|
}>;
|
|
116
117
|
post(path: string, data: any, headers?: Record<string, string>): Promise<any>;
|
|
117
118
|
get(path: string, headers?: Record<string, string>): Promise<any>;
|
|
118
|
-
postStreaming(path: string, data: any, headers?: Record<string, string>, signal?: AbortSignal): Promise<any>;
|
|
119
119
|
/**
|
|
120
120
|
* Provides a BSON implementation. The import nature of this varies depending on the platform
|
|
121
121
|
*/
|
|
122
122
|
abstract getBSON(): Promise<BSONImplementation>;
|
|
123
123
|
protected createSocket(url: string): WebSocket;
|
|
124
124
|
/**
|
|
125
|
-
* Connects to the sync/stream websocket endpoint
|
|
125
|
+
* Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
|
|
126
|
+
* sent by the server.
|
|
126
127
|
*/
|
|
127
128
|
socketStream(options: SocketSyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
|
|
128
129
|
/**
|
|
129
|
-
*
|
|
130
|
+
* Returns a data stream of sync line data.
|
|
131
|
+
*
|
|
132
|
+
* @param map Maps received payload frames to the typed event value.
|
|
133
|
+
* @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
|
|
134
|
+
* (required for compatibility with older sync services).
|
|
135
|
+
*/
|
|
136
|
+
socketStreamRaw<T>(options: SocketSyncStreamOptions, map: (buffer: Buffer) => T, bson?: typeof BSON): Promise<DataStream>;
|
|
137
|
+
/**
|
|
138
|
+
* Connects to the sync/stream http endpoint, parsing lines as JSON.
|
|
130
139
|
*/
|
|
131
140
|
postStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
|
|
141
|
+
/**
|
|
142
|
+
* Connects to the sync/stream http endpoint, mapping and emitting each received string line.
|
|
143
|
+
*/
|
|
144
|
+
postStreamRaw<T>(options: SyncStreamOptions, mapLine: (line: string) => T): Promise<DataStream<T>>;
|
|
132
145
|
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { Buffer } from 'buffer';
|
|
2
|
-
import ndjsonStream from 'can-ndjson-stream';
|
|
3
2
|
import Logger from 'js-logger';
|
|
4
3
|
import { RSocketConnector } from 'rsocket-core';
|
|
5
4
|
import PACKAGE from '../../../../package.json' with { type: 'json' };
|
|
@@ -172,41 +171,39 @@ export class AbstractRemote {
|
|
|
172
171
|
}
|
|
173
172
|
return res.json();
|
|
174
173
|
}
|
|
175
|
-
async postStreaming(path, data, headers = {}, signal) {
|
|
176
|
-
const request = await this.buildRequest(path);
|
|
177
|
-
const res = await this.fetch(request.url, {
|
|
178
|
-
method: 'POST',
|
|
179
|
-
headers: { ...headers, ...request.headers },
|
|
180
|
-
body: JSON.stringify(data),
|
|
181
|
-
signal,
|
|
182
|
-
cache: 'no-store'
|
|
183
|
-
}).catch((ex) => {
|
|
184
|
-
this.logger.error(`Caught ex when POST streaming to ${path}`, ex);
|
|
185
|
-
throw ex;
|
|
186
|
-
});
|
|
187
|
-
if (res.status === 401) {
|
|
188
|
-
this.invalidateCredentials();
|
|
189
|
-
}
|
|
190
|
-
if (!res.ok) {
|
|
191
|
-
const text = await res.text();
|
|
192
|
-
this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
|
|
193
|
-
const error = new Error(`HTTP ${res.statusText}: ${text}`);
|
|
194
|
-
error.status = res.status;
|
|
195
|
-
throw error;
|
|
196
|
-
}
|
|
197
|
-
return res;
|
|
198
|
-
}
|
|
199
174
|
createSocket(url) {
|
|
200
175
|
return new WebSocket(url);
|
|
201
176
|
}
|
|
202
177
|
/**
|
|
203
|
-
* Connects to the sync/stream websocket endpoint
|
|
178
|
+
* Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
|
|
179
|
+
* sent by the server.
|
|
204
180
|
*/
|
|
205
181
|
async socketStream(options) {
|
|
182
|
+
const bson = await this.getBSON();
|
|
183
|
+
return await this.socketStreamRaw(options, (data) => bson.deserialize(data), bson);
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Returns a data stream of sync line data.
|
|
187
|
+
*
|
|
188
|
+
* @param map Maps received payload frames to the typed event value.
|
|
189
|
+
* @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
|
|
190
|
+
* (required for compatibility with older sync services).
|
|
191
|
+
*/
|
|
192
|
+
async socketStreamRaw(options, map, bson) {
|
|
206
193
|
const { path, fetchStrategy = FetchStrategy.Buffered } = options;
|
|
194
|
+
const mimeType = bson == null ? 'application/json' : 'application/bson';
|
|
195
|
+
function toBuffer(js) {
|
|
196
|
+
let contents;
|
|
197
|
+
if (bson != null) {
|
|
198
|
+
contents = bson.serialize(js);
|
|
199
|
+
}
|
|
200
|
+
else {
|
|
201
|
+
contents = JSON.stringify(js);
|
|
202
|
+
}
|
|
203
|
+
return Buffer.from(contents);
|
|
204
|
+
}
|
|
207
205
|
const syncQueueRequestSize = fetchStrategy == FetchStrategy.Buffered ? 10 : 1;
|
|
208
206
|
const request = await this.buildRequest(path);
|
|
209
|
-
const bson = await this.getBSON();
|
|
210
207
|
// Add the user agent in the setup payload - we can't set custom
|
|
211
208
|
// headers with websockets on web. The browser userAgent is however added
|
|
212
209
|
// automatically as a header.
|
|
@@ -222,14 +219,14 @@ export class AbstractRemote {
|
|
|
222
219
|
setup: {
|
|
223
220
|
keepAlive: KEEP_ALIVE_MS,
|
|
224
221
|
lifetime: KEEP_ALIVE_LIFETIME_MS,
|
|
225
|
-
dataMimeType:
|
|
226
|
-
metadataMimeType:
|
|
222
|
+
dataMimeType: mimeType,
|
|
223
|
+
metadataMimeType: mimeType,
|
|
227
224
|
payload: {
|
|
228
225
|
data: null,
|
|
229
|
-
metadata:
|
|
226
|
+
metadata: toBuffer({
|
|
230
227
|
token: request.headers.Authorization,
|
|
231
228
|
user_agent: userAgent
|
|
232
|
-
})
|
|
229
|
+
})
|
|
233
230
|
}
|
|
234
231
|
}
|
|
235
232
|
});
|
|
@@ -268,10 +265,10 @@ export class AbstractRemote {
|
|
|
268
265
|
const socket = await new Promise((resolve, reject) => {
|
|
269
266
|
let connectionEstablished = false;
|
|
270
267
|
const res = rsocket.requestStream({
|
|
271
|
-
data:
|
|
272
|
-
metadata:
|
|
268
|
+
data: toBuffer(options.data),
|
|
269
|
+
metadata: toBuffer({
|
|
273
270
|
path
|
|
274
|
-
})
|
|
271
|
+
})
|
|
275
272
|
}, syncQueueRequestSize, // The initial N amount
|
|
276
273
|
{
|
|
277
274
|
onError: (e) => {
|
|
@@ -310,8 +307,7 @@ export class AbstractRemote {
|
|
|
310
307
|
if (!data) {
|
|
311
308
|
return;
|
|
312
309
|
}
|
|
313
|
-
|
|
314
|
-
stream.enqueueData(deserializedData);
|
|
310
|
+
stream.enqueueData(map(data));
|
|
315
311
|
},
|
|
316
312
|
onComplete: () => {
|
|
317
313
|
stream.close();
|
|
@@ -347,9 +343,17 @@ export class AbstractRemote {
|
|
|
347
343
|
return stream;
|
|
348
344
|
}
|
|
349
345
|
/**
|
|
350
|
-
* Connects to the sync/stream http endpoint
|
|
346
|
+
* Connects to the sync/stream http endpoint, parsing lines as JSON.
|
|
351
347
|
*/
|
|
352
348
|
async postStream(options) {
|
|
349
|
+
return await this.postStreamRaw(options, (line) => {
|
|
350
|
+
return JSON.parse(line);
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Connects to the sync/stream http endpoint, mapping and emitting each received string line.
|
|
355
|
+
*/
|
|
356
|
+
async postStreamRaw(options, mapLine) {
|
|
353
357
|
const { data, path, headers, abortSignal } = options;
|
|
354
358
|
const request = await this.buildRequest(path);
|
|
355
359
|
/**
|
|
@@ -395,11 +399,8 @@ export class AbstractRemote {
|
|
|
395
399
|
error.status = res.status;
|
|
396
400
|
throw error;
|
|
397
401
|
}
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
* This will intercept the readable stream and close the stream if
|
|
401
|
-
* aborted.
|
|
402
|
-
*/
|
|
402
|
+
// Create a new stream splitting the response at line endings while also handling cancellations
|
|
403
|
+
// by closing the reader.
|
|
403
404
|
const reader = res.body.getReader();
|
|
404
405
|
// This will close the network request and read stream
|
|
405
406
|
const closeReader = async () => {
|
|
@@ -414,49 +415,38 @@ export class AbstractRemote {
|
|
|
414
415
|
abortSignal?.addEventListener('abort', () => {
|
|
415
416
|
closeReader();
|
|
416
417
|
});
|
|
417
|
-
const
|
|
418
|
-
|
|
419
|
-
const processStream = async () => {
|
|
420
|
-
while (!abortSignal?.aborted) {
|
|
421
|
-
try {
|
|
422
|
-
const { done, value } = await reader.read();
|
|
423
|
-
// When no more data needs to be consumed, close the stream
|
|
424
|
-
if (done) {
|
|
425
|
-
break;
|
|
426
|
-
}
|
|
427
|
-
// Enqueue the next data chunk into our target stream
|
|
428
|
-
controller.enqueue(value);
|
|
429
|
-
}
|
|
430
|
-
catch (ex) {
|
|
431
|
-
this.logger.error('Caught exception when reading sync stream', ex);
|
|
432
|
-
break;
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
if (!abortSignal?.aborted) {
|
|
436
|
-
// Close the downstream readable stream
|
|
437
|
-
await closeReader();
|
|
438
|
-
}
|
|
439
|
-
controller.close();
|
|
440
|
-
};
|
|
441
|
-
processStream();
|
|
442
|
-
}
|
|
443
|
-
});
|
|
444
|
-
const jsonS = ndjsonStream(outputStream);
|
|
418
|
+
const decoder = new TextDecoder();
|
|
419
|
+
let buffer = '';
|
|
445
420
|
const stream = new DataStream({
|
|
446
421
|
logger: this.logger
|
|
447
422
|
});
|
|
448
|
-
const r = jsonS.getReader();
|
|
449
423
|
const l = stream.registerListener({
|
|
450
424
|
lowWater: async () => {
|
|
451
425
|
try {
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
426
|
+
let didCompleteLine = false;
|
|
427
|
+
while (!didCompleteLine) {
|
|
428
|
+
const { done, value } = await reader.read();
|
|
429
|
+
if (done) {
|
|
430
|
+
const remaining = buffer.trim();
|
|
431
|
+
if (remaining.length != 0) {
|
|
432
|
+
stream.enqueueData(mapLine(remaining));
|
|
433
|
+
}
|
|
434
|
+
stream.close();
|
|
435
|
+
await closeReader();
|
|
436
|
+
return;
|
|
437
|
+
}
|
|
438
|
+
const data = decoder.decode(value, { stream: true });
|
|
439
|
+
buffer += data;
|
|
440
|
+
const lines = buffer.split('\n');
|
|
441
|
+
for (var i = 0; i < lines.length - 1; i++) {
|
|
442
|
+
var l = lines[i].trim();
|
|
443
|
+
if (l.length > 0) {
|
|
444
|
+
stream.enqueueData(mapLine(l));
|
|
445
|
+
didCompleteLine = true;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
buffer = lines[lines.length - 1];
|
|
458
449
|
}
|
|
459
|
-
stream.enqueueData(value);
|
|
460
450
|
}
|
|
461
451
|
catch (ex) {
|
|
462
452
|
stream.close();
|
|
@@ -12,6 +12,48 @@ export declare enum SyncStreamConnectionMethod {
|
|
|
12
12
|
HTTP = "http",
|
|
13
13
|
WEB_SOCKET = "web-socket"
|
|
14
14
|
}
|
|
15
|
+
export declare enum SyncClientImplementation {
|
|
16
|
+
/**
|
|
17
|
+
* Decodes and handles sync lines received from the sync service in JavaScript.
|
|
18
|
+
*
|
|
19
|
+
* This is the default option.
|
|
20
|
+
*
|
|
21
|
+
* @deprecated Don't use {@link SyncClientImplementation.JAVASCRIPT} directly. Instead, use
|
|
22
|
+
* {@link DEFAULT_SYNC_CLIENT_IMPLEMENTATION} or omit the option. The explicit choice to use
|
|
23
|
+
* the JavaScript-based sync implementation will be removed from a future version of the SDK.
|
|
24
|
+
*/
|
|
25
|
+
JAVASCRIPT = "js",
|
|
26
|
+
/**
|
|
27
|
+
* This implementation offloads the sync line decoding and handling into the PowerSync
|
|
28
|
+
* core extension.
|
|
29
|
+
*
|
|
30
|
+
* @experimental
|
|
31
|
+
* While this implementation is more performant than {@link SyncClientImplementation.JAVASCRIPT},
|
|
32
|
+
* it has seen less real-world testing and is marked as __experimental__ at the moment.
|
|
33
|
+
*
|
|
34
|
+
* ## Compatibility warning
|
|
35
|
+
*
|
|
36
|
+
* The Rust sync client stores sync data in a format that is slightly different than the one used
|
|
37
|
+
* by the old {@link JAVASCRIPT} implementation. When adopting the {@link RUST} client on existing
|
|
38
|
+
* databases, the PowerSync SDK will migrate the format automatically.
|
|
39
|
+
* Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
|
|
40
|
+
* the version introducing {@link RUST} as an option) also supports the new format, so you can switch
|
|
41
|
+
* back to {@link JAVASCRIPT} later.
|
|
42
|
+
*
|
|
43
|
+
* __However__: Upgrading the SDK version, then adopting {@link RUST} as a sync client and later
|
|
44
|
+
* downgrading the SDK to an older version (necessarily using the JavaScript-based implementation then)
|
|
45
|
+
* can lead to sync issues.
|
|
46
|
+
*/
|
|
47
|
+
RUST = "rust"
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* The default {@link SyncClientImplementation} to use.
|
|
51
|
+
*
|
|
52
|
+
* Please use this field instead of {@link SyncClientImplementation.JAVASCRIPT} directly. A future version
|
|
53
|
+
* of the PowerSync SDK will enable {@link SyncClientImplementation.RUST} by default and remove the JavaScript
|
|
54
|
+
* option.
|
|
55
|
+
*/
|
|
56
|
+
export declare const DEFAULT_SYNC_CLIENT_IMPLEMENTATION = SyncClientImplementation.JAVASCRIPT;
|
|
15
57
|
/**
|
|
16
58
|
* Abstract Lock to be implemented by various JS environments
|
|
17
59
|
*/
|
|
@@ -50,6 +92,14 @@ export interface PowerSyncConnectionOptions extends BaseConnectionOptions, Addit
|
|
|
50
92
|
}
|
|
51
93
|
/** @internal */
|
|
52
94
|
export interface BaseConnectionOptions {
|
|
95
|
+
/**
|
|
96
|
+
* Whether to use a JavaScript implementation to handle received sync lines from the sync
|
|
97
|
+
* service, or whether this work should be offloaded to the PowerSync core extension.
|
|
98
|
+
*
|
|
99
|
+
* This defaults to the JavaScript implementation ({@link SyncClientImplementation.JAVASCRIPT})
|
|
100
|
+
* since the ({@link SyncClientImplementation.RUST}) implementation is experimental at the moment.
|
|
101
|
+
*/
|
|
102
|
+
clientImplementation?: SyncClientImplementation;
|
|
53
103
|
/**
|
|
54
104
|
* The connection method to use when streaming updates from
|
|
55
105
|
* the PowerSync backend instance.
|
|
@@ -117,6 +167,7 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
|
|
|
117
167
|
protected crudUpdateListener?: () => void;
|
|
118
168
|
protected streamingSyncPromise?: Promise<void>;
|
|
119
169
|
private pendingCrudUpload?;
|
|
170
|
+
private notifyCompletedUploads?;
|
|
120
171
|
syncStatus: SyncStatus;
|
|
121
172
|
triggerCrudUpload: () => void;
|
|
122
173
|
constructor(options: AbstractStreamingSyncImplementationOptions);
|
|
@@ -138,7 +189,25 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
|
|
|
138
189
|
*/
|
|
139
190
|
streamingSync(signal?: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
|
|
140
191
|
private collectLocalBucketState;
|
|
192
|
+
/**
|
|
193
|
+
* Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
|
|
194
|
+
* Because subkeys are always strings, this leads to quotes being added around them in `ps_oplog`.
|
|
195
|
+
* While this is not a problem as long as it's done consistently, it causes issues when a database
|
|
196
|
+
* created by the JS SDK is used with other SDKs, or (more likely) when the new Rust sync client
|
|
197
|
+
* is enabled.
|
|
198
|
+
*
|
|
199
|
+
* So, we add a migration from the old key format (with quotes) to the new one (no quotes). The
|
|
200
|
+
* migration is only triggered when necessary (for now). The function returns whether the new format
|
|
201
|
+
* should be used, so that the JS SDK is able to write to updated databases.
|
|
202
|
+
*
|
|
203
|
+
* @param requireFixedKeyFormat Whether we require the new format or also support the old one.
|
|
204
|
+
* The Rust client requires the new subkey format.
|
|
205
|
+
* @returns Whether the database is now using the new, fixed subkey format.
|
|
206
|
+
*/
|
|
207
|
+
private requireKeyFormat;
|
|
141
208
|
protected streamingSyncIteration(signal: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
|
|
209
|
+
private legacyStreamingSyncIteration;
|
|
210
|
+
private rustSyncIteration;
|
|
142
211
|
private updateSyncStatusForStartingCheckpoint;
|
|
143
212
|
private applyCheckpoint;
|
|
144
213
|
protected updateSyncStatus(options: SyncStatusOptions): void;
|