@peers-app/peers-device 0.15.0 → 0.15.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-download-manager.d.ts +2 -2
- package/dist/chunk-download-manager.js +1 -1
- package/dist/chunk-download-manager.js.map +1 -1
- package/dist/chunk-download-manager.test.js +57 -57
- package/dist/chunk-download-manager.test.js.map +1 -1
- package/dist/chunk-download.types.d.ts +1 -1
- package/dist/connection-manager/connection-manager-priorities.d.ts +1 -1
- package/dist/connection-manager/connection-manager-priorities.js +10 -6
- package/dist/connection-manager/connection-manager-priorities.js.map +1 -1
- package/dist/connection-manager/connection-manager-priorities.test.js +192 -194
- package/dist/connection-manager/connection-manager-priorities.test.js.map +1 -1
- package/dist/connection-manager/connection-manager.d.ts +2 -2
- package/dist/connection-manager/connection-manager.js +71 -55
- package/dist/connection-manager/connection-manager.js.map +1 -1
- package/dist/connection-manager/connection-manager.test.js +165 -147
- package/dist/connection-manager/connection-manager.test.js.map +1 -1
- package/dist/connection-manager/connection-state.type.d.ts +1 -1
- package/dist/connection-manager/device-message-handler.types.d.ts +6 -6
- package/dist/connection-manager/device-messages.d.ts +4 -4
- package/dist/connection-manager/device-messages.js +56 -40
- package/dist/connection-manager/device-messages.js.map +1 -1
- package/dist/connection-manager/group-invite-messages.d.ts +2 -2
- package/dist/connection-manager/group-invite-messages.js +36 -47
- package/dist/connection-manager/group-invite-messages.js.map +1 -1
- package/dist/connection-manager/hops-map.js +4 -4
- package/dist/connection-manager/hops-map.js.map +1 -1
- package/dist/connection-manager/hops-map.test.js +3 -3
- package/dist/connection-manager/hops-map.test.js.map +1 -1
- package/dist/connection-manager/network-manager.d.ts +2 -2
- package/dist/connection-manager/network-manager.js +81 -75
- package/dist/connection-manager/network-manager.js.map +1 -1
- package/dist/index.d.ts +12 -12
- package/dist/json-diff.d.ts +2 -2
- package/dist/json-diff.js +30 -27
- package/dist/json-diff.js.map +1 -1
- package/dist/local.data-source.d.ts +1 -1
- package/dist/local.data-source.js +23 -23
- package/dist/local.data-source.js.map +1 -1
- package/dist/local.data-source.test.js +17 -17
- package/dist/local.data-source.test.js.map +1 -1
- package/dist/machine-stats.js +57 -51
- package/dist/machine-stats.js.map +1 -1
- package/dist/machine-stats.test.js +42 -42
- package/dist/machine-stats.test.js.map +1 -1
- package/dist/main.d.ts +2 -2
- package/dist/main.js +10 -8
- package/dist/main.js.map +1 -1
- package/dist/packages.tracked-data-source.d.ts +1 -1
- package/dist/packages.tracked-data-source.js.map +1 -1
- package/dist/persistent-vars.test.js +148 -148
- package/dist/persistent-vars.test.js.map +1 -1
- package/dist/pvars.tracked-data-source.d.ts +1 -1
- package/dist/pvars.tracked-data-source.js +12 -10
- package/dist/pvars.tracked-data-source.js.map +1 -1
- package/dist/sync-group.d.ts +2 -2
- package/dist/sync-group.js +110 -88
- package/dist/sync-group.js.map +1 -1
- package/dist/sync-group.test.js +157 -120
- package/dist/sync-group.test.js.map +1 -1
- package/dist/tracked-data-source.d.ts +1 -1
- package/dist/tracked-data-source.js +61 -62
- package/dist/tracked-data-source.js.map +1 -1
- package/dist/tracked-data-source.test.js +507 -299
- package/dist/tracked-data-source.test.js.map +1 -1
- package/dist/websocket-client.d.ts +1 -1
- package/dist/websocket-client.js +50 -41
- package/dist/websocket-client.js.map +1 -1
- package/package.json +3 -3
package/dist/sync-group.js
CHANGED
|
@@ -25,7 +25,10 @@ class SyncGroup {
|
|
|
25
25
|
static getNetworkInfoCount = 0;
|
|
26
26
|
static electPreferredConnectionsCount = 0;
|
|
27
27
|
static syncWithRemoteDeviceCount = 0;
|
|
28
|
-
static averageSyncTimeMs = {
|
|
28
|
+
static averageSyncTimeMs = {
|
|
29
|
+
count: 0,
|
|
30
|
+
avgTimeMs: 0,
|
|
31
|
+
};
|
|
29
32
|
electionInterval = null;
|
|
30
33
|
preferredDeviceIds = [];
|
|
31
34
|
preferredByDeviceIds = [];
|
|
@@ -81,13 +84,13 @@ class SyncGroup {
|
|
|
81
84
|
async notifyRemoteDevicesOfChanges() {
|
|
82
85
|
this.pendingChangeCount = 0;
|
|
83
86
|
this.pendingChangeTimeout = null;
|
|
84
|
-
|
|
85
|
-
let priorityDevices = this.remoteDevices.filter(d => priorityDevicesIds.includes(d.deviceId));
|
|
87
|
+
const priorityDevicesIds = this.priorityDeviceIds.priorityDeviceIds;
|
|
88
|
+
let priorityDevices = this.remoteDevices.filter((d) => priorityDevicesIds.includes(d.deviceId));
|
|
86
89
|
if (!priorityDevices.length) {
|
|
87
90
|
priorityDevices = (0, lodash_1.shuffle)(this.remoteDevices);
|
|
88
91
|
}
|
|
89
|
-
priorityDevices.forEach(device => {
|
|
90
|
-
device.notifyOfChanges(this.deviceId, this.timestampLastApplied).catch(err => {
|
|
92
|
+
priorityDevices.forEach((device) => {
|
|
93
|
+
device.notifyOfChanges(this.deviceId, this.timestampLastApplied).catch((err) => {
|
|
91
94
|
// console.error("error notifying remote device of changes", err);
|
|
92
95
|
});
|
|
93
96
|
});
|
|
@@ -99,26 +102,28 @@ class SyncGroup {
|
|
|
99
102
|
// if we have preferred connections and this isn't one of them, don't sync
|
|
100
103
|
return;
|
|
101
104
|
}
|
|
102
|
-
const remoteDevice = this.remoteDevices.find(c => c.deviceId === deviceId);
|
|
103
|
-
const connection = this.connections.find(c => c.deviceId === deviceId);
|
|
104
|
-
if (!remoteDevice ||
|
|
105
|
+
const remoteDevice = this.remoteDevices.find((c) => c.deviceId === deviceId);
|
|
106
|
+
const connection = this.connections.find((c) => c.deviceId === deviceId);
|
|
107
|
+
if (!remoteDevice ||
|
|
108
|
+
!connection ||
|
|
109
|
+
(connection.timestampLastApplied ?? 0) >= timestampLastApplied) {
|
|
105
110
|
return; // device not connected or already up to date
|
|
106
111
|
}
|
|
107
|
-
this.syncWithRemoteDevice(remoteDevice).catch(err => {
|
|
112
|
+
this.syncWithRemoteDevice(remoteDevice).catch((err) => {
|
|
108
113
|
console.error("error syncing with remote device", err);
|
|
109
114
|
});
|
|
110
115
|
}
|
|
111
116
|
addConnection(remoteDevice, opts) {
|
|
112
|
-
if (this.connections.some(c => c.deviceId === remoteDevice.deviceId)) {
|
|
117
|
+
if (this.connections.some((c) => c.deviceId === remoteDevice.deviceId)) {
|
|
113
118
|
this.removeConnection(remoteDevice.deviceId);
|
|
114
119
|
}
|
|
115
120
|
const resyncInterval = opts?.resyncInterval ?? SyncGroup.RESYNC_INTERVAL;
|
|
116
|
-
if (this.remoteDevices.some(p => p.deviceId === remoteDevice.deviceId)) {
|
|
121
|
+
if (this.remoteDevices.some((p) => p.deviceId === remoteDevice.deviceId)) {
|
|
117
122
|
// TODO test this, used to be just return;
|
|
118
|
-
this.remoteDevices = this.remoteDevices.filter(p => p.deviceId !== remoteDevice.deviceId);
|
|
123
|
+
this.remoteDevices = this.remoteDevices.filter((p) => p.deviceId !== remoteDevice.deviceId);
|
|
119
124
|
}
|
|
120
125
|
// default to max latency of existing connections or 200ms - this discourages new peers from being used until they are proven
|
|
121
|
-
const defaultLatency = (0, lodash_1.maxBy)(this.connections,
|
|
126
|
+
const defaultLatency = (0, lodash_1.maxBy)(this.connections, "latencyMs")?.latencyMs || 200;
|
|
122
127
|
const connection = {
|
|
123
128
|
deviceId: remoteDevice.deviceId,
|
|
124
129
|
latencyMs: defaultLatency,
|
|
@@ -134,7 +139,7 @@ class SyncGroup {
|
|
|
134
139
|
return (0, peers_sdk_1.retryOnErrorOrTimeout)({
|
|
135
140
|
connection,
|
|
136
141
|
fn: () => rawRemoteDevice.listChanges(filter, opts),
|
|
137
|
-
}).catch(err => {
|
|
142
|
+
}).catch((err) => {
|
|
138
143
|
console.error("error listing changes on remote device", err);
|
|
139
144
|
throw err;
|
|
140
145
|
});
|
|
@@ -143,7 +148,7 @@ class SyncGroup {
|
|
|
143
148
|
return (0, peers_sdk_1.retryOnErrorOrTimeout)({
|
|
144
149
|
connection,
|
|
145
150
|
fn: () => rawRemoteDevice.getNetworkInfo(),
|
|
146
|
-
}).catch(err => {
|
|
151
|
+
}).catch((err) => {
|
|
147
152
|
console.error("error getting network info on remote device", err);
|
|
148
153
|
throw err;
|
|
149
154
|
});
|
|
@@ -152,7 +157,7 @@ class SyncGroup {
|
|
|
152
157
|
return (0, peers_sdk_1.retryOnErrorOrTimeout)({
|
|
153
158
|
connection,
|
|
154
159
|
fn: () => rawRemoteDevice.notifyOfChanges(deviceId, timestampLastApplied),
|
|
155
|
-
}).catch(err => {
|
|
160
|
+
}).catch((err) => {
|
|
156
161
|
console.error("error notifying remote device of changes", err);
|
|
157
162
|
throw err;
|
|
158
163
|
});
|
|
@@ -166,16 +171,17 @@ class SyncGroup {
|
|
|
166
171
|
}
|
|
167
172
|
this.syncTimeout = setTimeout(async () => {
|
|
168
173
|
// removing connections that are too slow or have too many errors
|
|
169
|
-
if (connection.errorRate > 0.8 || connection.latencyMs >
|
|
174
|
+
if (connection.errorRate > 0.8 || connection.latencyMs > SyncGroup.TIMEOUT_MAX / 2) {
|
|
170
175
|
console.warn(`removing connection to ${remoteDevice.deviceId} due to high error rate or latency`);
|
|
171
176
|
this.removeConnection(remoteDevice.deviceId);
|
|
172
177
|
return;
|
|
173
178
|
}
|
|
174
|
-
if (!this.connections.find(c => c.deviceId === remoteDevice.deviceId)) {
|
|
179
|
+
if (!this.connections.find((c) => c.deviceId === remoteDevice.deviceId)) {
|
|
175
180
|
return;
|
|
176
181
|
}
|
|
177
182
|
// if we have preferred connections, only sync with those
|
|
178
|
-
if (this.preferredDeviceIds.length &&
|
|
183
|
+
if (this.preferredDeviceIds.length &&
|
|
184
|
+
!this.preferredDeviceIds.includes(remoteDevice.deviceId)) {
|
|
179
185
|
return queueSync();
|
|
180
186
|
}
|
|
181
187
|
await this.syncWithRemoteDevice(remoteDevice).finally(queueSync);
|
|
@@ -197,18 +203,18 @@ class SyncGroup {
|
|
|
197
203
|
return addConnectionPromise;
|
|
198
204
|
}
|
|
199
205
|
removeConnection(deviceId) {
|
|
200
|
-
const connection = this.connections.find(conn => conn.deviceId === deviceId);
|
|
206
|
+
const connection = this.connections.find((conn) => conn.deviceId === deviceId);
|
|
201
207
|
if (!connection)
|
|
202
208
|
return;
|
|
203
209
|
connection.closed = true;
|
|
204
|
-
this.connections = this.connections.filter(conn => conn.deviceId !== deviceId);
|
|
205
|
-
this.remoteDevices = this.remoteDevices.filter(cd => cd.deviceId !== deviceId);
|
|
206
|
-
this.preferredByDeviceIds = this.preferredByDeviceIds.filter(id => id !== deviceId);
|
|
210
|
+
this.connections = this.connections.filter((conn) => conn.deviceId !== deviceId);
|
|
211
|
+
this.remoteDevices = this.remoteDevices.filter((cd) => cd.deviceId !== deviceId);
|
|
212
|
+
this.preferredByDeviceIds = this.preferredByDeviceIds.filter((id) => id !== deviceId);
|
|
207
213
|
const isPreferred = this.preferredDeviceIds.includes(deviceId);
|
|
208
214
|
connection?.onClose?.();
|
|
209
215
|
delete this.cachedNetworkInfos[deviceId];
|
|
210
216
|
if (isPreferred && !this.disposed) {
|
|
211
|
-
this.preferredDeviceIds = this.preferredDeviceIds.filter(id => id !== deviceId);
|
|
217
|
+
this.preferredDeviceIds = this.preferredDeviceIds.filter((id) => id !== deviceId);
|
|
212
218
|
this.electPreferredConnections();
|
|
213
219
|
}
|
|
214
220
|
}
|
|
@@ -219,7 +225,7 @@ class SyncGroup {
|
|
|
219
225
|
while (!networkInfo && retryCnt < 10 && !this.disposed) {
|
|
220
226
|
retryCnt++;
|
|
221
227
|
networkInfo = await Promise.race([
|
|
222
|
-
remoteDevice.getNetworkInfo().catch(err => null),
|
|
228
|
+
remoteDevice.getNetworkInfo().catch((err) => null),
|
|
223
229
|
(0, peers_sdk_1.sleep)(400 * retryCnt),
|
|
224
230
|
]);
|
|
225
231
|
if (networkInfo) {
|
|
@@ -240,21 +246,22 @@ class SyncGroup {
|
|
|
240
246
|
}
|
|
241
247
|
syncPromise = Promise.resolve();
|
|
242
248
|
async syncWithRemoteDevice(remoteDevice, opts) {
|
|
243
|
-
this.syncPromise = this.syncPromise
|
|
249
|
+
this.syncPromise = this.syncPromise
|
|
250
|
+
.then(async () => {
|
|
244
251
|
const sTime = Date.now();
|
|
245
252
|
await this._syncWithRemoteDevice(remoteDevice, opts);
|
|
246
253
|
const syncTime = Date.now() - sTime;
|
|
247
254
|
const oldAvgTimeMs = SyncGroup.averageSyncTimeMs.avgTimeMs;
|
|
248
255
|
const oldCount = SyncGroup.averageSyncTimeMs.count;
|
|
249
256
|
const newCount = Math.min(oldCount + 1, 100);
|
|
250
|
-
const newAvgTimeMs = (
|
|
257
|
+
const newAvgTimeMs = (oldAvgTimeMs * oldCount + syncTime) / newCount;
|
|
251
258
|
SyncGroup.averageSyncTimeMs = {
|
|
252
259
|
count: newCount,
|
|
253
260
|
avgTimeMs: newAvgTimeMs,
|
|
254
261
|
};
|
|
255
262
|
})
|
|
256
|
-
.catch(err => {
|
|
257
|
-
console.error(
|
|
263
|
+
.catch((err) => {
|
|
264
|
+
console.error("error syncing with remote device", err);
|
|
258
265
|
});
|
|
259
266
|
return this.syncPromise;
|
|
260
267
|
}
|
|
@@ -269,21 +276,22 @@ class SyncGroup {
|
|
|
269
276
|
timestampAppliedLast: 0,
|
|
270
277
|
};
|
|
271
278
|
}
|
|
272
|
-
const connection = this.connections.find(c => c.deviceId === remoteDevice.deviceId);
|
|
279
|
+
const connection = this.connections.find((c) => c.deviceId === remoteDevice.deviceId);
|
|
273
280
|
if (connection && !connection.timestampLastApplied && syncInfo.timestampAppliedLast) {
|
|
274
281
|
connection.timestampLastApplied = syncInfo.timestampAppliedLast;
|
|
275
282
|
}
|
|
276
|
-
const networkInfoStart = opts?.networkInfo || await this.getRemoteNetworkInfo(remoteDevice, { force: true });
|
|
277
|
-
if (connection?.timestampLastApplied &&
|
|
283
|
+
const networkInfoStart = opts?.networkInfo || (await this.getRemoteNetworkInfo(remoteDevice, { force: true }));
|
|
284
|
+
if (connection?.timestampLastApplied &&
|
|
285
|
+
networkInfoStart.timestampLastApplied <= connection.timestampLastApplied) {
|
|
278
286
|
return; // already synced
|
|
279
287
|
}
|
|
280
288
|
// Sync table definitions so we know about all table schemas before applying other changes
|
|
281
289
|
// Also sync groups first so they are as up-to-date as possible (security and sync performance)
|
|
282
|
-
const tablesToSyncFirst = [
|
|
290
|
+
const tablesToSyncFirst = ["TableDefinitions", "Groups"];
|
|
283
291
|
const tableAndGroupChanges = await remoteDevice.listChanges({
|
|
284
292
|
supersededAt: { $exists: false },
|
|
285
293
|
appliedAt: { $gte: syncInfo.timestampAppliedLast },
|
|
286
|
-
tableName: { $in: tablesToSyncFirst }
|
|
294
|
+
tableName: { $in: tablesToSyncFirst },
|
|
287
295
|
});
|
|
288
296
|
if (tableAndGroupChanges.length) {
|
|
289
297
|
console.warn(`table definitions and/or group changes found ${tableAndGroupChanges.length}, applying first`);
|
|
@@ -300,7 +308,7 @@ class SyncGroup {
|
|
|
300
308
|
const startTime = Date.now();
|
|
301
309
|
const batchSize = (0, lodash_1.clamp)(opts?.pageSize ?? SyncGroup.CHANGES_PAGE_SIZE, 2, 50_000);
|
|
302
310
|
const changeCursor = (0, peers_sdk_1.dataSourceCursor)({
|
|
303
|
-
primaryKeyName:
|
|
311
|
+
primaryKeyName: "changeId",
|
|
304
312
|
list: remoteDevice.listChanges.bind(remoteDevice),
|
|
305
313
|
}, {
|
|
306
314
|
// If we decide to support change history preservation, we'll need to set this conditionally
|
|
@@ -309,15 +317,15 @@ class SyncGroup {
|
|
|
309
317
|
{ appliedAt: { $gte: syncInfo.timestampAppliedLast } },
|
|
310
318
|
// We don't want to go past the current applied changes because we might skip changes that are actively being synced from other device to the remote device
|
|
311
319
|
{ appliedAt: { $lte: networkInfoStart.timestampLastApplied } },
|
|
312
|
-
{ tableName: { $nin: tablesToSyncFirst } }
|
|
313
|
-
]
|
|
314
|
-
}, { sortBy: [
|
|
320
|
+
{ tableName: { $nin: tablesToSyncFirst } },
|
|
321
|
+
],
|
|
322
|
+
}, { sortBy: ["createdAt"], pageSize: batchSize });
|
|
315
323
|
const changeBatch = [];
|
|
316
324
|
let maxTimestampApplied = syncInfo.timestampAppliedLast;
|
|
317
325
|
let totalChangesSynced = 0;
|
|
318
326
|
const applyBatch = async () => {
|
|
319
327
|
if (changeBatch.length) {
|
|
320
|
-
maxTimestampApplied = (0, lodash_1.max)([maxTimestampApplied, ...changeBatch.map(c => c.appliedAt)]);
|
|
328
|
+
maxTimestampApplied = (0, lodash_1.max)([maxTimestampApplied, ...changeBatch.map((c) => c.appliedAt)]);
|
|
321
329
|
const sTime = Date.now();
|
|
322
330
|
await this.applyChanges(changeBatch, remoteDevice);
|
|
323
331
|
totalChangesSynced += changeBatch.length;
|
|
@@ -346,7 +354,7 @@ class SyncGroup {
|
|
|
346
354
|
if (connection) {
|
|
347
355
|
connection.timestampLastApplied = syncInfo.timestampAppliedLast;
|
|
348
356
|
}
|
|
349
|
-
// transitive device syncing
|
|
357
|
+
// transitive device syncing
|
|
350
358
|
for (const conn of networkInfoStart.connections) {
|
|
351
359
|
if (!conn.timestampLastApplied) {
|
|
352
360
|
continue;
|
|
@@ -367,21 +375,28 @@ class SyncGroup {
|
|
|
367
375
|
}
|
|
368
376
|
async applyChanges(changes, remoteDevice) {
|
|
369
377
|
const tableTimestamps = {};
|
|
370
|
-
changes.forEach(c => {
|
|
378
|
+
changes.forEach((c) => {
|
|
371
379
|
if (c.createdAt < (tableTimestamps[c.tableName] || Number.MAX_VALUE)) {
|
|
372
380
|
tableTimestamps[c.tableName] = c.createdAt;
|
|
373
381
|
}
|
|
374
382
|
});
|
|
375
383
|
// always load Users, Devices, Groups, PersistentVars, PeerTypes, Packages, and TableDefinitions first since they can affect how other tables are handled
|
|
376
384
|
const tableNames = (0, lodash_1.sortBy)(Object.keys(tableTimestamps), (t) => {
|
|
377
|
-
return t === "Users"
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
+
return t === "Users"
|
|
386
|
+
? -7
|
|
387
|
+
: t === "Devices"
|
|
388
|
+
? -6
|
|
389
|
+
: t === "Groups"
|
|
390
|
+
? -5
|
|
391
|
+
: t === "PersistentVars"
|
|
392
|
+
? -4
|
|
393
|
+
: t === "PeerTypes"
|
|
394
|
+
? -3
|
|
395
|
+
: t === "Packages"
|
|
396
|
+
? -2
|
|
397
|
+
: t === "Files"
|
|
398
|
+
? 0
|
|
399
|
+
: tableTimestamps[t];
|
|
385
400
|
});
|
|
386
401
|
const tables = (0, lodash_1.keyBy)(this.tableContainer.getAllTables(), "tableName");
|
|
387
402
|
// WARNING: by grouping changes by table name, we're not necessarily applying them in the order they were created
|
|
@@ -396,14 +411,14 @@ class SyncGroup {
|
|
|
396
411
|
}
|
|
397
412
|
const table = tables[tableName];
|
|
398
413
|
let ds = table;
|
|
399
|
-
while (typeof ds?.applyChanges !==
|
|
414
|
+
while (typeof ds?.applyChanges !== "function" && ds.dataSource) {
|
|
400
415
|
ds = ds.dataSource;
|
|
401
416
|
}
|
|
402
|
-
if (typeof ds?.applyChanges !==
|
|
417
|
+
if (typeof ds?.applyChanges !== "function") {
|
|
403
418
|
console.warn(`Table or underlying data sources does not have an applyChanges function: ${tableName} - cannot continue with applyChanges`);
|
|
404
419
|
continue;
|
|
405
420
|
}
|
|
406
|
-
const tableChanges = changes.filter(c => c.tableName === tableName);
|
|
421
|
+
const tableChanges = changes.filter((c) => c.tableName === tableName);
|
|
407
422
|
await ds.applyChanges(tableChanges, remoteDevice, this.dataContext);
|
|
408
423
|
SyncGroup.applyChangesCount++;
|
|
409
424
|
}
|
|
@@ -422,7 +437,7 @@ class SyncGroup {
|
|
|
422
437
|
*/
|
|
423
438
|
async resolveInsertChangeValues(changes) {
|
|
424
439
|
// Find insert changes that need resolution (value is null/undefined)
|
|
425
|
-
const insertChanges = changes.filter(c => c.op ===
|
|
440
|
+
const insertChanges = changes.filter((c) => c.op === "set" && c.path === "/" && (c.value === null || c.value === undefined));
|
|
426
441
|
if (insertChanges.length === 0)
|
|
427
442
|
return;
|
|
428
443
|
// Group by tableName so we can batch-lookup from the right table
|
|
@@ -446,7 +461,7 @@ class SyncGroup {
|
|
|
446
461
|
}
|
|
447
462
|
}
|
|
448
463
|
const tableChanges = byTable[tableName];
|
|
449
|
-
const recordIds = [...new Set(tableChanges.map(c => c.recordId))];
|
|
464
|
+
const recordIds = [...new Set(tableChanges.map((c) => c.recordId))];
|
|
450
465
|
const primaryKeyName = table.primaryKeyName || table.dataSource?.primaryKeyName;
|
|
451
466
|
try {
|
|
452
467
|
const records = await table.list({
|
|
@@ -473,7 +488,7 @@ class SyncGroup {
|
|
|
473
488
|
if (this.electionInterval) {
|
|
474
489
|
clearInterval(this.electionInterval);
|
|
475
490
|
}
|
|
476
|
-
this.connections.forEach(c => {
|
|
491
|
+
this.connections.forEach((c) => {
|
|
477
492
|
this.removeConnection(c.deviceId);
|
|
478
493
|
});
|
|
479
494
|
this.connections.length = 0;
|
|
@@ -486,7 +501,7 @@ class SyncGroup {
|
|
|
486
501
|
async getNetworkInfo() {
|
|
487
502
|
SyncGroup.getNetworkInfoCount++;
|
|
488
503
|
if (!this.timestampLastApplied) {
|
|
489
|
-
const lastChangesApplied = await this.changeTrackingTable.list({}, { sortBy: [
|
|
504
|
+
const lastChangesApplied = await this.changeTrackingTable.list({}, { sortBy: ["-appliedAt"], pageSize: 1 });
|
|
490
505
|
if (lastChangesApplied.length) {
|
|
491
506
|
this.timestampLastApplied = lastChangesApplied[0].appliedAt;
|
|
492
507
|
}
|
|
@@ -495,7 +510,7 @@ class SyncGroup {
|
|
|
495
510
|
const networkInfo = {
|
|
496
511
|
deviceId: this.deviceId,
|
|
497
512
|
timestampLastApplied: this.timestampLastApplied,
|
|
498
|
-
connections: this.connections.map(c => {
|
|
513
|
+
connections: this.connections.map((c) => {
|
|
499
514
|
const connData = { ...c };
|
|
500
515
|
delete connData.onClose;
|
|
501
516
|
return connData;
|
|
@@ -503,7 +518,7 @@ class SyncGroup {
|
|
|
503
518
|
preferredDeviceIds: [...this.preferredDeviceIds],
|
|
504
519
|
cpuPercent: _machineStats.cpu.average,
|
|
505
520
|
memPercent: _machineStats.memory.current,
|
|
506
|
-
connectionSlotsAvailable: 0, // set by connection manager
|
|
521
|
+
connectionSlotsAvailable: 0, // set by connection manager
|
|
507
522
|
};
|
|
508
523
|
return networkInfo;
|
|
509
524
|
}
|
|
@@ -531,42 +546,44 @@ class SyncGroup {
|
|
|
531
546
|
}
|
|
532
547
|
async _electPreferredConnections() {
|
|
533
548
|
SyncGroup.electPreferredConnectionsCount++;
|
|
534
|
-
if (this.preferredDeviceIds.length &&
|
|
549
|
+
if (this.preferredDeviceIds.length &&
|
|
550
|
+
this.preferredDeviceIds.every((id) => this.connections.find((c) => c.deviceId === id))) {
|
|
535
551
|
// if we already have preferred connections, chill out for a bit
|
|
536
|
-
// in testing, waiting 20ms before re-electing seems to help prevent overabundance of elections
|
|
552
|
+
// in testing, waiting 20ms before re-electing seems to help prevent overabundance of elections
|
|
537
553
|
// await sleep(20 * Math.random());
|
|
538
554
|
}
|
|
539
|
-
const allNetworkInfo = (await Promise.all(this.remoteDevices.map(d => this.getRemoteNetworkInfo(d).catch(err => null))))
|
|
540
|
-
.filter(d => d);
|
|
555
|
+
const allNetworkInfo = (await Promise.all(this.remoteDevices.map((d) => this.getRemoteNetworkInfo(d).catch((err) => null)))).filter((d) => d);
|
|
541
556
|
const myConnections = this.getConnections();
|
|
542
557
|
const electionResult = (0, peers_sdk_1.electDevices)({
|
|
543
558
|
deviceId: this.deviceId,
|
|
544
559
|
myConnections,
|
|
545
560
|
allNetworkInfo,
|
|
546
561
|
});
|
|
547
|
-
const newPreferredDeviceIds = electionResult.preferredDeviceIds.filter(id => !this.preferredDeviceIds.includes(id));
|
|
562
|
+
const newPreferredDeviceIds = electionResult.preferredDeviceIds.filter((id) => !this.preferredDeviceIds.includes(id));
|
|
548
563
|
this.preferredDeviceIds = electionResult.preferredDeviceIds;
|
|
549
564
|
this.preferredByDeviceIds = electionResult.preferredByDeviceIds;
|
|
550
565
|
for (const deviceId of newPreferredDeviceIds) {
|
|
551
|
-
const remoteDevice = this.remoteDevices.find(c => c.deviceId === deviceId);
|
|
566
|
+
const remoteDevice = this.remoteDevices.find((c) => c.deviceId === deviceId);
|
|
552
567
|
if (remoteDevice) {
|
|
553
|
-
const networkInfo = allNetworkInfo.find(c => c.deviceId === deviceId);
|
|
554
|
-
await this.syncWithRemoteDevice(remoteDevice, { networkInfo }).catch(err => {
|
|
568
|
+
const networkInfo = allNetworkInfo.find((c) => c.deviceId === deviceId);
|
|
569
|
+
await this.syncWithRemoteDevice(remoteDevice, { networkInfo }).catch((err) => {
|
|
555
570
|
console.error("error syncing with remote device", err);
|
|
556
571
|
});
|
|
557
572
|
}
|
|
558
573
|
}
|
|
559
574
|
}
|
|
560
575
|
async getRemoteNetworkInfo(device, opts) {
|
|
561
|
-
const deviceId = typeof device ===
|
|
576
|
+
const deviceId = typeof device === "string" ? device : device.deviceId;
|
|
562
577
|
// if (1 + 1 === 2) {
|
|
563
578
|
// return this.remoteDevices.find(d => d.deviceId === deviceId)!.getNetworkInfo();
|
|
564
579
|
// }
|
|
565
580
|
const cachedNetworkInfo = this.cachedNetworkInfos[deviceId];
|
|
566
|
-
if (!opts?.force &&
|
|
581
|
+
if (!opts?.force &&
|
|
582
|
+
cachedNetworkInfo &&
|
|
583
|
+
Date.now() - cachedNetworkInfo.lastUpdated < SyncGroup.NETWORK_INFO_CACHE_TIME) {
|
|
567
584
|
return cachedNetworkInfo.networkInfo;
|
|
568
585
|
}
|
|
569
|
-
const remoteDevice = typeof device !==
|
|
586
|
+
const remoteDevice = typeof device !== "string" ? device : this.remoteDevices.find((c) => c.deviceId === deviceId);
|
|
570
587
|
if (!remoteDevice) {
|
|
571
588
|
throw new Error(`Device not connected: ${deviceId}`);
|
|
572
589
|
}
|
|
@@ -575,10 +592,12 @@ class SyncGroup {
|
|
|
575
592
|
networkInfo: networkInfoPromise,
|
|
576
593
|
lastUpdated: Date.now(),
|
|
577
594
|
};
|
|
578
|
-
return networkInfoPromise
|
|
595
|
+
return networkInfoPromise
|
|
596
|
+
.then((networkInfo) => {
|
|
579
597
|
this.rebuildNetworkMap();
|
|
580
598
|
return networkInfo;
|
|
581
|
-
})
|
|
599
|
+
})
|
|
600
|
+
.catch((err) => {
|
|
582
601
|
delete this.cachedNetworkInfos[deviceId];
|
|
583
602
|
throw err;
|
|
584
603
|
});
|
|
@@ -604,7 +623,7 @@ class SyncGroup {
|
|
|
604
623
|
return [];
|
|
605
624
|
}
|
|
606
625
|
// Delete all orphaned entries in a single SQL call
|
|
607
|
-
const placeholders = orphanedTableNames.map(() =>
|
|
626
|
+
const placeholders = orphanedTableNames.map(() => "?").join(", ");
|
|
608
627
|
await db.exec(`DELETE FROM "${changeTrackingTableName}" WHERE tableName IN (${placeholders})`, orphanedTableNames);
|
|
609
628
|
for (const tableName of orphanedTableNames) {
|
|
610
629
|
console.log(`Cleaned up orphaned change tracking entries for deleted table: ${tableName}`);
|
|
@@ -620,7 +639,7 @@ class SyncGroup {
|
|
|
620
639
|
try {
|
|
621
640
|
const orphaned = await this.cleanupOrphanedChangeTrackingEntries();
|
|
622
641
|
if (orphaned.length > 0) {
|
|
623
|
-
console.log(`Cleaned up orphaned change tracking entries for ${orphaned.length} deleted table(s): ${orphaned.join(
|
|
642
|
+
console.log(`Cleaned up orphaned change tracking entries for ${orphaned.length} deleted table(s): ${orphaned.join(", ")}`);
|
|
624
643
|
}
|
|
625
644
|
}
|
|
626
645
|
catch (err) {
|
|
@@ -630,11 +649,14 @@ class SyncGroup {
|
|
|
630
649
|
const dataLocksTableName = (0, peers_sdk_1.DataLocks)(this.dataContext).tableName;
|
|
631
650
|
try {
|
|
632
651
|
const changeTrackingTableId = this.changeTrackingTable.metaData.name;
|
|
633
|
-
const oneDayAgoMs = Date.now() -
|
|
652
|
+
const oneDayAgoMs = Date.now() - 24 * 60 * 60 * 1000;
|
|
634
653
|
// ensure tables are initialized
|
|
635
654
|
await this.changeTrackingTable.initTable();
|
|
636
655
|
await (0, peers_sdk_1.DataLocks)().list({}, { pageSize: 1 });
|
|
637
|
-
await db.exec(`DELETE FROM ${changeTrackingTableId} WHERE tableName = ? AND createdAt < ?`, [
|
|
656
|
+
await db.exec(`DELETE FROM ${changeTrackingTableId} WHERE tableName = ? AND createdAt < ?`, [
|
|
657
|
+
dataLocksTableName,
|
|
658
|
+
oneDayAgoMs,
|
|
659
|
+
]);
|
|
638
660
|
await db.exec(`DELETE FROM ${dataLocksTableName} WHERE lockedUntil < ?`, [oneDayAgoMs]);
|
|
639
661
|
}
|
|
640
662
|
catch (err) {
|
|
@@ -720,7 +742,7 @@ class SyncGroup {
|
|
|
720
742
|
hops: message.hops,
|
|
721
743
|
};
|
|
722
744
|
}
|
|
723
|
-
if ((0, lodash_1.isEqual)(message.payload, [
|
|
745
|
+
if ((0, lodash_1.isEqual)(message.payload, ["getNetworkInfo"])) {
|
|
724
746
|
return {
|
|
725
747
|
statusCode: 200,
|
|
726
748
|
statusMessage: "OK",
|
|
@@ -728,12 +750,12 @@ class SyncGroup {
|
|
|
728
750
|
payload: await this.getNetworkInfo(),
|
|
729
751
|
};
|
|
730
752
|
}
|
|
731
|
-
if ((0, lodash_1.isArray)(message.payload) && (0, lodash_1.isEqual)(message.payload[0],
|
|
753
|
+
if ((0, lodash_1.isArray)(message.payload) && (0, lodash_1.isEqual)(message.payload[0], "ping")) {
|
|
732
754
|
return {
|
|
733
755
|
statusCode: 200,
|
|
734
756
|
statusMessage: "OK",
|
|
735
757
|
hops: message.hops,
|
|
736
|
-
payload: [
|
|
758
|
+
payload: ["pong", Date.now(), ...message.payload.slice(1)],
|
|
737
759
|
};
|
|
738
760
|
}
|
|
739
761
|
// default behavior
|
|
@@ -750,11 +772,11 @@ class SyncGroup {
|
|
|
750
772
|
const networkMap = {};
|
|
751
773
|
const NETWORK_INFO_MAX_AGE = SyncGroup.RESYNC_INTERVAL + SyncGroup.TIMEOUT_MAX;
|
|
752
774
|
for (const [deviceId, cachedNetworkInfo] of Object.entries(this.cachedNetworkInfos)) {
|
|
753
|
-
if (
|
|
775
|
+
if (Date.now() - cachedNetworkInfo.lastUpdated > NETWORK_INFO_MAX_AGE) {
|
|
754
776
|
continue;
|
|
755
777
|
}
|
|
756
|
-
const networkInfo = await cachedNetworkInfo.networkInfo.catch(err => undefined);
|
|
757
|
-
networkInfo?.connections.forEach(c => {
|
|
778
|
+
const networkInfo = await cachedNetworkInfo.networkInfo.catch((err) => undefined);
|
|
779
|
+
networkInfo?.connections.forEach((c) => {
|
|
758
780
|
networkMap[c.deviceId] ??= new Set();
|
|
759
781
|
networkMap[c.deviceId].add(deviceId);
|
|
760
782
|
networkMap[deviceId] ??= new Set();
|
|
@@ -764,25 +786,25 @@ class SyncGroup {
|
|
|
764
786
|
this.networkMap = networkMap;
|
|
765
787
|
}
|
|
766
788
|
getPathToDevice(deviceId) {
|
|
767
|
-
if (this.connections.some(c => c.deviceId === deviceId)) {
|
|
789
|
+
if (this.connections.some((c) => c.deviceId === deviceId)) {
|
|
768
790
|
return {
|
|
769
|
-
pathType:
|
|
791
|
+
pathType: "direct",
|
|
770
792
|
throughDeviceIds: [this.deviceId],
|
|
771
793
|
};
|
|
772
794
|
}
|
|
773
795
|
if (this.networkMap[deviceId]?.size) {
|
|
774
796
|
return {
|
|
775
|
-
pathType:
|
|
776
|
-
throughDeviceIds: Array.from(this.networkMap[deviceId].values())
|
|
797
|
+
pathType: "indirect",
|
|
798
|
+
throughDeviceIds: Array.from(this.networkMap[deviceId].values()),
|
|
777
799
|
};
|
|
778
800
|
}
|
|
779
801
|
return {
|
|
780
|
-
pathType:
|
|
781
|
-
throughDeviceIds: []
|
|
802
|
+
pathType: "unknown",
|
|
803
|
+
throughDeviceIds: [],
|
|
782
804
|
};
|
|
783
805
|
}
|
|
784
806
|
reportRemoteDeviceConnectedIds(deviceId, connectedDeviceIds) {
|
|
785
|
-
this.networkMap[deviceId]?.forEach(connectedDeviceId => {
|
|
807
|
+
this.networkMap[deviceId]?.forEach((connectedDeviceId) => {
|
|
786
808
|
connectedDeviceIds.add(connectedDeviceId);
|
|
787
809
|
});
|
|
788
810
|
}
|