document-drive 1.0.0-websockets.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/package.json +74 -88
- package/src/cache/index.ts +2 -2
- package/src/cache/memory.ts +22 -13
- package/src/cache/redis.ts +43 -16
- package/src/cache/types.ts +4 -4
- package/src/index.ts +6 -3
- package/src/queue/base.ts +276 -214
- package/src/queue/index.ts +2 -2
- package/src/queue/redis.ts +138 -127
- package/src/queue/types.ts +44 -38
- package/src/read-mode/errors.ts +19 -0
- package/src/read-mode/index.ts +125 -0
- package/src/read-mode/service.ts +207 -0
- package/src/read-mode/types.ts +108 -0
- package/src/server/error.ts +61 -26
- package/src/server/index.ts +2160 -1785
- package/src/server/listener/index.ts +2 -2
- package/src/server/listener/manager.ts +475 -437
- package/src/server/listener/transmitter/index.ts +4 -5
- package/src/server/listener/transmitter/internal.ts +77 -79
- package/src/server/listener/transmitter/pull-responder.ts +363 -329
- package/src/server/listener/transmitter/switchboard-push.ts +72 -55
- package/src/server/listener/transmitter/types.ts +19 -25
- package/src/server/types.ts +536 -349
- package/src/server/utils.ts +26 -27
- package/src/storage/base.ts +81 -0
- package/src/storage/browser.ts +233 -216
- package/src/storage/filesystem.ts +257 -256
- package/src/storage/index.ts +2 -1
- package/src/storage/memory.ts +206 -214
- package/src/storage/prisma.ts +575 -568
- package/src/storage/sequelize.ts +460 -471
- package/src/storage/types.ts +83 -67
- package/src/utils/default-drives-manager.ts +341 -0
- package/src/utils/document-helpers.ts +19 -18
- package/src/utils/graphql.ts +288 -34
- package/src/utils/index.ts +61 -59
- package/src/utils/logger.ts +39 -37
- package/src/utils/migrations.ts +58 -0
- package/src/utils/run-asap.ts +156 -0
- package/CHANGELOG.md +0 -818
- package/src/server/listener/transmitter/subscription.ts +0 -364
|
@@ -1,491 +1,529 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
} from
|
|
5
|
-
import { OperationScope } from
|
|
6
|
-
import {
|
|
2
|
+
DocumentDriveDocument,
|
|
3
|
+
ListenerFilter,
|
|
4
|
+
} from "document-model-libs/document-drive";
|
|
5
|
+
import { OperationScope } from "document-model/document";
|
|
6
|
+
import { logger } from "../../utils/logger";
|
|
7
|
+
import { OperationError } from "../error";
|
|
7
8
|
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
9
|
+
BaseListenerManager,
|
|
10
|
+
ErrorStatus,
|
|
11
|
+
GetStrandsOptions,
|
|
12
|
+
Listener,
|
|
13
|
+
ListenerState,
|
|
14
|
+
ListenerUpdate,
|
|
15
|
+
OperationUpdate,
|
|
16
|
+
StrandUpdate,
|
|
17
|
+
SynchronizationUnit,
|
|
18
|
+
} from "../types";
|
|
19
|
+
import { PullResponderTransmitter } from "./transmitter";
|
|
20
|
+
import { InternalTransmitter } from "./transmitter/internal";
|
|
21
|
+
import { SwitchboardPushTransmitter } from "./transmitter/switchboard-push";
|
|
22
|
+
import { ITransmitter, StrandUpdateSource } from "./transmitter/types";
|
|
22
23
|
|
|
23
24
|
function debounce<T extends unknown[], R>(
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
func: (...args: T) => Promise<R>,
|
|
26
|
+
delay = 250,
|
|
26
27
|
) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
};
|
|
28
|
+
let timer: number;
|
|
29
|
+
return (immediate: boolean, ...args: T) => {
|
|
30
|
+
if (timer) {
|
|
31
|
+
clearTimeout(timer);
|
|
32
|
+
}
|
|
33
|
+
return new Promise<R>((resolve, reject) => {
|
|
34
|
+
if (immediate) {
|
|
35
|
+
func(...args)
|
|
36
|
+
.then(resolve)
|
|
37
|
+
.catch(reject);
|
|
38
|
+
} else {
|
|
39
|
+
timer = setTimeout(() => {
|
|
40
|
+
func(...args)
|
|
41
|
+
.then(resolve)
|
|
42
|
+
.catch(reject);
|
|
43
|
+
}, delay) as unknown as number;
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
};
|
|
47
47
|
}
|
|
48
|
-
|
|
49
48
|
export class ListenerManager extends BaseListenerManager {
|
|
50
|
-
|
|
49
|
+
static LISTENER_UPDATE_DELAY = 250;
|
|
51
50
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
51
|
+
async getTransmitter(
|
|
52
|
+
driveId: string,
|
|
53
|
+
listenerId: string,
|
|
54
|
+
): Promise<ITransmitter | undefined> {
|
|
55
|
+
return Promise.resolve(this.transmitters[driveId]?.[listenerId]);
|
|
56
|
+
}
|
|
58
57
|
|
|
59
|
-
|
|
60
|
-
|
|
58
|
+
driveHasListeners(driveId: string) {
|
|
59
|
+
return this.listenerState.has(driveId);
|
|
60
|
+
}
|
|
61
61
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
}
|
|
62
|
+
async addListener(listener: Listener) {
|
|
63
|
+
const drive = listener.driveId;
|
|
65
64
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
block: listener.block,
|
|
70
|
-
driveId: listener.driveId,
|
|
71
|
-
pendingTimeout: '0',
|
|
72
|
-
listener,
|
|
73
|
-
listenerStatus: 'CREATED',
|
|
74
|
-
syncUnits: new Map()
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
let transmitter: ITransmitter | undefined;
|
|
65
|
+
if (!this.listenerState.has(drive)) {
|
|
66
|
+
this.listenerState.set(drive, new Map());
|
|
67
|
+
}
|
|
78
68
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
69
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
70
|
+
const driveMap = this.listenerState.get(drive)!;
|
|
71
|
+
driveMap.set(listener.listenerId, {
|
|
72
|
+
block: listener.block,
|
|
73
|
+
driveId: listener.driveId,
|
|
74
|
+
pendingTimeout: "0",
|
|
75
|
+
listener,
|
|
76
|
+
listenerStatus: "CREATED",
|
|
77
|
+
syncUnits: new Map(),
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
let transmitter: ITransmitter | undefined;
|
|
81
|
+
|
|
82
|
+
switch (listener.callInfo?.transmitterType) {
|
|
83
|
+
case "SwitchboardPush": {
|
|
84
|
+
transmitter = new SwitchboardPushTransmitter(listener, this.drive);
|
|
85
|
+
break;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
case "PullResponder": {
|
|
89
|
+
transmitter = new PullResponderTransmitter(listener, this.drive, this);
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
case "Internal": {
|
|
93
|
+
transmitter = new InternalTransmitter(listener, this.drive);
|
|
94
|
+
break;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
87
97
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
this.drive,
|
|
92
|
-
this
|
|
93
|
-
);
|
|
94
|
-
break;
|
|
95
|
-
}
|
|
96
|
-
case "Subscription": {
|
|
97
|
-
transmitter = new SubscriptionTransmitter(listener, this);
|
|
98
|
-
break;
|
|
99
|
-
}
|
|
100
|
-
case 'Internal': {
|
|
101
|
-
transmitter = new InternalTransmitter(listener, this.drive);
|
|
102
|
-
break;
|
|
103
|
-
}
|
|
104
|
-
}
|
|
98
|
+
if (!transmitter) {
|
|
99
|
+
throw new Error("Transmitter not found");
|
|
100
|
+
}
|
|
105
101
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
102
|
+
const driveTransmitters = this.transmitters[drive] || {};
|
|
103
|
+
driveTransmitters[listener.listenerId] = transmitter;
|
|
104
|
+
this.transmitters[drive] = driveTransmitters;
|
|
105
|
+
return Promise.resolve(transmitter);
|
|
106
|
+
}
|
|
109
107
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
108
|
+
async removeListener(driveId: string, listenerId: string) {
|
|
109
|
+
const driveMap = this.listenerState.get(driveId);
|
|
110
|
+
if (!driveMap) {
|
|
111
|
+
return false;
|
|
114
112
|
}
|
|
115
113
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
if (!driveMap) {
|
|
119
|
-
return false;
|
|
120
|
-
}
|
|
114
|
+
return Promise.resolve(driveMap.delete(listenerId));
|
|
115
|
+
}
|
|
121
116
|
|
|
122
|
-
|
|
117
|
+
async removeSyncUnits(
|
|
118
|
+
driveId: string,
|
|
119
|
+
syncUnits: Pick<SynchronizationUnit, "syncId">[],
|
|
120
|
+
) {
|
|
121
|
+
const listeners = this.listenerState.get(driveId);
|
|
122
|
+
if (!listeners) {
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
for (const [, listener] of listeners) {
|
|
126
|
+
for (const syncUnit of syncUnits) {
|
|
127
|
+
listener.syncUnits.delete(syncUnit.syncId);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return Promise.resolve();
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async updateSynchronizationRevisions(
|
|
134
|
+
driveId: string,
|
|
135
|
+
syncUnits: SynchronizationUnit[],
|
|
136
|
+
source: StrandUpdateSource,
|
|
137
|
+
willUpdate?: (listeners: Listener[]) => void,
|
|
138
|
+
onError?: (error: Error, driveId: string, listener: ListenerState) => void,
|
|
139
|
+
forceSync = false,
|
|
140
|
+
) {
|
|
141
|
+
const drive = this.listenerState.get(driveId);
|
|
142
|
+
if (!drive) {
|
|
143
|
+
return [];
|
|
123
144
|
}
|
|
124
145
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
146
|
+
const outdatedListeners: Listener[] = [];
|
|
147
|
+
for (const [, listener] of drive) {
|
|
148
|
+
if (
|
|
149
|
+
outdatedListeners.find(
|
|
150
|
+
(l) => l.listenerId === listener.listener.listenerId,
|
|
151
|
+
)
|
|
152
|
+
) {
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const transmitter = await this.getTransmitter(
|
|
157
|
+
driveId,
|
|
158
|
+
listener.listener.listenerId,
|
|
159
|
+
);
|
|
160
|
+
if (!transmitter?.transmit) {
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
for (const syncUnit of syncUnits) {
|
|
165
|
+
if (!this._checkFilter(listener.listener.filter, syncUnit)) {
|
|
166
|
+
continue;
|
|
129
167
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
168
|
+
|
|
169
|
+
const listenerRev = listener.syncUnits.get(syncUnit.syncId);
|
|
170
|
+
|
|
171
|
+
if (!listenerRev || listenerRev.listenerRev < syncUnit.revision) {
|
|
172
|
+
outdatedListeners.push(listener.listener);
|
|
173
|
+
break;
|
|
134
174
|
}
|
|
135
|
-
|
|
175
|
+
}
|
|
136
176
|
}
|
|
137
177
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
178
|
+
if (outdatedListeners.length) {
|
|
179
|
+
willUpdate?.(outdatedListeners);
|
|
180
|
+
return this.triggerUpdate(forceSync, source, onError);
|
|
181
|
+
}
|
|
182
|
+
return [];
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async updateListenerRevision(
|
|
186
|
+
listenerId: string,
|
|
187
|
+
driveId: string,
|
|
188
|
+
syncId: string,
|
|
189
|
+
listenerRev: number,
|
|
190
|
+
): Promise<void> {
|
|
191
|
+
const drive = this.listenerState.get(driveId);
|
|
192
|
+
if (!drive) {
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
153
195
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
l => l.listenerId === listener.listener.listenerId
|
|
159
|
-
)
|
|
160
|
-
) {
|
|
161
|
-
continue;
|
|
162
|
-
}
|
|
163
|
-
for (const syncUnit of syncUnits) {
|
|
164
|
-
if (!this._checkFilter(listener.listener.filter, syncUnit)) {
|
|
165
|
-
continue;
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
const listenerRev = listener.syncUnits.get(syncUnit.syncId);
|
|
169
|
-
|
|
170
|
-
if (
|
|
171
|
-
!listenerRev ||
|
|
172
|
-
listenerRev.listenerRev < syncUnit.revision
|
|
173
|
-
) {
|
|
174
|
-
outdatedListeners.push(listener.listener);
|
|
175
|
-
break;
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
}
|
|
196
|
+
const listener = drive.get(listenerId);
|
|
197
|
+
if (!listener) {
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
179
200
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
201
|
+
const lastUpdated = new Date().toISOString();
|
|
202
|
+
const entry = listener.syncUnits.get(syncId);
|
|
203
|
+
if (entry) {
|
|
204
|
+
entry.listenerRev = listenerRev;
|
|
205
|
+
entry.lastUpdated = lastUpdated;
|
|
206
|
+
} else {
|
|
207
|
+
listener.syncUnits.set(syncId, { listenerRev, lastUpdated });
|
|
185
208
|
}
|
|
186
209
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
210
|
+
return Promise.resolve();
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
triggerUpdate = debounce(
|
|
214
|
+
this._triggerUpdate.bind(this),
|
|
215
|
+
ListenerManager.LISTENER_UPDATE_DELAY,
|
|
216
|
+
);
|
|
217
|
+
|
|
218
|
+
private async _triggerUpdate(
|
|
219
|
+
source: StrandUpdateSource,
|
|
220
|
+
onError?: (error: Error, driveId: string, listener: ListenerState) => void,
|
|
221
|
+
) {
|
|
222
|
+
const listenerUpdates: ListenerUpdate[] = [];
|
|
223
|
+
for (const [driveId, drive] of this.listenerState) {
|
|
224
|
+
for (const [id, listener] of drive) {
|
|
225
|
+
const transmitter = await this.getTransmitter(driveId, id);
|
|
226
|
+
if (!transmitter?.transmit) {
|
|
227
|
+
continue;
|
|
196
228
|
}
|
|
197
229
|
|
|
198
|
-
const
|
|
199
|
-
|
|
230
|
+
const syncUnits = await this.getListenerSyncUnits(
|
|
231
|
+
driveId,
|
|
232
|
+
listener.listener.listenerId,
|
|
233
|
+
);
|
|
234
|
+
|
|
235
|
+
const strandUpdates: StrandUpdate[] = [];
|
|
236
|
+
// TODO change to push one after the other, reusing operation data
|
|
237
|
+
const tasks = syncUnits.map((syncUnit) => async () => {
|
|
238
|
+
const unitState = listener.syncUnits.get(syncUnit.syncId);
|
|
239
|
+
|
|
240
|
+
if (unitState && unitState.listenerRev >= syncUnit.revision) {
|
|
200
241
|
return;
|
|
201
|
-
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const opData: OperationUpdate[] = [];
|
|
245
|
+
try {
|
|
246
|
+
const data = await this.drive.getOperationData(
|
|
247
|
+
// TODO - join queries, DEAL WITH INVALID SYNC ID ERROR
|
|
248
|
+
driveId,
|
|
249
|
+
syncUnit.syncId,
|
|
250
|
+
{
|
|
251
|
+
fromRevision: unitState?.listenerRev,
|
|
252
|
+
},
|
|
253
|
+
);
|
|
254
|
+
opData.push(...data);
|
|
255
|
+
} catch (e) {
|
|
256
|
+
logger.error(e);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
if (!opData.length) {
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
202
262
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
263
|
+
strandUpdates.push({
|
|
264
|
+
driveId,
|
|
265
|
+
documentId: syncUnit.documentId,
|
|
266
|
+
branch: syncUnit.branch,
|
|
267
|
+
operations: opData,
|
|
268
|
+
scope: syncUnit.scope as OperationScope,
|
|
269
|
+
});
|
|
270
|
+
});
|
|
271
|
+
if (this.options.sequentialUpdates) {
|
|
272
|
+
for (const task of tasks) {
|
|
273
|
+
await task();
|
|
274
|
+
}
|
|
208
275
|
} else {
|
|
209
|
-
|
|
276
|
+
await Promise.all(tasks.map((task) => task()));
|
|
210
277
|
}
|
|
211
278
|
|
|
212
|
-
|
|
213
|
-
|
|
279
|
+
if (strandUpdates.length == 0) {
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
214
282
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
283
|
+
listener.pendingTimeout = new Date(
|
|
284
|
+
new Date().getTime() / 1000 + 300,
|
|
285
|
+
).toISOString();
|
|
286
|
+
listener.listenerStatus = "PENDING";
|
|
287
|
+
|
|
288
|
+
// TODO update listeners in parallel, blocking for listeners with block=true
|
|
289
|
+
try {
|
|
290
|
+
const listenerRevisions = await transmitter.transmit(
|
|
291
|
+
strandUpdates,
|
|
292
|
+
source,
|
|
293
|
+
);
|
|
294
|
+
|
|
295
|
+
listener.pendingTimeout = "0";
|
|
296
|
+
listener.listenerStatus = "PENDING";
|
|
297
|
+
|
|
298
|
+
const lastUpdated = new Date().toISOString();
|
|
299
|
+
|
|
300
|
+
for (const revision of listenerRevisions) {
|
|
301
|
+
const syncUnit = syncUnits.find(
|
|
302
|
+
(unit) =>
|
|
303
|
+
revision.documentId === unit.documentId &&
|
|
304
|
+
revision.scope === unit.scope &&
|
|
305
|
+
revision.branch === unit.branch,
|
|
306
|
+
);
|
|
307
|
+
if (syncUnit) {
|
|
308
|
+
listener.syncUnits.set(syncUnit.syncId, {
|
|
309
|
+
lastUpdated,
|
|
310
|
+
listenerRev: revision.revision,
|
|
311
|
+
});
|
|
312
|
+
} else {
|
|
313
|
+
logger.warn(
|
|
314
|
+
`Received revision for untracked unit for listener ${listener.listener.listenerId}`,
|
|
315
|
+
revision,
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
219
319
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
error
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
driveId,
|
|
237
|
-
listener.listener.listenerId
|
|
320
|
+
for (const revision of listenerRevisions) {
|
|
321
|
+
const error = revision.status === "ERROR";
|
|
322
|
+
if (revision.error?.includes("Missing operations")) {
|
|
323
|
+
const updates = await this._triggerUpdate(source, onError);
|
|
324
|
+
listenerUpdates.push(...updates);
|
|
325
|
+
} else {
|
|
326
|
+
listenerUpdates.push({
|
|
327
|
+
listenerId: listener.listener.listenerId,
|
|
328
|
+
listenerRevisions,
|
|
329
|
+
});
|
|
330
|
+
if (error) {
|
|
331
|
+
throw new OperationError(
|
|
332
|
+
revision.status as ErrorStatus,
|
|
333
|
+
undefined,
|
|
334
|
+
revision.error,
|
|
335
|
+
revision.error,
|
|
238
336
|
);
|
|
239
|
-
|
|
240
|
-
const strandUpdates: StrandUpdate[] = [];
|
|
241
|
-
for (const syncUnit of syncUnits) {
|
|
242
|
-
const unitState = listener.syncUnits.get(syncUnit.syncId);
|
|
243
|
-
|
|
244
|
-
if (
|
|
245
|
-
unitState &&
|
|
246
|
-
unitState.listenerRev >= syncUnit.revision
|
|
247
|
-
) {
|
|
248
|
-
continue;
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
const opData: OperationUpdate[] = [];
|
|
252
|
-
try {
|
|
253
|
-
const data = await this.drive.getOperationData( // TODO - join queries, DEAL WITH INVALID SYNC ID ERROR
|
|
254
|
-
driveId,
|
|
255
|
-
syncUnit.syncId,
|
|
256
|
-
{
|
|
257
|
-
fromRevision: unitState?.listenerRev
|
|
258
|
-
}
|
|
259
|
-
);
|
|
260
|
-
opData.push(...data);
|
|
261
|
-
} catch (e) {
|
|
262
|
-
logger.error(e);
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
if (!opData.length) {
|
|
266
|
-
continue;
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
strandUpdates.push({
|
|
270
|
-
driveId,
|
|
271
|
-
documentId: syncUnit.documentId,
|
|
272
|
-
branch: syncUnit.branch,
|
|
273
|
-
operations: opData,
|
|
274
|
-
scope: syncUnit.scope as OperationScope
|
|
275
|
-
});
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
if (strandUpdates.length == 0) {
|
|
279
|
-
continue;
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
listener.pendingTimeout = new Date(
|
|
283
|
-
new Date().getTime() / 1000 + 300
|
|
284
|
-
).toISOString();
|
|
285
|
-
listener.listenerStatus = 'PENDING';
|
|
286
|
-
|
|
287
|
-
// TODO update listeners in parallel, blocking for listeners with block=true
|
|
288
|
-
try {
|
|
289
|
-
const listenerRevisions =
|
|
290
|
-
await transmitter?.transmit(strandUpdates);
|
|
291
|
-
|
|
292
|
-
listener.pendingTimeout = '0';
|
|
293
|
-
listener.listenerStatus = 'PENDING';
|
|
294
|
-
|
|
295
|
-
const lastUpdated = new Date().toISOString();
|
|
296
|
-
|
|
297
|
-
for (const revision of listenerRevisions) {
|
|
298
|
-
const syncUnit = syncUnits.find(
|
|
299
|
-
unit =>
|
|
300
|
-
revision.documentId === unit.documentId &&
|
|
301
|
-
revision.scope === unit.scope &&
|
|
302
|
-
revision.branch === unit.branch
|
|
303
|
-
);
|
|
304
|
-
if (syncUnit) {
|
|
305
|
-
listener.syncUnits.set(syncUnit.syncId, {
|
|
306
|
-
lastUpdated,
|
|
307
|
-
listenerRev: revision.revision
|
|
308
|
-
});
|
|
309
|
-
} else {
|
|
310
|
-
logger.warn(
|
|
311
|
-
`Received revision for untracked unit for listener ${listener.listener.listenerId}`,
|
|
312
|
-
revision
|
|
313
|
-
);
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
const revisionError = listenerRevisions.find(
|
|
317
|
-
l => l.status !== 'SUCCESS'
|
|
318
|
-
);
|
|
319
|
-
if (revisionError) {
|
|
320
|
-
throw new OperationError(
|
|
321
|
-
revisionError.status as ErrorStatus,
|
|
322
|
-
undefined,
|
|
323
|
-
revisionError.error,
|
|
324
|
-
revisionError.error
|
|
325
|
-
);
|
|
326
|
-
}
|
|
327
|
-
listener.listenerStatus = 'SUCCESS';
|
|
328
|
-
listenerUpdates.push({
|
|
329
|
-
listenerId: listener.listener.listenerId,
|
|
330
|
-
listenerRevisions
|
|
331
|
-
});
|
|
332
|
-
} catch (e) {
|
|
333
|
-
// TODO: Handle error based on listener params (blocking, retry, etc)
|
|
334
|
-
onError?.(e as Error, driveId, listener);
|
|
335
|
-
listener.listenerStatus =
|
|
336
|
-
e instanceof OperationError ? e.status : 'ERROR';
|
|
337
|
-
}
|
|
337
|
+
}
|
|
338
338
|
}
|
|
339
|
+
}
|
|
340
|
+
listener.listenerStatus = "SUCCESS";
|
|
341
|
+
} catch (e) {
|
|
342
|
+
// TODO: Handle error based on listener params (blocking, retry, etc)
|
|
343
|
+
onError?.(e as Error, driveId, listener);
|
|
344
|
+
listener.listenerStatus =
|
|
345
|
+
e instanceof OperationError ? e.status : "ERROR";
|
|
339
346
|
}
|
|
340
|
-
|
|
347
|
+
}
|
|
341
348
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
349
|
+
return listenerUpdates;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
private _checkFilter(filter: ListenerFilter, syncUnit: SynchronizationUnit) {
|
|
353
|
+
const { branch, documentId, scope, documentType } = syncUnit;
|
|
354
|
+
// TODO: Needs to be optimized
|
|
355
|
+
if (
|
|
356
|
+
(!filter.branch ||
|
|
357
|
+
filter.branch.includes(branch) ||
|
|
358
|
+
filter.branch.includes("*")) &&
|
|
359
|
+
(!filter.documentId ||
|
|
360
|
+
filter.documentId.includes(documentId) ||
|
|
361
|
+
filter.documentId.includes("*")) &&
|
|
362
|
+
(!filter.scope ||
|
|
363
|
+
filter.scope.includes(scope) ||
|
|
364
|
+
filter.scope.includes("*")) &&
|
|
365
|
+
(!filter.documentType ||
|
|
366
|
+
filter.documentType.includes(documentType) ||
|
|
367
|
+
filter.documentType.includes("*"))
|
|
346
368
|
) {
|
|
347
|
-
|
|
348
|
-
// TODO: Needs to be optimized
|
|
349
|
-
if (
|
|
350
|
-
(!filter.branch ||
|
|
351
|
-
filter.branch.includes(branch) ||
|
|
352
|
-
filter.branch.includes('*')) &&
|
|
353
|
-
(!filter.documentId ||
|
|
354
|
-
filter.documentId.includes(documentId) ||
|
|
355
|
-
filter.documentId.includes('*')) &&
|
|
356
|
-
(!filter.scope ||
|
|
357
|
-
filter.scope.includes(scope) ||
|
|
358
|
-
filter.scope.includes('*')) &&
|
|
359
|
-
(!filter.documentType ||
|
|
360
|
-
filter.documentType.includes(documentType) ||
|
|
361
|
-
filter.documentType.includes('*'))
|
|
362
|
-
) {
|
|
363
|
-
return true;
|
|
364
|
-
}
|
|
365
|
-
return false;
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
getListenerSyncUnits(driveId: string, listenerId: string) {
|
|
369
|
-
const listener = this.listenerState.get(driveId)?.get(listenerId);
|
|
370
|
-
if (!listener) {
|
|
371
|
-
return [];
|
|
372
|
-
}
|
|
373
|
-
const filter = listener.listener.filter;
|
|
374
|
-
return this.drive.getSynchronizationUnits(
|
|
375
|
-
driveId,
|
|
376
|
-
filter.documentId ?? ['*'],
|
|
377
|
-
filter.scope ?? ['*'],
|
|
378
|
-
filter.branch ?? ['*'],
|
|
379
|
-
filter.documentType ?? ['*']
|
|
380
|
-
);
|
|
369
|
+
return true;
|
|
381
370
|
}
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
filter.branch ?? ['*'],
|
|
394
|
-
filter.documentType ?? ['*']
|
|
395
|
-
);
|
|
371
|
+
return false;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
getListenerSyncUnits(
|
|
375
|
+
driveId: string,
|
|
376
|
+
listenerId: string,
|
|
377
|
+
loadedDrive?: DocumentDriveDocument,
|
|
378
|
+
) {
|
|
379
|
+
const listener = this.listenerState.get(driveId)?.get(listenerId);
|
|
380
|
+
if (!listener) {
|
|
381
|
+
return [];
|
|
396
382
|
}
|
|
383
|
+
const filter = listener.listener.filter;
|
|
384
|
+
return this.drive.getSynchronizationUnits(
|
|
385
|
+
driveId,
|
|
386
|
+
filter.documentId ?? ["*"],
|
|
387
|
+
filter.scope ?? ["*"],
|
|
388
|
+
filter.branch ?? ["*"],
|
|
389
|
+
filter.documentType ?? ["*"],
|
|
390
|
+
loadedDrive,
|
|
391
|
+
);
|
|
392
|
+
}
|
|
397
393
|
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
}
|
|
403
|
-
} = drive;
|
|
404
|
-
|
|
405
|
-
for (const listener of listeners) {
|
|
406
|
-
await this.addListener({
|
|
407
|
-
block: listener.block,
|
|
408
|
-
driveId: drive.state.global.id,
|
|
409
|
-
filter: {
|
|
410
|
-
branch: listener.filter.branch ?? [],
|
|
411
|
-
documentId: listener.filter.documentId ?? [],
|
|
412
|
-
documentType: listener.filter.documentType,
|
|
413
|
-
scope: listener.filter.scope ?? []
|
|
414
|
-
},
|
|
415
|
-
listenerId: listener.listenerId,
|
|
416
|
-
system: listener.system,
|
|
417
|
-
callInfo: listener.callInfo ?? undefined,
|
|
418
|
-
label: listener.label ?? ''
|
|
419
|
-
});
|
|
420
|
-
}
|
|
394
|
+
getListenerSyncUnitIds(driveId: string, listenerId: string) {
|
|
395
|
+
const listener = this.listenerState.get(driveId)?.get(listenerId);
|
|
396
|
+
if (!listener) {
|
|
397
|
+
return [];
|
|
421
398
|
}
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
399
|
+
const filter = listener.listener.filter;
|
|
400
|
+
return this.drive.getSynchronizationUnitsIds(
|
|
401
|
+
driveId,
|
|
402
|
+
filter.documentId ?? ["*"],
|
|
403
|
+
filter.scope ?? ["*"],
|
|
404
|
+
filter.branch ?? ["*"],
|
|
405
|
+
filter.documentType ?? ["*"],
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
async initDrive(drive: DocumentDriveDocument) {
|
|
410
|
+
const {
|
|
411
|
+
state: {
|
|
412
|
+
local: { listeners },
|
|
413
|
+
},
|
|
414
|
+
} = drive;
|
|
415
|
+
|
|
416
|
+
for (const listener of listeners) {
|
|
417
|
+
await this.addListener({
|
|
418
|
+
block: listener.block,
|
|
419
|
+
driveId: drive.state.global.id,
|
|
420
|
+
filter: {
|
|
421
|
+
branch: listener.filter.branch ?? [],
|
|
422
|
+
documentId: listener.filter.documentId ?? [],
|
|
423
|
+
documentType: listener.filter.documentType,
|
|
424
|
+
scope: listener.filter.scope ?? [],
|
|
425
|
+
},
|
|
426
|
+
listenerId: listener.listenerId,
|
|
427
|
+
system: listener.system,
|
|
428
|
+
callInfo: listener.callInfo ?? undefined,
|
|
429
|
+
label: listener.label ?? "",
|
|
430
|
+
});
|
|
429
431
|
}
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
async removeDrive(driveId: string): Promise<void> {
|
|
435
|
+
this.listenerState.delete(driveId);
|
|
436
|
+
const transmitters = this.transmitters[driveId];
|
|
437
|
+
if (transmitters) {
|
|
438
|
+
await Promise.all(
|
|
439
|
+
Object.values(transmitters).map((t) => t.disconnect?.()),
|
|
440
|
+
);
|
|
437
441
|
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
getListener(driveId: string, listenerId: string): Promise<ListenerState> {
|
|
445
|
+
const drive = this.listenerState.get(driveId);
|
|
446
|
+
if (!drive) throw new Error("Drive not found");
|
|
447
|
+
const listener = drive.get(listenerId);
|
|
448
|
+
if (!listener) throw new Error("Listener not found");
|
|
449
|
+
return Promise.resolve(listener);
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
async getStrands(
|
|
453
|
+
driveId: string,
|
|
454
|
+
listenerId: string,
|
|
455
|
+
options?: GetStrandsOptions,
|
|
456
|
+
): Promise<StrandUpdate[]> {
|
|
457
|
+
// fetch listenerState from listenerManager
|
|
458
|
+
const listener = await this.getListener(driveId, listenerId);
|
|
459
|
+
|
|
460
|
+
// fetch operations from drive and prepare strands
|
|
461
|
+
const strands: StrandUpdate[] = [];
|
|
462
|
+
|
|
463
|
+
const drive = await this.drive.getDrive(driveId);
|
|
464
|
+
const syncUnits = await this.getListenerSyncUnits(
|
|
465
|
+
driveId,
|
|
466
|
+
listenerId,
|
|
467
|
+
drive,
|
|
468
|
+
);
|
|
438
469
|
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
//
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
fromRevision: entry?.listenerRev
|
|
469
|
-
}
|
|
470
|
-
);
|
|
470
|
+
const limit = options?.limit; // maximum number of operations to send across all sync units
|
|
471
|
+
let operationsCount = 0; // total amount of operations that have been retrieved
|
|
472
|
+
|
|
473
|
+
const tasks = syncUnits.map((syncUnit) => async () => {
|
|
474
|
+
if (limit && operationsCount >= limit) {
|
|
475
|
+
// break;
|
|
476
|
+
return;
|
|
477
|
+
}
|
|
478
|
+
if (syncUnit.revision < 0) {
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
const entry = listener.syncUnits.get(syncUnit.syncId);
|
|
482
|
+
if (entry && entry.listenerRev >= syncUnit.revision) {
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
const { documentId, driveId, scope, branch } = syncUnit;
|
|
487
|
+
try {
|
|
488
|
+
const operations = await this.drive.getOperationData(
|
|
489
|
+
// DEAL WITH INVALID SYNC ID ERROR
|
|
490
|
+
driveId,
|
|
491
|
+
syncUnit.syncId,
|
|
492
|
+
{
|
|
493
|
+
since: options?.since,
|
|
494
|
+
fromRevision: options?.fromRevision ?? entry?.listenerRev,
|
|
495
|
+
limit: limit ? limit - operationsCount : undefined,
|
|
496
|
+
},
|
|
497
|
+
drive,
|
|
498
|
+
);
|
|
471
499
|
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
strands.push({
|
|
477
|
-
driveId,
|
|
478
|
-
documentId,
|
|
479
|
-
scope: scope as OperationScope,
|
|
480
|
-
branch,
|
|
481
|
-
operations
|
|
482
|
-
});
|
|
483
|
-
} catch (error) {
|
|
484
|
-
logger.error(error);
|
|
485
|
-
continue;
|
|
486
|
-
}
|
|
500
|
+
if (!operations.length) {
|
|
501
|
+
return;
|
|
487
502
|
}
|
|
488
503
|
|
|
489
|
-
|
|
504
|
+
operationsCount += operations.length;
|
|
505
|
+
|
|
506
|
+
strands.push({
|
|
507
|
+
driveId,
|
|
508
|
+
documentId,
|
|
509
|
+
scope: scope as OperationScope,
|
|
510
|
+
branch,
|
|
511
|
+
operations,
|
|
512
|
+
});
|
|
513
|
+
} catch (error) {
|
|
514
|
+
logger.error(error);
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
if (this.options.sequentialUpdates) {
|
|
520
|
+
for (const task of tasks) {
|
|
521
|
+
await task();
|
|
522
|
+
}
|
|
523
|
+
} else {
|
|
524
|
+
await Promise.all(tasks.map((task) => task()));
|
|
490
525
|
}
|
|
526
|
+
|
|
527
|
+
return strands;
|
|
528
|
+
}
|
|
491
529
|
}
|