document-drive 1.0.0-websockets.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +1 -0
  2. package/package.json +74 -88
  3. package/src/cache/index.ts +2 -2
  4. package/src/cache/memory.ts +22 -13
  5. package/src/cache/redis.ts +43 -16
  6. package/src/cache/types.ts +4 -4
  7. package/src/index.ts +6 -3
  8. package/src/queue/base.ts +276 -214
  9. package/src/queue/index.ts +2 -2
  10. package/src/queue/redis.ts +138 -127
  11. package/src/queue/types.ts +44 -38
  12. package/src/read-mode/errors.ts +19 -0
  13. package/src/read-mode/index.ts +125 -0
  14. package/src/read-mode/service.ts +207 -0
  15. package/src/read-mode/types.ts +108 -0
  16. package/src/server/error.ts +61 -26
  17. package/src/server/index.ts +2160 -1785
  18. package/src/server/listener/index.ts +2 -2
  19. package/src/server/listener/manager.ts +475 -437
  20. package/src/server/listener/transmitter/index.ts +4 -5
  21. package/src/server/listener/transmitter/internal.ts +77 -79
  22. package/src/server/listener/transmitter/pull-responder.ts +363 -329
  23. package/src/server/listener/transmitter/switchboard-push.ts +72 -55
  24. package/src/server/listener/transmitter/types.ts +19 -25
  25. package/src/server/types.ts +536 -349
  26. package/src/server/utils.ts +26 -27
  27. package/src/storage/base.ts +81 -0
  28. package/src/storage/browser.ts +233 -216
  29. package/src/storage/filesystem.ts +257 -256
  30. package/src/storage/index.ts +2 -1
  31. package/src/storage/memory.ts +206 -214
  32. package/src/storage/prisma.ts +575 -568
  33. package/src/storage/sequelize.ts +460 -471
  34. package/src/storage/types.ts +83 -67
  35. package/src/utils/default-drives-manager.ts +341 -0
  36. package/src/utils/document-helpers.ts +19 -18
  37. package/src/utils/graphql.ts +288 -34
  38. package/src/utils/index.ts +61 -59
  39. package/src/utils/logger.ts +39 -37
  40. package/src/utils/migrations.ts +58 -0
  41. package/src/utils/run-asap.ts +156 -0
  42. package/CHANGELOG.md +0 -818
  43. package/src/server/listener/transmitter/subscription.ts +0 -364
@@ -1,491 +1,529 @@
1
1
  import {
2
- DocumentDriveDocument,
3
- ListenerFilter
4
- } from 'document-model-libs/document-drive';
5
- import { OperationScope } from 'document-model/document';
6
- import { OperationError } from '../error';
2
+ DocumentDriveDocument,
3
+ ListenerFilter,
4
+ } from "document-model-libs/document-drive";
5
+ import { OperationScope } from "document-model/document";
6
+ import { logger } from "../../utils/logger";
7
+ import { OperationError } from "../error";
7
8
  import {
8
- BaseListenerManager,
9
- ErrorStatus,
10
- Listener,
11
- ListenerState,
12
- ListenerUpdate,
13
- OperationUpdate,
14
- StrandUpdate,
15
- SynchronizationUnit
16
- } from '../types';
17
- import { PullResponderTransmitter, SubscriptionTransmitter } from './transmitter';
18
- import { InternalTransmitter } from './transmitter/internal';
19
- import { SwitchboardPushTransmitter } from './transmitter/switchboard-push';
20
- import { ITransmitter } from './transmitter/types';
21
- import { logger } from '../../utils/logger';
9
+ BaseListenerManager,
10
+ ErrorStatus,
11
+ GetStrandsOptions,
12
+ Listener,
13
+ ListenerState,
14
+ ListenerUpdate,
15
+ OperationUpdate,
16
+ StrandUpdate,
17
+ SynchronizationUnit,
18
+ } from "../types";
19
+ import { PullResponderTransmitter } from "./transmitter";
20
+ import { InternalTransmitter } from "./transmitter/internal";
21
+ import { SwitchboardPushTransmitter } from "./transmitter/switchboard-push";
22
+ import { ITransmitter, StrandUpdateSource } from "./transmitter/types";
22
23
 
23
24
  function debounce<T extends unknown[], R>(
24
- func: (...args: T) => Promise<R>,
25
- delay = 250
25
+ func: (...args: T) => Promise<R>,
26
+ delay = 250,
26
27
  ) {
27
- let timer: number;
28
- return (immediate: boolean, ...args: T) => {
29
- if (timer) {
30
- clearTimeout(timer);
31
- }
32
- return new Promise<R>((resolve, reject) => {
33
- if (immediate) {
34
- func(...args)
35
- .then(resolve)
36
- .catch(reject);
37
- } else {
38
- timer = setTimeout(() => {
39
- // eslint-disable-next-line @typescript-eslint/no-unsafe-call
40
- func(...args)
41
- .then(resolve)
42
- .catch(reject);
43
- }, delay) as unknown as number;
44
- }
45
- });
46
- };
28
+ let timer: number;
29
+ return (immediate: boolean, ...args: T) => {
30
+ if (timer) {
31
+ clearTimeout(timer);
32
+ }
33
+ return new Promise<R>((resolve, reject) => {
34
+ if (immediate) {
35
+ func(...args)
36
+ .then(resolve)
37
+ .catch(reject);
38
+ } else {
39
+ timer = setTimeout(() => {
40
+ func(...args)
41
+ .then(resolve)
42
+ .catch(reject);
43
+ }, delay) as unknown as number;
44
+ }
45
+ });
46
+ };
47
47
  }
48
-
49
48
  export class ListenerManager extends BaseListenerManager {
50
- static LISTENER_UPDATE_DELAY = 250;
49
+ static LISTENER_UPDATE_DELAY = 250;
51
50
 
52
- async getTransmitter(
53
- driveId: string,
54
- listenerId: string
55
- ): Promise<ITransmitter | undefined> {
56
- return Promise.resolve(this.transmitters[driveId]?.[listenerId]);
57
- }
51
+ async getTransmitter(
52
+ driveId: string,
53
+ listenerId: string,
54
+ ): Promise<ITransmitter | undefined> {
55
+ return Promise.resolve(this.transmitters[driveId]?.[listenerId]);
56
+ }
58
57
 
59
- async addListener(listener: Listener) {
60
- const drive = listener.driveId;
58
+ driveHasListeners(driveId: string) {
59
+ return this.listenerState.has(driveId);
60
+ }
61
61
 
62
- if (!this.listenerState.has(drive)) {
63
- this.listenerState.set(drive, new Map());
64
- }
62
+ async addListener(listener: Listener) {
63
+ const drive = listener.driveId;
65
64
 
66
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
67
- const driveMap = this.listenerState.get(drive)!;
68
- driveMap.set(listener.listenerId, {
69
- block: listener.block,
70
- driveId: listener.driveId,
71
- pendingTimeout: '0',
72
- listener,
73
- listenerStatus: 'CREATED',
74
- syncUnits: new Map()
75
- });
76
-
77
- let transmitter: ITransmitter | undefined;
65
+ if (!this.listenerState.has(drive)) {
66
+ this.listenerState.set(drive, new Map());
67
+ }
78
68
 
79
- switch (listener.callInfo?.transmitterType) {
80
- case 'SwitchboardPush': {
81
- transmitter = new SwitchboardPushTransmitter(
82
- listener,
83
- this.drive
84
- );
85
- break;
86
- }
69
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
70
+ const driveMap = this.listenerState.get(drive)!;
71
+ driveMap.set(listener.listenerId, {
72
+ block: listener.block,
73
+ driveId: listener.driveId,
74
+ pendingTimeout: "0",
75
+ listener,
76
+ listenerStatus: "CREATED",
77
+ syncUnits: new Map(),
78
+ });
79
+
80
+ let transmitter: ITransmitter | undefined;
81
+
82
+ switch (listener.callInfo?.transmitterType) {
83
+ case "SwitchboardPush": {
84
+ transmitter = new SwitchboardPushTransmitter(listener, this.drive);
85
+ break;
86
+ }
87
+
88
+ case "PullResponder": {
89
+ transmitter = new PullResponderTransmitter(listener, this.drive, this);
90
+ break;
91
+ }
92
+ case "Internal": {
93
+ transmitter = new InternalTransmitter(listener, this.drive);
94
+ break;
95
+ }
96
+ }
87
97
 
88
- case 'PullResponder': {
89
- transmitter = new PullResponderTransmitter(
90
- listener,
91
- this.drive,
92
- this
93
- );
94
- break;
95
- }
96
- case "Subscription": {
97
- transmitter = new SubscriptionTransmitter(listener, this);
98
- break;
99
- }
100
- case 'Internal': {
101
- transmitter = new InternalTransmitter(listener, this.drive);
102
- break;
103
- }
104
- }
98
+ if (!transmitter) {
99
+ throw new Error("Transmitter not found");
100
+ }
105
101
 
106
- if (!transmitter) {
107
- throw new Error('Transmitter not found');
108
- }
102
+ const driveTransmitters = this.transmitters[drive] || {};
103
+ driveTransmitters[listener.listenerId] = transmitter;
104
+ this.transmitters[drive] = driveTransmitters;
105
+ return Promise.resolve(transmitter);
106
+ }
109
107
 
110
- const driveTransmitters = this.transmitters[drive] || {};
111
- driveTransmitters[listener.listenerId] = transmitter;
112
- this.transmitters[drive] = driveTransmitters;
113
- return Promise.resolve(transmitter);
108
+ async removeListener(driveId: string, listenerId: string) {
109
+ const driveMap = this.listenerState.get(driveId);
110
+ if (!driveMap) {
111
+ return false;
114
112
  }
115
113
 
116
- async removeListener(driveId: string, listenerId: string) {
117
- const driveMap = this.listenerState.get(driveId);
118
- if (!driveMap) {
119
- return false;
120
- }
114
+ return Promise.resolve(driveMap.delete(listenerId));
115
+ }
121
116
 
122
- return Promise.resolve(driveMap.delete(listenerId));
117
+ async removeSyncUnits(
118
+ driveId: string,
119
+ syncUnits: Pick<SynchronizationUnit, "syncId">[],
120
+ ) {
121
+ const listeners = this.listenerState.get(driveId);
122
+ if (!listeners) {
123
+ return;
124
+ }
125
+ for (const [, listener] of listeners) {
126
+ for (const syncUnit of syncUnits) {
127
+ listener.syncUnits.delete(syncUnit.syncId);
128
+ }
129
+ }
130
+ return Promise.resolve();
131
+ }
132
+
133
+ async updateSynchronizationRevisions(
134
+ driveId: string,
135
+ syncUnits: SynchronizationUnit[],
136
+ source: StrandUpdateSource,
137
+ willUpdate?: (listeners: Listener[]) => void,
138
+ onError?: (error: Error, driveId: string, listener: ListenerState) => void,
139
+ forceSync = false,
140
+ ) {
141
+ const drive = this.listenerState.get(driveId);
142
+ if (!drive) {
143
+ return [];
123
144
  }
124
145
 
125
- async removeSyncUnits(driveId: string, syncUnits: Pick<SynchronizationUnit, "syncId">[]) {
126
- const listeners = this.listenerState.get(driveId);
127
- if (!listeners) {
128
- return;
146
+ const outdatedListeners: Listener[] = [];
147
+ for (const [, listener] of drive) {
148
+ if (
149
+ outdatedListeners.find(
150
+ (l) => l.listenerId === listener.listener.listenerId,
151
+ )
152
+ ) {
153
+ continue;
154
+ }
155
+
156
+ const transmitter = await this.getTransmitter(
157
+ driveId,
158
+ listener.listener.listenerId,
159
+ );
160
+ if (!transmitter?.transmit) {
161
+ continue;
162
+ }
163
+
164
+ for (const syncUnit of syncUnits) {
165
+ if (!this._checkFilter(listener.listener.filter, syncUnit)) {
166
+ continue;
129
167
  }
130
- for (const [, listener] of listeners) {
131
- for (const syncUnit of syncUnits) {
132
- listener.syncUnits.delete(syncUnit.syncId);
133
- }
168
+
169
+ const listenerRev = listener.syncUnits.get(syncUnit.syncId);
170
+
171
+ if (!listenerRev || listenerRev.listenerRev < syncUnit.revision) {
172
+ outdatedListeners.push(listener.listener);
173
+ break;
134
174
  }
135
- return Promise.resolve();
175
+ }
136
176
  }
137
177
 
138
- async updateSynchronizationRevisions(
139
- driveId: string,
140
- syncUnits: SynchronizationUnit[],
141
- willUpdate?: (listeners: Listener[]) => void,
142
- onError?: (
143
- error: Error,
144
- driveId: string,
145
- listener: ListenerState
146
- ) => void,
147
- forceSync = false
148
- ) {
149
- const drive = this.listenerState.get(driveId);
150
- if (!drive) {
151
- return [];
152
- }
178
+ if (outdatedListeners.length) {
179
+ willUpdate?.(outdatedListeners);
180
+ return this.triggerUpdate(forceSync, source, onError);
181
+ }
182
+ return [];
183
+ }
184
+
185
+ async updateListenerRevision(
186
+ listenerId: string,
187
+ driveId: string,
188
+ syncId: string,
189
+ listenerRev: number,
190
+ ): Promise<void> {
191
+ const drive = this.listenerState.get(driveId);
192
+ if (!drive) {
193
+ return;
194
+ }
153
195
 
154
- const outdatedListeners: Listener[] = [];
155
- for (const [, listener] of drive) {
156
- if (
157
- outdatedListeners.find(
158
- l => l.listenerId === listener.listener.listenerId
159
- )
160
- ) {
161
- continue;
162
- }
163
- for (const syncUnit of syncUnits) {
164
- if (!this._checkFilter(listener.listener.filter, syncUnit)) {
165
- continue;
166
- }
167
-
168
- const listenerRev = listener.syncUnits.get(syncUnit.syncId);
169
-
170
- if (
171
- !listenerRev ||
172
- listenerRev.listenerRev < syncUnit.revision
173
- ) {
174
- outdatedListeners.push(listener.listener);
175
- break;
176
- }
177
- }
178
- }
196
+ const listener = drive.get(listenerId);
197
+ if (!listener) {
198
+ return;
199
+ }
179
200
 
180
- if (outdatedListeners.length) {
181
- willUpdate?.(outdatedListeners);
182
- return this.triggerUpdate(forceSync, onError);
183
- }
184
- return [];
201
+ const lastUpdated = new Date().toISOString();
202
+ const entry = listener.syncUnits.get(syncId);
203
+ if (entry) {
204
+ entry.listenerRev = listenerRev;
205
+ entry.lastUpdated = lastUpdated;
206
+ } else {
207
+ listener.syncUnits.set(syncId, { listenerRev, lastUpdated });
185
208
  }
186
209
 
187
- async updateListenerRevision(
188
- listenerId: string,
189
- driveId: string,
190
- syncId: string,
191
- listenerRev: number
192
- ): Promise<void> {
193
- const drive = this.listenerState.get(driveId);
194
- if (!drive) {
195
- return;
210
+ return Promise.resolve();
211
+ }
212
+
213
+ triggerUpdate = debounce(
214
+ this._triggerUpdate.bind(this),
215
+ ListenerManager.LISTENER_UPDATE_DELAY,
216
+ );
217
+
218
+ private async _triggerUpdate(
219
+ source: StrandUpdateSource,
220
+ onError?: (error: Error, driveId: string, listener: ListenerState) => void,
221
+ ) {
222
+ const listenerUpdates: ListenerUpdate[] = [];
223
+ for (const [driveId, drive] of this.listenerState) {
224
+ for (const [id, listener] of drive) {
225
+ const transmitter = await this.getTransmitter(driveId, id);
226
+ if (!transmitter?.transmit) {
227
+ continue;
196
228
  }
197
229
 
198
- const listener = drive.get(listenerId);
199
- if (!listener) {
230
+ const syncUnits = await this.getListenerSyncUnits(
231
+ driveId,
232
+ listener.listener.listenerId,
233
+ );
234
+
235
+ const strandUpdates: StrandUpdate[] = [];
236
+ // TODO change to push one after the other, reusing operation data
237
+ const tasks = syncUnits.map((syncUnit) => async () => {
238
+ const unitState = listener.syncUnits.get(syncUnit.syncId);
239
+
240
+ if (unitState && unitState.listenerRev >= syncUnit.revision) {
200
241
  return;
201
- }
242
+ }
243
+
244
+ const opData: OperationUpdate[] = [];
245
+ try {
246
+ const data = await this.drive.getOperationData(
247
+ // TODO - join queries, DEAL WITH INVALID SYNC ID ERROR
248
+ driveId,
249
+ syncUnit.syncId,
250
+ {
251
+ fromRevision: unitState?.listenerRev,
252
+ },
253
+ );
254
+ opData.push(...data);
255
+ } catch (e) {
256
+ logger.error(e);
257
+ }
258
+
259
+ if (!opData.length) {
260
+ return;
261
+ }
202
262
 
203
- const lastUpdated = new Date().toISOString();
204
- const entry = listener.syncUnits.get(syncId);
205
- if (entry) {
206
- entry.listenerRev = listenerRev;
207
- entry.lastUpdated = lastUpdated;
263
+ strandUpdates.push({
264
+ driveId,
265
+ documentId: syncUnit.documentId,
266
+ branch: syncUnit.branch,
267
+ operations: opData,
268
+ scope: syncUnit.scope as OperationScope,
269
+ });
270
+ });
271
+ if (this.options.sequentialUpdates) {
272
+ for (const task of tasks) {
273
+ await task();
274
+ }
208
275
  } else {
209
- listener.syncUnits.set(syncId, { listenerRev, lastUpdated });
276
+ await Promise.all(tasks.map((task) => task()));
210
277
  }
211
278
 
212
- return Promise.resolve();
213
- }
279
+ if (strandUpdates.length == 0) {
280
+ continue;
281
+ }
214
282
 
215
- triggerUpdate = debounce(
216
- this._triggerUpdate.bind(this),
217
- ListenerManager.LISTENER_UPDATE_DELAY
218
- );
283
+ listener.pendingTimeout = new Date(
284
+ new Date().getTime() / 1000 + 300,
285
+ ).toISOString();
286
+ listener.listenerStatus = "PENDING";
287
+
288
+ // TODO update listeners in parallel, blocking for listeners with block=true
289
+ try {
290
+ const listenerRevisions = await transmitter.transmit(
291
+ strandUpdates,
292
+ source,
293
+ );
294
+
295
+ listener.pendingTimeout = "0";
296
+ listener.listenerStatus = "PENDING";
297
+
298
+ const lastUpdated = new Date().toISOString();
299
+
300
+ for (const revision of listenerRevisions) {
301
+ const syncUnit = syncUnits.find(
302
+ (unit) =>
303
+ revision.documentId === unit.documentId &&
304
+ revision.scope === unit.scope &&
305
+ revision.branch === unit.branch,
306
+ );
307
+ if (syncUnit) {
308
+ listener.syncUnits.set(syncUnit.syncId, {
309
+ lastUpdated,
310
+ listenerRev: revision.revision,
311
+ });
312
+ } else {
313
+ logger.warn(
314
+ `Received revision for untracked unit for listener ${listener.listener.listenerId}`,
315
+ revision,
316
+ );
317
+ }
318
+ }
219
319
 
220
- private async _triggerUpdate(
221
- onError?: (
222
- error: Error,
223
- driveId: string,
224
- listener: ListenerState
225
- ) => void
226
- ) {
227
- const listenerUpdates: ListenerUpdate[] = [];
228
- for (const [driveId, drive] of this.listenerState) {
229
- for (const [id, listener] of drive) {
230
- const transmitter = await this.getTransmitter(driveId, id);
231
- if (!transmitter) {
232
- continue;
233
- }
234
-
235
- const syncUnits = await this.getListenerSyncUnits(
236
- driveId,
237
- listener.listener.listenerId
320
+ for (const revision of listenerRevisions) {
321
+ const error = revision.status === "ERROR";
322
+ if (revision.error?.includes("Missing operations")) {
323
+ const updates = await this._triggerUpdate(source, onError);
324
+ listenerUpdates.push(...updates);
325
+ } else {
326
+ listenerUpdates.push({
327
+ listenerId: listener.listener.listenerId,
328
+ listenerRevisions,
329
+ });
330
+ if (error) {
331
+ throw new OperationError(
332
+ revision.status as ErrorStatus,
333
+ undefined,
334
+ revision.error,
335
+ revision.error,
238
336
  );
239
-
240
- const strandUpdates: StrandUpdate[] = [];
241
- for (const syncUnit of syncUnits) {
242
- const unitState = listener.syncUnits.get(syncUnit.syncId);
243
-
244
- if (
245
- unitState &&
246
- unitState.listenerRev >= syncUnit.revision
247
- ) {
248
- continue;
249
- }
250
-
251
- const opData: OperationUpdate[] = [];
252
- try {
253
- const data = await this.drive.getOperationData( // TODO - join queries, DEAL WITH INVALID SYNC ID ERROR
254
- driveId,
255
- syncUnit.syncId,
256
- {
257
- fromRevision: unitState?.listenerRev
258
- }
259
- );
260
- opData.push(...data);
261
- } catch (e) {
262
- logger.error(e);
263
- }
264
-
265
- if (!opData.length) {
266
- continue;
267
- }
268
-
269
- strandUpdates.push({
270
- driveId,
271
- documentId: syncUnit.documentId,
272
- branch: syncUnit.branch,
273
- operations: opData,
274
- scope: syncUnit.scope as OperationScope
275
- });
276
- }
277
-
278
- if (strandUpdates.length == 0) {
279
- continue;
280
- }
281
-
282
- listener.pendingTimeout = new Date(
283
- new Date().getTime() / 1000 + 300
284
- ).toISOString();
285
- listener.listenerStatus = 'PENDING';
286
-
287
- // TODO update listeners in parallel, blocking for listeners with block=true
288
- try {
289
- const listenerRevisions =
290
- await transmitter?.transmit(strandUpdates);
291
-
292
- listener.pendingTimeout = '0';
293
- listener.listenerStatus = 'PENDING';
294
-
295
- const lastUpdated = new Date().toISOString();
296
-
297
- for (const revision of listenerRevisions) {
298
- const syncUnit = syncUnits.find(
299
- unit =>
300
- revision.documentId === unit.documentId &&
301
- revision.scope === unit.scope &&
302
- revision.branch === unit.branch
303
- );
304
- if (syncUnit) {
305
- listener.syncUnits.set(syncUnit.syncId, {
306
- lastUpdated,
307
- listenerRev: revision.revision
308
- });
309
- } else {
310
- logger.warn(
311
- `Received revision for untracked unit for listener ${listener.listener.listenerId}`,
312
- revision
313
- );
314
- }
315
- }
316
- const revisionError = listenerRevisions.find(
317
- l => l.status !== 'SUCCESS'
318
- );
319
- if (revisionError) {
320
- throw new OperationError(
321
- revisionError.status as ErrorStatus,
322
- undefined,
323
- revisionError.error,
324
- revisionError.error
325
- );
326
- }
327
- listener.listenerStatus = 'SUCCESS';
328
- listenerUpdates.push({
329
- listenerId: listener.listener.listenerId,
330
- listenerRevisions
331
- });
332
- } catch (e) {
333
- // TODO: Handle error based on listener params (blocking, retry, etc)
334
- onError?.(e as Error, driveId, listener);
335
- listener.listenerStatus =
336
- e instanceof OperationError ? e.status : 'ERROR';
337
- }
337
+ }
338
338
  }
339
+ }
340
+ listener.listenerStatus = "SUCCESS";
341
+ } catch (e) {
342
+ // TODO: Handle error based on listener params (blocking, retry, etc)
343
+ onError?.(e as Error, driveId, listener);
344
+ listener.listenerStatus =
345
+ e instanceof OperationError ? e.status : "ERROR";
339
346
  }
340
- return listenerUpdates;
347
+ }
341
348
  }
342
-
343
- private _checkFilter(
344
- filter: ListenerFilter,
345
- syncUnit: SynchronizationUnit
349
+ return listenerUpdates;
350
+ }
351
+
352
+ private _checkFilter(filter: ListenerFilter, syncUnit: SynchronizationUnit) {
353
+ const { branch, documentId, scope, documentType } = syncUnit;
354
+ // TODO: Needs to be optimized
355
+ if (
356
+ (!filter.branch ||
357
+ filter.branch.includes(branch) ||
358
+ filter.branch.includes("*")) &&
359
+ (!filter.documentId ||
360
+ filter.documentId.includes(documentId) ||
361
+ filter.documentId.includes("*")) &&
362
+ (!filter.scope ||
363
+ filter.scope.includes(scope) ||
364
+ filter.scope.includes("*")) &&
365
+ (!filter.documentType ||
366
+ filter.documentType.includes(documentType) ||
367
+ filter.documentType.includes("*"))
346
368
  ) {
347
- const { branch, documentId, scope, documentType } = syncUnit;
348
- // TODO: Needs to be optimized
349
- if (
350
- (!filter.branch ||
351
- filter.branch.includes(branch) ||
352
- filter.branch.includes('*')) &&
353
- (!filter.documentId ||
354
- filter.documentId.includes(documentId) ||
355
- filter.documentId.includes('*')) &&
356
- (!filter.scope ||
357
- filter.scope.includes(scope) ||
358
- filter.scope.includes('*')) &&
359
- (!filter.documentType ||
360
- filter.documentType.includes(documentType) ||
361
- filter.documentType.includes('*'))
362
- ) {
363
- return true;
364
- }
365
- return false;
366
- }
367
-
368
- getListenerSyncUnits(driveId: string, listenerId: string) {
369
- const listener = this.listenerState.get(driveId)?.get(listenerId);
370
- if (!listener) {
371
- return [];
372
- }
373
- const filter = listener.listener.filter;
374
- return this.drive.getSynchronizationUnits(
375
- driveId,
376
- filter.documentId ?? ['*'],
377
- filter.scope ?? ['*'],
378
- filter.branch ?? ['*'],
379
- filter.documentType ?? ['*']
380
- );
369
+ return true;
381
370
  }
382
-
383
- getListenerSyncUnitIds(driveId: string, listenerId: string) {
384
- const listener = this.listenerState.get(driveId)?.get(listenerId);
385
- if (!listener) {
386
- return [];
387
- }
388
- const filter = listener.listener.filter;
389
- return this.drive.getSynchronizationUnitsIds(
390
- driveId,
391
- filter.documentId ?? ['*'],
392
- filter.scope ?? ['*'],
393
- filter.branch ?? ['*'],
394
- filter.documentType ?? ['*']
395
- );
371
+ return false;
372
+ }
373
+
374
+ getListenerSyncUnits(
375
+ driveId: string,
376
+ listenerId: string,
377
+ loadedDrive?: DocumentDriveDocument,
378
+ ) {
379
+ const listener = this.listenerState.get(driveId)?.get(listenerId);
380
+ if (!listener) {
381
+ return [];
396
382
  }
383
+ const filter = listener.listener.filter;
384
+ return this.drive.getSynchronizationUnits(
385
+ driveId,
386
+ filter.documentId ?? ["*"],
387
+ filter.scope ?? ["*"],
388
+ filter.branch ?? ["*"],
389
+ filter.documentType ?? ["*"],
390
+ loadedDrive,
391
+ );
392
+ }
397
393
 
398
- async initDrive(drive: DocumentDriveDocument) {
399
- const {
400
- state: {
401
- local: { listeners }
402
- }
403
- } = drive;
404
-
405
- for (const listener of listeners) {
406
- await this.addListener({
407
- block: listener.block,
408
- driveId: drive.state.global.id,
409
- filter: {
410
- branch: listener.filter.branch ?? [],
411
- documentId: listener.filter.documentId ?? [],
412
- documentType: listener.filter.documentType,
413
- scope: listener.filter.scope ?? []
414
- },
415
- listenerId: listener.listenerId,
416
- system: listener.system,
417
- callInfo: listener.callInfo ?? undefined,
418
- label: listener.label ?? ''
419
- });
420
- }
394
+ getListenerSyncUnitIds(driveId: string, listenerId: string) {
395
+ const listener = this.listenerState.get(driveId)?.get(listenerId);
396
+ if (!listener) {
397
+ return [];
421
398
  }
422
-
423
- async removeDrive(driveId: string): Promise<void> {
424
- this.listenerState.delete(driveId);
425
- const transmitters = this.transmitters[driveId];
426
- if (transmitters) {
427
- await Promise.all(Object.values(transmitters).map(t => t.disconnect?.()));
428
- }
399
+ const filter = listener.listener.filter;
400
+ return this.drive.getSynchronizationUnitsIds(
401
+ driveId,
402
+ filter.documentId ?? ["*"],
403
+ filter.scope ?? ["*"],
404
+ filter.branch ?? ["*"],
405
+ filter.documentType ?? ["*"],
406
+ );
407
+ }
408
+
409
+ async initDrive(drive: DocumentDriveDocument) {
410
+ const {
411
+ state: {
412
+ local: { listeners },
413
+ },
414
+ } = drive;
415
+
416
+ for (const listener of listeners) {
417
+ await this.addListener({
418
+ block: listener.block,
419
+ driveId: drive.state.global.id,
420
+ filter: {
421
+ branch: listener.filter.branch ?? [],
422
+ documentId: listener.filter.documentId ?? [],
423
+ documentType: listener.filter.documentType,
424
+ scope: listener.filter.scope ?? [],
425
+ },
426
+ listenerId: listener.listenerId,
427
+ system: listener.system,
428
+ callInfo: listener.callInfo ?? undefined,
429
+ label: listener.label ?? "",
430
+ });
429
431
  }
430
-
431
- getListener(driveId: string, listenerId: string): Promise<ListenerState> {
432
- const drive = this.listenerState.get(driveId);
433
- if (!drive) throw new Error('Drive not found');
434
- const listener = drive.get(listenerId);
435
- if (!listener) throw new Error('Listener not found');
436
- return Promise.resolve(listener);
432
+ }
433
+
434
+ async removeDrive(driveId: string): Promise<void> {
435
+ this.listenerState.delete(driveId);
436
+ const transmitters = this.transmitters[driveId];
437
+ if (transmitters) {
438
+ await Promise.all(
439
+ Object.values(transmitters).map((t) => t.disconnect?.()),
440
+ );
437
441
  }
442
+ }
443
+
444
+ getListener(driveId: string, listenerId: string): Promise<ListenerState> {
445
+ const drive = this.listenerState.get(driveId);
446
+ if (!drive) throw new Error("Drive not found");
447
+ const listener = drive.get(listenerId);
448
+ if (!listener) throw new Error("Listener not found");
449
+ return Promise.resolve(listener);
450
+ }
451
+
452
+ async getStrands(
453
+ driveId: string,
454
+ listenerId: string,
455
+ options?: GetStrandsOptions,
456
+ ): Promise<StrandUpdate[]> {
457
+ // fetch listenerState from listenerManager
458
+ const listener = await this.getListener(driveId, listenerId);
459
+
460
+ // fetch operations from drive and prepare strands
461
+ const strands: StrandUpdate[] = [];
462
+
463
+ const drive = await this.drive.getDrive(driveId);
464
+ const syncUnits = await this.getListenerSyncUnits(
465
+ driveId,
466
+ listenerId,
467
+ drive,
468
+ );
438
469
 
439
- async getStrands(
440
- driveId: string,
441
- listenerId: string,
442
- since?: string
443
- ): Promise<StrandUpdate[]> {
444
- // fetch listenerState from listenerManager
445
- const listener = await this.getListener(driveId, listenerId);
446
-
447
- // fetch operations from drive and prepare strands
448
- const strands: StrandUpdate[] = [];
449
-
450
- const syncUnits = await this.getListenerSyncUnits(driveId, listenerId);
451
-
452
- for (const syncUnit of syncUnits) {
453
- if (syncUnit.revision < 0) {
454
- continue;
455
- }
456
- const entry = listener.syncUnits.get(syncUnit.syncId);
457
- if (entry && entry.listenerRev >= syncUnit.revision) {
458
- continue;
459
- }
460
-
461
- const { documentId, driveId, scope, branch } = syncUnit;
462
- try {
463
- const operations = await this.drive.getOperationData( // DEAL WITH INVALID SYNC ID ERROR
464
- driveId,
465
- syncUnit.syncId,
466
- {
467
- since,
468
- fromRevision: entry?.listenerRev
469
- }
470
- );
470
+ const limit = options?.limit; // maximum number of operations to send across all sync units
471
+ let operationsCount = 0; // total amount of operations that have been retrieved
472
+
473
+ const tasks = syncUnits.map((syncUnit) => async () => {
474
+ if (limit && operationsCount >= limit) {
475
+ // break;
476
+ return;
477
+ }
478
+ if (syncUnit.revision < 0) {
479
+ return;
480
+ }
481
+ const entry = listener.syncUnits.get(syncUnit.syncId);
482
+ if (entry && entry.listenerRev >= syncUnit.revision) {
483
+ return;
484
+ }
485
+
486
+ const { documentId, driveId, scope, branch } = syncUnit;
487
+ try {
488
+ const operations = await this.drive.getOperationData(
489
+ // DEAL WITH INVALID SYNC ID ERROR
490
+ driveId,
491
+ syncUnit.syncId,
492
+ {
493
+ since: options?.since,
494
+ fromRevision: options?.fromRevision ?? entry?.listenerRev,
495
+ limit: limit ? limit - operationsCount : undefined,
496
+ },
497
+ drive,
498
+ );
471
499
 
472
- if (!operations.length) {
473
- continue;
474
- }
475
-
476
- strands.push({
477
- driveId,
478
- documentId,
479
- scope: scope as OperationScope,
480
- branch,
481
- operations
482
- });
483
- } catch (error) {
484
- logger.error(error);
485
- continue;
486
- }
500
+ if (!operations.length) {
501
+ return;
487
502
  }
488
503
 
489
- return strands;
504
+ operationsCount += operations.length;
505
+
506
+ strands.push({
507
+ driveId,
508
+ documentId,
509
+ scope: scope as OperationScope,
510
+ branch,
511
+ operations,
512
+ });
513
+ } catch (error) {
514
+ logger.error(error);
515
+ return;
516
+ }
517
+ });
518
+
519
+ if (this.options.sequentialUpdates) {
520
+ for (const task of tasks) {
521
+ await task();
522
+ }
523
+ } else {
524
+ await Promise.all(tasks.map((task) => task()));
490
525
  }
526
+
527
+ return strands;
528
+ }
491
529
  }