document-drive 1.0.0-websockets → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +1 -0
  2. package/package.json +74 -88
  3. package/src/cache/index.ts +2 -2
  4. package/src/cache/memory.ts +22 -13
  5. package/src/cache/redis.ts +43 -16
  6. package/src/cache/types.ts +4 -4
  7. package/src/index.ts +6 -3
  8. package/src/queue/base.ts +276 -214
  9. package/src/queue/index.ts +2 -2
  10. package/src/queue/redis.ts +138 -127
  11. package/src/queue/types.ts +44 -38
  12. package/src/read-mode/errors.ts +19 -0
  13. package/src/read-mode/index.ts +125 -0
  14. package/src/read-mode/service.ts +207 -0
  15. package/src/read-mode/types.ts +108 -0
  16. package/src/server/error.ts +61 -26
  17. package/src/server/index.ts +2160 -1785
  18. package/src/server/listener/index.ts +2 -2
  19. package/src/server/listener/manager.ts +475 -437
  20. package/src/server/listener/transmitter/index.ts +4 -5
  21. package/src/server/listener/transmitter/internal.ts +77 -79
  22. package/src/server/listener/transmitter/pull-responder.ts +363 -329
  23. package/src/server/listener/transmitter/switchboard-push.ts +72 -55
  24. package/src/server/listener/transmitter/types.ts +19 -25
  25. package/src/server/types.ts +536 -349
  26. package/src/server/utils.ts +26 -27
  27. package/src/storage/base.ts +81 -0
  28. package/src/storage/browser.ts +233 -216
  29. package/src/storage/filesystem.ts +257 -256
  30. package/src/storage/index.ts +2 -1
  31. package/src/storage/memory.ts +206 -214
  32. package/src/storage/prisma.ts +575 -568
  33. package/src/storage/sequelize.ts +460 -471
  34. package/src/storage/types.ts +83 -67
  35. package/src/utils/default-drives-manager.ts +341 -0
  36. package/src/utils/document-helpers.ts +19 -18
  37. package/src/utils/graphql.ts +288 -34
  38. package/src/utils/index.ts +61 -59
  39. package/src/utils/logger.ts +39 -37
  40. package/src/utils/migrations.ts +58 -0
  41. package/src/utils/run-asap.ts +156 -0
  42. package/CHANGELOG.md +0 -818
  43. package/src/server/listener/transmitter/subscription.ts +0 -364
@@ -1,1941 +1,2316 @@
1
1
  import {
2
- actions,
3
- AddListenerInput,
4
- DocumentDriveAction,
5
- DocumentDriveDocument,
6
- DocumentDriveState,
7
- FileNode,
8
- isFileNode,
9
- ListenerFilter,
10
- RemoveListenerInput,
11
- Trigger,
12
- utils
13
- } from 'document-model-libs/document-drive';
2
+ actions,
3
+ AddListenerInput,
4
+ DocumentDriveAction,
5
+ DocumentDriveDocument,
6
+ DocumentDriveState,
7
+ FileNode,
8
+ isFileNode,
9
+ ListenerFilter,
10
+ RemoveListenerInput,
11
+ Trigger,
12
+ utils,
13
+ } from "document-model-libs/document-drive";
14
14
  import {
15
- Action,
16
- BaseAction,
17
- utils as baseUtils,
18
- Document,
19
- DocumentHeader,
20
- DocumentModel,
21
- utils as DocumentUtils,
22
- Operation,
23
- OperationScope
24
- } from 'document-model/document';
25
- import { createNanoEvents, Unsubscribe } from 'nanoevents';
26
- import { ICache } from '../cache';
27
- import InMemoryCache from '../cache/memory';
28
- import { BaseQueueManager } from '../queue/base';
15
+ Action,
16
+ BaseAction,
17
+ utils as baseUtils,
18
+ Document,
19
+ DocumentHeader,
20
+ DocumentModel,
21
+ utils as DocumentUtils,
22
+ Operation,
23
+ OperationScope,
24
+ } from "document-model/document";
25
+ import { ClientError } from "graphql-request";
26
+ import { createNanoEvents, Unsubscribe } from "nanoevents";
27
+ import { ICache } from "../cache";
28
+ import InMemoryCache from "../cache/memory";
29
+ import { BaseQueueManager } from "../queue/base";
29
30
  import {
30
- ActionJob,
31
- IQueueManager,
32
- isActionJob,
33
- isOperationJob,
34
- Job,
35
- OperationJob
36
- } from '../queue/types';
37
- import { MemoryStorage } from '../storage/memory';
31
+ ActionJob,
32
+ IQueueManager,
33
+ isActionJob,
34
+ isOperationJob,
35
+ Job,
36
+ OperationJob,
37
+ } from "../queue/types";
38
+ import { ReadModeServer } from "../read-mode";
39
+ import { MemoryStorage } from "../storage/memory";
38
40
  import type {
39
- DocumentDriveStorage,
40
- DocumentStorage,
41
- IDriveStorage
42
- } from '../storage/types';
43
- import { generateUUID, isBefore, isDocumentDrive } from '../utils';
41
+ DocumentDriveStorage,
42
+ DocumentStorage,
43
+ IDriveStorage,
44
+ } from "../storage/types";
44
45
  import {
45
- attachBranch,
46
- garbageCollect,
47
- groupOperationsByScope,
48
- merge,
49
- precedes,
50
- removeExistingOperations,
51
- reshuffleByTimestamp,
52
- sortOperations
53
- } from '../utils/document-helpers';
54
- import { requestPublicDrive } from '../utils/graphql';
55
- import { logger } from '../utils/logger';
56
- import { ConflictOperationError, OperationError } from './error';
57
- import { ListenerManager } from './listener/manager';
46
+ generateUUID,
47
+ isBefore,
48
+ isDocumentDrive,
49
+ RunAsap,
50
+ runAsapAsync,
51
+ } from "../utils";
52
+ import { DefaultDrivesManager } from "../utils/default-drives-manager";
58
53
  import {
59
- CancelPullLoop,
60
- InternalTransmitter,
61
- IReceiver,
62
- ITransmitter,
63
- PullResponderTransmitter,
64
- SubscriptionTransmitter
65
- } from './listener/transmitter';
54
+ attachBranch,
55
+ garbageCollect,
56
+ groupOperationsByScope,
57
+ merge,
58
+ precedes,
59
+ removeExistingOperations,
60
+ reshuffleByTimestamp,
61
+ sortOperations,
62
+ } from "../utils/document-helpers";
63
+ import { requestPublicDrive } from "../utils/graphql";
64
+ import { logger } from "../utils/logger";
66
65
  import {
67
- BaseDocumentDriveServer,
68
- DriveEvents,
69
- GetDocumentOptions,
70
- IOperationResult,
71
- ListenerState,
72
- RemoteDriveOptions,
73
- StrandUpdate,
74
- SynchronizationUnitQuery,
75
- SyncStatus,
76
- type CreateDocumentInput,
77
- type DriveInput,
78
- type OperationUpdate,
79
- type SignalResult,
80
- type SynchronizationUnit
81
- } from './types';
82
- import { filterOperationsByRevision } from './utils';
83
-
84
- export * from './listener';
85
- export type * from './types';
66
+ ConflictOperationError,
67
+ DriveAlreadyExistsError,
68
+ OperationError,
69
+ SynchronizationUnitNotFoundError,
70
+ } from "./error";
71
+ import { ListenerManager } from "./listener/manager";
72
+ import {
73
+ CancelPullLoop,
74
+ InternalTransmitter,
75
+ IReceiver,
76
+ ITransmitter,
77
+ PullResponderTransmitter,
78
+ StrandUpdateSource,
79
+ } from "./listener/transmitter";
80
+ import {
81
+ AbstractDocumentDriveServer,
82
+ AddOperationOptions,
83
+ DefaultListenerManagerOptions,
84
+ DocumentDriveServerOptions,
85
+ DriveEvents,
86
+ GetDocumentOptions,
87
+ GetStrandsOptions,
88
+ IBaseDocumentDriveServer,
89
+ IOperationResult,
90
+ ListenerState,
91
+ RemoteDriveAccessLevel,
92
+ RemoteDriveOptions,
93
+ StrandUpdate,
94
+ SynchronizationUnitQuery,
95
+ SyncStatus,
96
+ SyncUnitStatusObject,
97
+ type CreateDocumentInput,
98
+ type DriveInput,
99
+ type OperationUpdate,
100
+ type SignalResult,
101
+ type SynchronizationUnit,
102
+ } from "./types";
103
+ import { filterOperationsByRevision } from "./utils";
104
+
105
+ export * from "./listener";
106
+ export type * from "./types";
107
+
108
+ export * from "../read-mode";
86
109
 
87
110
  export const PULL_DRIVE_INTERVAL = 5000;
88
111
 
89
- export class DocumentDriveServer extends BaseDocumentDriveServer {
90
- private emitter = createNanoEvents<DriveEvents>();
91
- private cache: ICache;
92
- private documentModels: DocumentModel[];
93
- private storage: IDriveStorage;
94
- private listenerStateManager: ListenerManager;
95
- private triggerMap = new Map<
96
- DocumentDriveState['id'],
97
- Map<Trigger['id'], CancelPullLoop>
98
- >();
99
- private syncStatus = new Map<string, SyncStatus>();
100
-
101
- private queueManager: IQueueManager;
102
-
103
- constructor(
104
- documentModels: DocumentModel[],
105
- storage: IDriveStorage = new MemoryStorage(),
106
- cache: ICache = new InMemoryCache(),
107
- queueManager: IQueueManager = new BaseQueueManager()
108
- ) {
109
- super();
110
- this.listenerStateManager = new ListenerManager(this);
111
- this.documentModels = documentModels;
112
- this.storage = storage;
113
- this.cache = cache;
114
- this.queueManager = queueManager;
115
-
116
- this.storage.setStorageDelegate?.({
117
- getCachedOperations: async (drive, id) => {
118
- try {
119
- const document = await this.cache.getDocument(drive, id);
120
- return document?.operations;
121
- } catch (error) {
122
- logger.error(error);
123
- return undefined;
124
- }
125
- }
126
- });
127
- }
112
+ export class BaseDocumentDriveServer
113
+ extends AbstractDocumentDriveServer
114
+ implements IBaseDocumentDriveServer
115
+ {
116
+ private emitter = createNanoEvents<DriveEvents>();
117
+ private cache: ICache;
118
+ private documentModels: DocumentModel[];
119
+ private storage: IDriveStorage;
120
+ private listenerStateManager: ListenerManager;
121
+ private triggerMap = new Map<
122
+ DocumentDriveState["id"],
123
+ Map<Trigger["id"], CancelPullLoop>
124
+ >();
125
+ private syncStatus = new Map<string, SyncUnitStatusObject>();
126
+
127
+ private queueManager: IQueueManager;
128
+ private initializePromise: Promise<Error[] | null>;
129
+
130
+ private defaultDrivesManager: DefaultDrivesManager;
131
+
132
+ protected options: Required<DocumentDriveServerOptions>;
133
+
134
+ constructor(
135
+ documentModels: DocumentModel[],
136
+ storage: IDriveStorage = new MemoryStorage(),
137
+ cache: ICache = new InMemoryCache(),
138
+ queueManager: IQueueManager = new BaseQueueManager(),
139
+ options?: DocumentDriveServerOptions,
140
+ ) {
141
+ super();
142
+ this.options = {
143
+ ...options,
144
+ defaultDrives: {
145
+ ...options?.defaultDrives,
146
+ },
147
+ listenerManager: {
148
+ ...DefaultListenerManagerOptions,
149
+ ...options?.listenerManager,
150
+ },
151
+ taskQueueMethod:
152
+ options?.taskQueueMethod === undefined
153
+ ? RunAsap.runAsap
154
+ : options.taskQueueMethod,
155
+ };
128
156
 
129
- private updateSyncStatus(
130
- driveId: string,
131
- status: SyncStatus | null,
132
- error?: Error
133
- ) {
134
- if (status === null) {
135
- this.syncStatus.delete(driveId);
136
- } else if (this.syncStatus.get(driveId) !== status) {
137
- this.syncStatus.set(driveId, status);
138
- this.emit('syncStatus', driveId, status, error);
157
+ this.listenerStateManager = new ListenerManager(
158
+ this,
159
+ undefined,
160
+ options?.listenerManager,
161
+ );
162
+ this.documentModels = documentModels;
163
+ this.storage = storage;
164
+ this.cache = cache;
165
+ this.queueManager = queueManager;
166
+ this.defaultDrivesManager = new DefaultDrivesManager(
167
+ this,
168
+ this.defaultDrivesManagerDelegate,
169
+ options,
170
+ );
171
+
172
+ this.storage.setStorageDelegate?.({
173
+ getCachedOperations: async (drive, id) => {
174
+ try {
175
+ const document = await this.cache.getDocument(drive, id);
176
+ return document?.operations;
177
+ } catch (error) {
178
+ logger.error(error);
179
+ return undefined;
139
180
  }
140
- }
181
+ },
182
+ });
183
+
184
+ this.initializePromise = this._initialize();
185
+ }
186
+
187
+ setDocumentModels(models: DocumentModel[]): void {
188
+ this.documentModels = [...models];
189
+ this.emit("documentModels", [...models]);
190
+ }
191
+
192
+ initializeDefaultRemoteDrives() {
193
+ return this.defaultDrivesManager.initializeDefaultRemoteDrives();
194
+ }
195
+
196
+ getDefaultRemoteDrives() {
197
+ return this.defaultDrivesManager.getDefaultRemoteDrives();
198
+ }
199
+
200
+ setDefaultDriveAccessLevel(url: string, level: RemoteDriveAccessLevel) {
201
+ return this.defaultDrivesManager.setDefaultDriveAccessLevel(url, level);
202
+ }
203
+
204
+ setAllDefaultDrivesAccessLevel(level: RemoteDriveAccessLevel) {
205
+ return this.defaultDrivesManager.setAllDefaultDrivesAccessLevel(level);
206
+ }
207
+
208
+ private getOperationSource(source: StrandUpdateSource) {
209
+ return source.type === "local" ? "push" : "pull";
210
+ }
211
+
212
+ private getCombinedSyncUnitStatus(
213
+ syncUnitStatus: SyncUnitStatusObject,
214
+ ): SyncStatus {
215
+ if (!syncUnitStatus.pull && !syncUnitStatus.push) return "INITIAL_SYNC";
216
+ if (syncUnitStatus.pull === "INITIAL_SYNC") return "INITIAL_SYNC";
217
+ if (syncUnitStatus.push === "INITIAL_SYNC")
218
+ return syncUnitStatus.pull || "INITIAL_SYNC";
219
+
220
+ const order: Array<SyncStatus> = [
221
+ "ERROR",
222
+ "MISSING",
223
+ "CONFLICT",
224
+ "SYNCING",
225
+ "SUCCESS",
226
+ ];
227
+ const sortedStatus = Object.values(syncUnitStatus).sort(
228
+ (a, b) => order.indexOf(a) - order.indexOf(b),
229
+ );
230
+
231
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
232
+ return sortedStatus[0]!;
233
+ }
234
+
235
+ private initSyncStatus(
236
+ syncUnitId: string,
237
+ status: Partial<SyncUnitStatusObject>,
238
+ ) {
239
+ const defaultSyncUnitStatus: SyncUnitStatusObject = Object.entries(
240
+ status,
241
+ ).reduce((acc, [key, _status]) => {
242
+ return {
243
+ ...acc,
244
+ [key]: _status !== "SYNCING" ? _status : "INITIAL_SYNC",
245
+ };
246
+ }, {});
247
+
248
+ this.syncStatus.set(syncUnitId, defaultSyncUnitStatus);
249
+ this.emit(
250
+ "syncStatus",
251
+ syncUnitId,
252
+ this.getCombinedSyncUnitStatus(defaultSyncUnitStatus),
253
+ undefined,
254
+ defaultSyncUnitStatus,
255
+ );
256
+ }
257
+
258
+ private async initializeDriveSyncStatus(
259
+ driveId: string,
260
+ drive: DocumentDriveDocument,
261
+ ) {
262
+ const syncUnits = await this.getSynchronizationUnitsIds(driveId);
263
+ const syncStatus: SyncUnitStatusObject = {
264
+ pull: drive.state.local.triggers.length > 0 ? "INITIAL_SYNC" : undefined,
265
+ push: drive.state.local.listeners.length > 0 ? "SUCCESS" : undefined,
266
+ };
141
267
 
142
- private async saveStrand(strand: StrandUpdate) {
143
- const operations: Operation[] = strand.operations.map(op => ({
144
- ...op,
145
- scope: strand.scope,
146
- branch: strand.branch
147
- }));
268
+ if (!syncStatus.pull && !syncStatus.push) return;
148
269
 
149
- const result = await (!strand.documentId
150
- ? this.queueDriveOperations(
151
- strand.driveId,
152
- operations as Operation<DocumentDriveAction | BaseAction>[],
153
- false
154
- )
155
- : this.queueOperations(
156
- strand.driveId,
157
- strand.documentId,
158
- operations,
159
- false
160
- ));
161
-
162
- if (result.status === 'ERROR') {
163
- this.updateSyncStatus(strand.driveId, result.status, result.error);
164
- } else {
165
- this.emit('strandUpdate', strand);
166
- }
167
- return result;
168
- }
270
+ const syncUnitsIds = [driveId, ...syncUnits.map((s) => s.syncId)];
169
271
 
170
- private handleListenerError(
171
- error: Error,
172
- driveId: string,
173
- listener: ListenerState
174
- ) {
175
- logger.error(
176
- `Listener ${listener.listener.label ?? listener.listener.listenerId} error:`,
177
- error
178
- );
179
- this.updateSyncStatus(
180
- driveId,
181
- error instanceof OperationError ? error.status : 'ERROR',
182
- error
183
- );
272
+ for (const syncUnitId of syncUnitsIds) {
273
+ this.initSyncStatus(syncUnitId, syncStatus);
184
274
  }
185
-
186
- private shouldSyncRemoteDrive(drive: DocumentDriveDocument) {
187
- return (
188
- drive.state.local.availableOffline &&
189
- drive.state.local.triggers.length > 0
190
- );
275
+ }
276
+
277
+ private updateSyncUnitStatus(
278
+ syncUnitId: string,
279
+ status: Partial<SyncUnitStatusObject> | null,
280
+ error?: Error,
281
+ ) {
282
+ if (status === null) {
283
+ this.syncStatus.delete(syncUnitId);
284
+ return;
191
285
  }
192
286
 
193
- private async startSyncRemoteDrive(driveId: string) {
194
- const drive = await this.getDrive(driveId);
195
- let driveTriggers = this.triggerMap.get(driveId);
287
+ const syncUnitStatus = this.syncStatus.get(syncUnitId);
196
288
 
197
- const syncUnits = await this.getSynchronizationUnitsIds(driveId);
198
-
199
- for (const trigger of drive.state.local.triggers) {
200
- if (driveTriggers?.get(trigger.id)) {
201
- continue;
202
- }
289
+ if (!syncUnitStatus) {
290
+ this.initSyncStatus(syncUnitId, status);
291
+ return;
292
+ }
203
293
 
204
- if (!driveTriggers) {
205
- driveTriggers = new Map();
206
- }
294
+ const shouldUpdateStatus = Object.entries(status).some(
295
+ ([key, _status]) =>
296
+ syncUnitStatus[key as keyof SyncUnitStatusObject] !== _status,
297
+ );
207
298
 
208
- this.updateSyncStatus(driveId, 'SYNCING');
299
+ if (shouldUpdateStatus) {
300
+ const newstatus = Object.entries(status).reduce((acc, [key, _status]) => {
301
+ return {
302
+ ...acc,
303
+ // do not replace initial_syncing if it has not finished yet
304
+ [key]:
305
+ acc[key as keyof SyncUnitStatusObject] === "INITIAL_SYNC" &&
306
+ _status === "SYNCING"
307
+ ? "INITIAL_SYNC"
308
+ : _status,
309
+ };
310
+ }, syncUnitStatus);
209
311
 
210
- for (const syncUnit of syncUnits) {
211
- this.updateSyncStatus(syncUnit.syncId, 'SYNCING');
212
- }
312
+ const previousCombinedStatus =
313
+ this.getCombinedSyncUnitStatus(syncUnitStatus);
314
+ const newCombinedStatus = this.getCombinedSyncUnitStatus(newstatus);
213
315
 
214
- let cancelTrigger: (() => void) | undefined = undefined;
215
- if (PullResponderTransmitter.isPullResponderTrigger(trigger)) {
216
- cancelTrigger = PullResponderTransmitter.setupPull(
217
- driveId,
218
- trigger,
219
- this.saveStrand.bind(this),
220
- error => {
221
- this.updateSyncStatus(
222
- driveId,
223
- error instanceof OperationError
224
- ? error.status
225
- : 'ERROR',
226
- error
227
- );
228
- },
229
- revisions => {
230
- const errorRevision = revisions.filter(
231
- r => r.status !== 'SUCCESS'
232
- );
233
- if (errorRevision.length < 1) {
234
- this.updateSyncStatus(driveId, 'SUCCESS');
235
- }
236
-
237
- for (const syncUnit of syncUnits) {
238
- const fileErrorRevision = errorRevision.find(
239
- r => r.documentId === syncUnit.documentId
240
- );
241
-
242
- if (fileErrorRevision) {
243
- this.updateSyncStatus(
244
- syncUnit.syncId,
245
- fileErrorRevision.status,
246
- fileErrorRevision.error
247
- );
248
- } else {
249
- this.updateSyncStatus(
250
- syncUnit.syncId,
251
- 'SUCCESS'
252
- );
253
- }
254
- }
255
- }
256
- );
257
- } else if (SubscriptionTransmitter.isTrigger(trigger)) {
258
- cancelTrigger = SubscriptionTransmitter.setup(
259
- driveId,
260
- trigger,
261
- this.saveStrand.bind(this),
262
- error => {
263
- this.updateSyncStatus(
264
- driveId,
265
- error instanceof OperationError
266
- ? error.status
267
- : 'ERROR',
268
- error
269
- );
270
- },
271
- revisions => {
272
- const errorRevision = revisions.find(
273
- r => r.status !== 'SUCCESS'
274
- );
275
- if (!errorRevision) {
276
- this.updateSyncStatus(driveId, 'SUCCESS');
277
- }
278
- }
279
- );
280
- }
316
+ this.syncStatus.set(syncUnitId, newstatus);
281
317
 
282
- if (cancelTrigger) {
283
- driveTriggers.set(trigger.id, cancelTrigger);
284
- this.triggerMap.set(driveId, driveTriggers);
285
- }
286
- }
318
+ if (previousCombinedStatus !== newCombinedStatus) {
319
+ this.emit(
320
+ "syncStatus",
321
+ syncUnitId,
322
+ this.getCombinedSyncUnitStatus(newstatus),
323
+ error,
324
+ newstatus,
325
+ );
326
+ }
287
327
  }
328
+ }
329
+
330
+ private async saveStrand(strand: StrandUpdate, source: StrandUpdateSource) {
331
+ const operations: Operation[] = strand.operations.map((op) => ({
332
+ ...op,
333
+ scope: strand.scope,
334
+ branch: strand.branch,
335
+ }));
336
+
337
+ const result = await (!strand.documentId
338
+ ? this.queueDriveOperations(
339
+ strand.driveId,
340
+ operations as Operation<DocumentDriveAction | BaseAction>[],
341
+ { source },
342
+ )
343
+ : this.queueOperations(strand.driveId, strand.documentId, operations, {
344
+ source,
345
+ }));
288
346
 
289
- private async stopSyncRemoteDrive(driveId: string) {
290
- const syncUnits = await this.getSynchronizationUnitsIds(driveId);
291
- const fileNodes = syncUnits
292
- .filter(syncUnit => syncUnit.documentId !== '')
293
- .map(syncUnit => syncUnit.documentId);
347
+ if (result.status === "ERROR") {
348
+ const syncUnits =
349
+ strand.documentId !== ""
350
+ ? (
351
+ await this.getSynchronizationUnitsIds(
352
+ strand.driveId,
353
+ [strand.documentId],
354
+ [strand.scope],
355
+ [strand.branch],
356
+ )
357
+ ).map((s) => s.syncId)
358
+ : [strand.driveId];
294
359
 
295
- const triggers = this.triggerMap.get(driveId);
296
- triggers?.forEach(cancel => cancel());
297
- this.updateSyncStatus(driveId, null);
360
+ const operationSource = this.getOperationSource(source);
298
361
 
299
- for (const fileNode of fileNodes) {
300
- this.updateSyncStatus(fileNode, null);
301
- }
302
- return this.triggerMap.delete(driveId);
362
+ for (const syncUnit of syncUnits) {
363
+ this.updateSyncUnitStatus(
364
+ syncUnit,
365
+ { [operationSource]: result.status },
366
+ result.error,
367
+ );
368
+ }
303
369
  }
370
+ this.emit("strandUpdate", strand);
371
+ return result;
372
+ }
373
+
374
+ private handleListenerError(
375
+ error: Error,
376
+ driveId: string,
377
+ listener: ListenerState,
378
+ ) {
379
+ logger.error(
380
+ `Listener ${listener.listener.label ?? listener.listener.listenerId} error:`,
381
+ error,
382
+ );
383
+
384
+ const status = error instanceof OperationError ? error.status : "ERROR";
385
+
386
+ this.updateSyncUnitStatus(driveId, { push: status }, error);
387
+ }
388
+
389
+ private shouldSyncRemoteDrive(drive: DocumentDriveDocument) {
390
+ return (
391
+ drive.state.local.availableOffline &&
392
+ drive.state.local.triggers.length > 0
393
+ );
394
+ }
395
+
396
+ private async startSyncRemoteDrive(driveId: string) {
397
+ const drive = await this.getDrive(driveId);
398
+ let driveTriggers = this.triggerMap.get(driveId);
399
+
400
+ const syncUnits = await this.getSynchronizationUnitsIds(
401
+ driveId,
402
+ undefined,
403
+ undefined,
404
+ undefined,
405
+ undefined,
406
+ drive,
407
+ );
408
+
409
+ for (const trigger of drive.state.local.triggers) {
410
+ if (driveTriggers?.get(trigger.id)) {
411
+ continue;
412
+ }
413
+
414
+ if (!driveTriggers) {
415
+ driveTriggers = new Map();
416
+ }
417
+
418
+ this.updateSyncUnitStatus(driveId, { pull: "SYNCING" });
419
+
420
+ for (const syncUnit of syncUnits) {
421
+ this.updateSyncUnitStatus(syncUnit.syncId, { pull: "SYNCING" });
422
+ }
423
+
424
+ if (PullResponderTransmitter.isPullResponderTrigger(trigger)) {
425
+ let firstPull = true;
426
+ const cancelPullLoop = PullResponderTransmitter.setupPull(
427
+ driveId,
428
+ trigger,
429
+ this.saveStrand.bind(this),
430
+ (error) => {
431
+ const statusError =
432
+ error instanceof OperationError ? error.status : "ERROR";
433
+
434
+ this.updateSyncUnitStatus(driveId, { pull: statusError }, error);
435
+
436
+ if (error instanceof ClientError) {
437
+ this.emit(
438
+ "clientStrandsError",
439
+ driveId,
440
+ trigger,
441
+ error.response.status,
442
+ error.message,
443
+ );
444
+ }
445
+ },
446
+ (revisions) => {
447
+ const errorRevision = revisions.filter(
448
+ (r) => r.status !== "SUCCESS",
449
+ );
304
450
 
305
- private queueDelegate = {
306
- checkDocumentExists: (
307
- driveId: string,
308
- documentId: string
309
- ): Promise<boolean> =>
310
- this.storage.checkDocumentExists(driveId, documentId),
311
- processOperationJob: async ({
312
- driveId,
313
- documentId,
314
- operations,
315
- forceSync
316
- }: OperationJob) => {
317
- return documentId
318
- ? this.addOperations(driveId, documentId, operations, forceSync)
319
- : this.addDriveOperations(
320
- driveId,
321
- operations as Operation<
322
- DocumentDriveAction | BaseAction
323
- >[],
324
- forceSync
325
- );
326
- },
327
- processActionJob: async ({
328
- driveId,
329
- documentId,
330
- actions,
331
- forceSync
332
- }: ActionJob) => {
333
- return documentId
334
- ? this.addActions(driveId, documentId, actions, forceSync)
335
- : this.addDriveActions(
336
- driveId,
337
- actions as Operation<DocumentDriveAction | BaseAction>[],
338
- forceSync
339
- );
340
- },
341
- processJob: async (job: Job) => {
342
- if (isOperationJob(job)) {
343
- return this.queueDelegate.processOperationJob(job);
344
- } else if (isActionJob(job)) {
345
- return this.queueDelegate.processActionJob(job);
346
- } else {
347
- throw new Error('Unknown job type', job);
451
+ if (errorRevision.length < 1) {
452
+ this.updateSyncUnitStatus(driveId, {
453
+ pull: "SUCCESS",
454
+ });
348
455
  }
349
- }
350
- };
351
456
 
352
- async initialize() {
353
- const errors: Error[] = [];
354
- const drives = await this.getDrives();
355
- for (const drive of drives) {
356
- await this._initializeDrive(drive).catch(error => {
357
- logger.error(`Error initializing drive ${drive}`, error);
358
- errors.push(error as Error);
359
- });
360
- }
457
+ const documentIdsFromRevision = revisions
458
+ .filter((rev) => rev.documentId !== "")
459
+ .map((rev) => rev.documentId);
361
460
 
362
- await this.queueManager.init(this.queueDelegate, error => {
363
- logger.error(`Error initializing queue manager`, error);
364
- errors.push(error);
365
- });
461
+ this.getSynchronizationUnitsIds(driveId, documentIdsFromRevision)
462
+ .then((revSyncUnits) => {
463
+ for (const syncUnit of revSyncUnits) {
464
+ const fileErrorRevision = errorRevision.find(
465
+ (r) => r.documentId === syncUnit.documentId,
466
+ );
366
467
 
367
- // if network connect comes online then
368
- // triggers the listeners update
369
- if (typeof window !== 'undefined') {
370
- window.addEventListener('online', () => {
371
- this.listenerStateManager
372
- .triggerUpdate(false, this.handleListenerError.bind(this))
373
- .catch(error => {
374
- logger.error(
375
- 'Non handled error updating listeners',
376
- error
377
- );
468
+ if (fileErrorRevision) {
469
+ this.updateSyncUnitStatus(
470
+ syncUnit.syncId,
471
+ { pull: fileErrorRevision.status },
472
+ fileErrorRevision.error,
473
+ );
474
+ } else {
475
+ this.updateSyncUnitStatus(syncUnit.syncId, {
476
+ pull: "SUCCESS",
378
477
  });
379
- });
380
- }
381
-
382
- return errors.length === 0 ? null : errors;
383
- }
384
-
385
- private async _initializeDrive(driveId: string) {
386
- const drive = await this.getDrive(driveId);
387
-
388
- if (this.shouldSyncRemoteDrive(drive)) {
389
- await this.startSyncRemoteDrive(driveId);
390
- }
391
-
392
- await this.listenerStateManager.initDrive(drive);
393
- }
394
-
395
- public async getSynchronizationUnits(
396
- driveId: string,
397
- documentId?: string[],
398
- scope?: string[],
399
- branch?: string[],
400
- documentType?: string[]
401
- ) {
402
- const drive = await this.getDrive(driveId);
403
-
404
- const synchronizationUnitsQuery = await this.getSynchronizationUnitsIds(
405
- driveId,
406
- documentId,
407
- scope,
408
- branch,
409
- documentType,
410
- drive
411
- );
412
- const revisions = await this.storage.getSynchronizationUnitsRevision(
413
- synchronizationUnitsQuery
414
- );
415
-
416
- const synchronizationUnits: SynchronizationUnit[] =
417
- synchronizationUnitsQuery.map(s => ({
418
- ...s,
419
- lastUpdated: drive.created,
420
- revision: -1
421
- }));
422
- for (const revision of revisions) {
423
- const syncUnit = synchronizationUnits.find(
424
- s =>
425
- revision.driveId === s.driveId &&
426
- revision.documentId === s.documentId &&
427
- revision.scope === s.scope &&
428
- revision.branch === s.branch
429
- );
430
- if (syncUnit) {
431
- syncUnit.revision = revision.revision;
432
- syncUnit.lastUpdated = revision.lastUpdated;
433
- }
434
- }
435
- return synchronizationUnits;
436
- }
437
-
438
- public async getSynchronizationUnitsIds(
439
- driveId: string,
440
- documentId?: string[],
441
- scope?: string[],
442
- branch?: string[],
443
- documentType?: string[],
444
- loadedDrive?: DocumentDriveDocument
445
- ): Promise<SynchronizationUnitQuery[]> {
446
- const drive = loadedDrive ?? (await this.getDrive(driveId));
447
- const nodes = drive.state.global.nodes.filter(
448
- node =>
449
- isFileNode(node) &&
450
- (!documentId?.length ||
451
- documentId.includes(node.id) ||
452
- documentId.includes('*')) &&
453
- (!documentType?.length ||
454
- documentType.includes(node.documentType) ||
455
- documentType.includes('*'))
456
- ) as Pick<FileNode, 'id' | 'documentType' | 'synchronizationUnits'>[];
457
-
458
- // checks if document drive synchronization unit should be added
459
- if (
460
- (!documentId ||
461
- documentId.includes('*') ||
462
- documentId.includes('')) &&
463
- (!documentType?.length ||
464
- documentType.includes('powerhouse/document-drive') ||
465
- documentType.includes('*'))
466
- ) {
467
- nodes.unshift({
468
- id: '',
469
- documentType: 'powerhouse/document-drive',
470
- synchronizationUnits: [
471
- {
472
- syncId: '0',
473
- scope: 'global',
474
- branch: 'main'
478
+ }
479
+ }
480
+ })
481
+ .catch(console.error);
482
+
483
+ // if it is the first pull and returns empty
484
+ // then updates corresponding push transmitter
485
+ if (firstPull) {
486
+ firstPull = false;
487
+ const pushListener = drive.state.local.listeners.find(
488
+ (listener) => trigger.data.url === listener.callInfo?.data,
489
+ );
490
+ if (pushListener) {
491
+ this.getSynchronizationUnitsRevision(driveId, syncUnits)
492
+ .then((syncUnitRevisions) => {
493
+ for (const revision of syncUnitRevisions) {
494
+ this.listenerStateManager
495
+ .updateListenerRevision(
496
+ pushListener.listenerId,
497
+ driveId,
498
+ revision.syncId,
499
+ revision.revision,
500
+ )
501
+ .catch(logger.error);
475
502
  }
476
- ]
477
- });
478
- }
479
-
480
- const synchronizationUnitsQuery: Omit<
481
- SynchronizationUnit,
482
- 'revision' | 'lastUpdated'
483
- >[] = [];
484
- for (const node of nodes) {
485
- const nodeUnits =
486
- scope?.length || branch?.length
487
- ? node.synchronizationUnits.filter(
488
- unit =>
489
- (!scope?.length ||
490
- scope.includes(unit.scope) ||
491
- scope.includes('*')) &&
492
- (!branch?.length ||
493
- branch.includes(unit.branch) ||
494
- branch.includes('*'))
495
- )
496
- : node.synchronizationUnits;
497
- if (!nodeUnits.length) {
498
- continue;
503
+ })
504
+ .catch(logger.error);
505
+ }
499
506
  }
500
- synchronizationUnitsQuery.push(
501
- ...nodeUnits.map(n => ({
502
- driveId,
503
- documentId: node.id,
504
- syncId: n.syncId,
505
- documentType: node.documentType,
506
- scope: n.scope,
507
- branch: n.branch
508
- }))
509
- );
510
- }
511
- return synchronizationUnitsQuery;
512
- }
513
-
514
- public async getSynchronizationUnitIdInfo(
515
- driveId: string,
516
- syncId: string
517
- ): Promise<SynchronizationUnitQuery | undefined> {
518
- const drive = await this.getDrive(driveId);
519
- const node = drive.state.global.nodes.find(
520
- node =>
521
- isFileNode(node) &&
522
- node.synchronizationUnits.find(unit => unit.syncId === syncId)
507
+ },
523
508
  );
509
+ driveTriggers.set(trigger.id, cancelPullLoop);
510
+ this.triggerMap.set(driveId, driveTriggers);
511
+ }
512
+ }
513
+ }
524
514
 
525
- if (!node || !isFileNode(node)) {
526
- return undefined;
527
- }
515
+ private async stopSyncRemoteDrive(driveId: string) {
516
+ const syncUnits = await this.getSynchronizationUnitsIds(driveId);
517
+ const filesNodeSyncId = syncUnits
518
+ .filter((syncUnit) => syncUnit.documentId !== "")
519
+ .map((syncUnit) => syncUnit.syncId);
528
520
 
529
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
530
- const syncUnit = node.synchronizationUnits.find(
531
- unit => unit.syncId === syncId
532
- );
533
- if (!syncUnit) {
534
- return undefined;
535
- }
521
+ const triggers = this.triggerMap.get(driveId);
522
+ triggers?.forEach((cancel) => cancel());
523
+ this.updateSyncUnitStatus(driveId, null);
536
524
 
537
- return {
538
- syncId,
539
- scope: syncUnit.scope,
540
- branch: syncUnit.branch,
541
- driveId,
542
- documentId: node.id,
543
- documentType: node.documentType
544
- };
525
+ for (const fileNodeSyncId of filesNodeSyncId) {
526
+ this.updateSyncUnitStatus(fileNodeSyncId, null);
545
527
  }
546
-
547
- public async getSynchronizationUnit(
548
- driveId: string,
549
- syncId: string
550
- ): Promise<SynchronizationUnit | undefined> {
551
- const syncUnit = await this.getSynchronizationUnitIdInfo(
528
+ return this.triggerMap.delete(driveId);
529
+ }
530
+
531
+ private defaultDrivesManagerDelegate = {
532
+ detachDrive: this.detachDrive.bind(this),
533
+ emit: (...args: Parameters<DriveEvents["defaultRemoteDrive"]>) =>
534
+ this.emit("defaultRemoteDrive", ...args),
535
+ };
536
+
537
+ private queueDelegate = {
538
+ checkDocumentExists: (
539
+ driveId: string,
540
+ documentId: string,
541
+ ): Promise<boolean> =>
542
+ this.storage.checkDocumentExists(driveId, documentId),
543
+ processOperationJob: async ({
544
+ driveId,
545
+ documentId,
546
+ operations,
547
+ options,
548
+ }: OperationJob) => {
549
+ return documentId
550
+ ? this.addOperations(driveId, documentId, operations, options)
551
+ : this.addDriveOperations(
552
552
  driveId,
553
- syncId
554
- );
555
-
556
- if (!syncUnit) {
557
- return undefined;
558
- }
559
-
560
- const { scope, branch, documentId, documentType } = syncUnit;
561
-
562
- // TODO: REPLACE WITH GET DOCUMENT OPERATIONS
563
- const document = await this.getDocument(driveId, documentId);
564
- const operations = document.operations[scope as OperationScope] ?? [];
565
- const lastOperation = operations[operations.length - 1];
566
-
567
- return {
568
- syncId,
569
- scope,
570
- branch,
553
+ operations as Operation<DocumentDriveAction | BaseAction>[],
554
+ options,
555
+ );
556
+ },
557
+ processActionJob: async ({
558
+ driveId,
559
+ documentId,
560
+ actions,
561
+ options,
562
+ }: ActionJob) => {
563
+ return documentId
564
+ ? this.addActions(driveId, documentId, actions, options)
565
+ : this.addDriveActions(
571
566
  driveId,
572
- documentId,
573
- documentType,
574
- lastUpdated: lastOperation?.timestamp ?? document.lastModified,
575
- revision: lastOperation?.index ?? 0
576
- };
567
+ actions as Operation<DocumentDriveAction | BaseAction>[],
568
+ options,
569
+ );
570
+ },
571
+ processJob: async (job: Job) => {
572
+ if (isOperationJob(job)) {
573
+ return this.queueDelegate.processOperationJob(job);
574
+ } else if (isActionJob(job)) {
575
+ return this.queueDelegate.processActionJob(job);
576
+ } else {
577
+ throw new Error("Unknown job type", job);
578
+ }
579
+ },
580
+ };
581
+
582
+ initialize() {
583
+ return this.initializePromise;
584
+ }
585
+
586
+ private async _initialize() {
587
+ await this.queueManager.init(this.queueDelegate, (error) => {
588
+ logger.error(`Error initializing queue manager`, error);
589
+ errors.push(error);
590
+ });
591
+
592
+ try {
593
+ await this.defaultDrivesManager.removeOldremoteDrives();
594
+ } catch (error) {
595
+ logger.error(error);
577
596
  }
578
597
 
579
- async getOperationData(
580
- driveId: string,
581
- syncId: string,
582
- filter: {
583
- since?: string | undefined;
584
- fromRevision?: number | undefined;
585
- }
586
- ): Promise<OperationUpdate[]> {
587
- const syncUnit =
588
- syncId === '0'
589
- ? { documentId: '', scope: 'global' }
590
- : await this.getSynchronizationUnitIdInfo(driveId, syncId);
591
-
592
- if (!syncUnit) {
593
- throw new Error(`Invalid Sync Id ${syncId} in drive ${driveId}`);
594
- }
595
-
596
- const document =
597
- syncId === '0'
598
- ? await this.getDrive(driveId)
599
- : await this.getDocument(driveId, syncUnit.documentId); // TODO replace with getDocumentOperations
600
-
601
- const operations =
602
- document.operations[syncUnit.scope as OperationScope] ?? []; // TODO filter by branch also
603
- const filteredOperations = operations.filter(
604
- operation =>
605
- Object.keys(filter).length === 0 ||
606
- ((filter.since === undefined ||
607
- isBefore(filter.since, operation.timestamp)) &&
608
- (filter.fromRevision === undefined ||
609
- operation.index > filter.fromRevision))
610
- );
611
-
612
- return filteredOperations.map(operation => ({
613
- hash: operation.hash,
614
- index: operation.index,
615
- timestamp: operation.timestamp,
616
- type: operation.type,
617
- input: operation.input as object,
618
- skip: operation.skip,
619
- context: operation.context,
620
- id: operation.id
621
- }));
598
+ const errors: Error[] = [];
599
+ const drives = await this.getDrives();
600
+ for (const drive of drives) {
601
+ await this._initializeDrive(drive).catch((error) => {
602
+ logger.error(`Error initializing drive ${drive}`, error);
603
+ errors.push(error as Error);
604
+ });
622
605
  }
623
606
 
624
- private _getDocumentModel(documentType: string) {
625
- const documentModel = this.documentModels.find(
626
- model => model.documentModel.id === documentType
627
- );
628
- if (!documentModel) {
629
- throw new Error(`Document type ${documentType} not supported`);
630
- }
631
- return documentModel;
607
+ if (this.options.defaultDrives.loadOnInit !== false) {
608
+ await this.defaultDrivesManager.initializeDefaultRemoteDrives();
632
609
  }
633
610
 
634
- async addDrive(drive: DriveInput): Promise<DocumentDriveDocument> {
635
- const id = drive.global.id || generateUUID();
636
- if (!id) {
637
- throw new Error('Invalid Drive Id');
638
- }
639
-
640
- const drives = await this.storage.getDrives();
641
- if (drives.includes(id)) {
642
- throw new Error('Drive already exists');
643
- }
644
-
645
- const document = utils.createDocument({
646
- state: drive
647
- });
648
-
649
- await this.storage.createDrive(id, document);
611
+ // if network connect comes back online
612
+ // then triggers the listeners update
613
+ if (typeof window !== "undefined") {
614
+ window.addEventListener("online", () => {
615
+ this.listenerStateManager
616
+ .triggerUpdate(
617
+ false,
618
+ { type: "local" },
619
+ this.handleListenerError.bind(this),
620
+ )
621
+ .catch((error) => {
622
+ logger.error("Non handled error updating listeners", error);
623
+ });
624
+ });
625
+ }
650
626
 
651
- if (drive.global.slug) {
652
- await this.cache.deleteDocument('drives-slug', drive.global.slug);
653
- }
627
+ return errors.length === 0 ? null : errors;
628
+ }
654
629
 
655
- await this._initializeDrive(id);
630
+ private async _initializeDrive(driveId: string) {
631
+ const drive = await this.getDrive(driveId);
632
+ await this.initializeDriveSyncStatus(driveId, drive);
656
633
 
657
- return document;
634
+ if (this.shouldSyncRemoteDrive(drive)) {
635
+ await this.startSyncRemoteDrive(driveId);
658
636
  }
659
637
 
660
- async addRemoteDrive(
661
- url: string,
662
- options: RemoteDriveOptions
663
- ): Promise<DocumentDriveDocument> {
664
- const { id, name, slug, icon } = await requestPublicDrive(url);
665
- const {
666
- pullFilter,
667
- pullInterval,
668
- availableOffline,
669
- sharingType,
670
- listeners,
671
- triggers
672
- } = options;
673
-
674
- const trigger = await SubscriptionTransmitter.createTrigger(id, url, {
675
- pullFilter
676
- });
677
-
678
- return await this.addDrive({
679
- global: {
680
- id: id,
681
- name,
682
- slug,
683
- icon: icon ?? null
684
- },
685
- local: {
686
- triggers: [...triggers, trigger],
687
- listeners: listeners,
688
- availableOffline,
689
- sharingType
690
- }
691
- });
638
+ await this.listenerStateManager.initDrive(drive);
639
+ }
640
+
641
+ public async getSynchronizationUnits(
642
+ driveId: string,
643
+ documentId?: string[],
644
+ scope?: string[],
645
+ branch?: string[],
646
+ documentType?: string[],
647
+ loadedDrive?: DocumentDriveDocument,
648
+ ) {
649
+ const drive = loadedDrive || (await this.getDrive(driveId));
650
+
651
+ const synchronizationUnitsQuery = await this.getSynchronizationUnitsIds(
652
+ driveId,
653
+ documentId,
654
+ scope,
655
+ branch,
656
+ documentType,
657
+ drive,
658
+ );
659
+ return this.getSynchronizationUnitsRevision(
660
+ driveId,
661
+ synchronizationUnitsQuery,
662
+ drive,
663
+ );
664
+ }
665
+
666
+ public async getSynchronizationUnitsRevision(
667
+ driveId: string,
668
+ syncUnitsQuery: SynchronizationUnitQuery[],
669
+ loadedDrive?: DocumentDriveDocument,
670
+ ): Promise<SynchronizationUnit[]> {
671
+ const drive = loadedDrive || (await this.getDrive(driveId));
672
+
673
+ const revisions =
674
+ await this.storage.getSynchronizationUnitsRevision(syncUnitsQuery);
675
+
676
+ const synchronizationUnits: SynchronizationUnit[] = syncUnitsQuery.map(
677
+ (s) => ({
678
+ ...s,
679
+ lastUpdated: drive.created,
680
+ revision: -1,
681
+ }),
682
+ );
683
+ for (const revision of revisions) {
684
+ const syncUnit = synchronizationUnits.find(
685
+ (s) =>
686
+ revision.driveId === s.driveId &&
687
+ revision.documentId === s.documentId &&
688
+ revision.scope === s.scope &&
689
+ revision.branch === s.branch,
690
+ );
691
+ if (syncUnit) {
692
+ syncUnit.revision = revision.revision;
693
+ syncUnit.lastUpdated = revision.lastUpdated;
694
+ }
692
695
  }
693
-
694
- async deleteDrive(id: string) {
695
- const result = await Promise.allSettled([
696
- this.stopSyncRemoteDrive(id),
697
- this.listenerStateManager.removeDrive(id),
698
- this.cache.deleteDocument('drives', id),
699
- this.storage.deleteDrive(id)
700
- ]);
701
-
702
- result.forEach(r => {
703
- if (r.status === 'rejected') {
704
- throw r.reason;
705
- }
706
- });
696
+ return synchronizationUnits;
697
+ }
698
+
699
+ public async getSynchronizationUnitsIds(
700
+ driveId: string,
701
+ documentId?: string[],
702
+ scope?: string[],
703
+ branch?: string[],
704
+ documentType?: string[],
705
+ loadedDrive?: DocumentDriveDocument,
706
+ ): Promise<SynchronizationUnitQuery[]> {
707
+ const drive = loadedDrive ?? (await this.getDrive(driveId));
708
+ const nodes = drive.state.global.nodes.filter(
709
+ (node) =>
710
+ isFileNode(node) &&
711
+ (!documentId?.length ||
712
+ documentId.includes(node.id) ||
713
+ documentId.includes("*")) &&
714
+ (!documentType?.length ||
715
+ documentType.includes(node.documentType) ||
716
+ documentType.includes("*")),
717
+ ) as Pick<FileNode, "id" | "documentType" | "synchronizationUnits">[];
718
+
719
+ // checks if document drive synchronization unit should be added
720
+ if (
721
+ (!documentId || documentId.includes("*") || documentId.includes("")) &&
722
+ (!documentType?.length ||
723
+ documentType.includes("powerhouse/document-drive") ||
724
+ documentType.includes("*"))
725
+ ) {
726
+ nodes.unshift({
727
+ id: "",
728
+ documentType: "powerhouse/document-drive",
729
+ synchronizationUnits: [
730
+ {
731
+ syncId: "0",
732
+ scope: "global",
733
+ branch: "main",
734
+ },
735
+ ],
736
+ });
707
737
  }
708
738
 
709
- getDrives() {
710
- return this.storage.getDrives();
739
+ const synchronizationUnitsQuery: Omit<
740
+ SynchronizationUnit,
741
+ "revision" | "lastUpdated"
742
+ >[] = [];
743
+ for (const node of nodes) {
744
+ const nodeUnits =
745
+ scope?.length || branch?.length
746
+ ? node.synchronizationUnits.filter(
747
+ (unit) =>
748
+ (!scope?.length ||
749
+ scope.includes(unit.scope) ||
750
+ scope.includes("*")) &&
751
+ (!branch?.length ||
752
+ branch.includes(unit.branch) ||
753
+ branch.includes("*")),
754
+ )
755
+ : node.synchronizationUnits;
756
+ if (!nodeUnits.length) {
757
+ continue;
758
+ }
759
+ synchronizationUnitsQuery.push(
760
+ ...nodeUnits.map((n) => ({
761
+ driveId,
762
+ documentId: node.id,
763
+ syncId: n.syncId,
764
+ documentType: node.documentType,
765
+ scope: n.scope,
766
+ branch: n.branch,
767
+ })),
768
+ );
711
769
  }
712
-
713
- async getDrive(drive: string, options?: GetDocumentOptions) {
714
- try {
715
- const document = await this.cache.getDocument('drives', drive); // TODO support GetDocumentOptions
716
- if (document && isDocumentDrive(document)) {
717
- return document;
718
- }
719
- } catch (e) {
720
- logger.error('Error getting drive from cache', e);
721
- }
722
- const driveStorage = await this.storage.getDrive(drive);
723
- const document = this._buildDocument(driveStorage, options);
724
- if (!isDocumentDrive(document)) {
725
- throw new Error(
726
- `Document with id ${drive} is not a Document Drive`
727
- );
728
- } else {
729
- this.cache
730
- .setDocument('drives', drive, document)
731
- .catch(logger.error);
732
- return document;
733
- }
770
+ return synchronizationUnitsQuery;
771
+ }
772
+
773
+ public async getSynchronizationUnitIdInfo(
774
+ driveId: string,
775
+ syncId: string,
776
+ loadedDrive?: DocumentDriveDocument,
777
+ ): Promise<SynchronizationUnitQuery | undefined> {
778
+ const drive = loadedDrive || (await this.getDrive(driveId));
779
+ const node = drive.state.global.nodes.find(
780
+ (node) =>
781
+ isFileNode(node) &&
782
+ node.synchronizationUnits.find((unit) => unit.syncId === syncId),
783
+ );
784
+
785
+ if (!node || !isFileNode(node)) {
786
+ return undefined;
734
787
  }
735
788
 
736
- async getDriveBySlug(slug: string, options?: GetDocumentOptions) {
737
- try {
738
- const document = await this.cache.getDocument('drives-slug', slug);
739
- if (document && isDocumentDrive(document)) {
740
- return document;
741
- }
742
- } catch (e) {
743
- logger.error('Error getting drive from cache', e);
744
- }
745
-
746
- const driveStorage = await this.storage.getDriveBySlug(slug);
747
- const document = this._buildDocument(driveStorage, options);
748
- if (!isDocumentDrive(document)) {
749
- throw new Error(
750
- `Document with slug ${slug} is not a Document Drive`
751
- );
752
- } else {
753
- this.cache
754
- .setDocument('drives-slug', slug, document)
755
- .catch(logger.error);
756
- return document;
757
- }
789
+ const syncUnit = node.synchronizationUnits.find(
790
+ (unit) => unit.syncId === syncId,
791
+ );
792
+ if (!syncUnit) {
793
+ return undefined;
758
794
  }
759
795
 
760
- async getDocument(drive: string, id: string, options?: GetDocumentOptions) {
761
- try {
762
- const document = await this.cache.getDocument(drive, id); // TODO support GetDocumentOptions
763
- if (document) {
764
- return document;
765
- }
766
- } catch (e) {
767
- logger.error('Error getting document from cache', e);
768
- }
769
- const documentStorage = await this.storage.getDocument(drive, id);
770
- const document = this._buildDocument(documentStorage, options);
771
-
772
- this.cache.setDocument(drive, id, document).catch(logger.error);
773
- return document;
796
+ return {
797
+ syncId,
798
+ scope: syncUnit.scope,
799
+ branch: syncUnit.branch,
800
+ driveId,
801
+ documentId: node.id,
802
+ documentType: node.documentType,
803
+ };
804
+ }
805
+
806
+ public async getSynchronizationUnit(
807
+ driveId: string,
808
+ syncId: string,
809
+ loadedDrive?: DocumentDriveDocument,
810
+ ): Promise<SynchronizationUnit | undefined> {
811
+ const syncUnit = await this.getSynchronizationUnitIdInfo(
812
+ driveId,
813
+ syncId,
814
+ loadedDrive,
815
+ );
816
+
817
+ if (!syncUnit) {
818
+ return undefined;
774
819
  }
775
820
 
776
- getDocuments(drive: string) {
777
- return this.storage.getDocuments(drive);
821
+ const { scope, branch, documentId, documentType } = syncUnit;
822
+
823
+ // TODO: REPLACE WITH GET DOCUMENT OPERATIONS
824
+ const document = await this.getDocument(driveId, documentId);
825
+ const operations = document.operations[scope as OperationScope] ?? [];
826
+ const lastOperation = operations[operations.length - 1];
827
+
828
+ return {
829
+ syncId,
830
+ scope,
831
+ branch,
832
+ driveId,
833
+ documentId,
834
+ documentType,
835
+ lastUpdated: lastOperation?.timestamp ?? document.lastModified,
836
+ revision: lastOperation?.index ?? 0,
837
+ };
838
+ }
839
+
840
+ async getOperationData(
841
+ driveId: string,
842
+ syncId: string,
843
+ filter: GetStrandsOptions,
844
+ loadedDrive?: DocumentDriveDocument,
845
+ ): Promise<OperationUpdate[]> {
846
+ const syncUnit =
847
+ syncId === "0"
848
+ ? { documentId: "", scope: "global" }
849
+ : await this.getSynchronizationUnitIdInfo(driveId, syncId, loadedDrive);
850
+
851
+ if (!syncUnit) {
852
+ throw new Error(`Invalid Sync Id ${syncId} in drive ${driveId}`);
778
853
  }
779
854
 
780
- protected async createDocument(
781
- driveId: string,
782
- input: CreateDocumentInput
783
- ) {
784
- // if a document was provided then checks if it's valid
785
- let state = undefined;
786
- if (input.document) {
787
- if (input.documentType !== input.document.documentType) {
788
- throw new Error(
789
- `Provided document is not ${input.documentType}`
790
- );
791
- }
792
- const doc = this._buildDocument(input.document);
793
- state = doc.state;
794
- }
795
-
796
- // if no document was provided then create a new one
797
- const document =
798
- input.document ??
799
- this._getDocumentModel(input.documentType).utils.createDocument();
800
-
801
- // stores document information
802
- const documentStorage: DocumentStorage = {
803
- name: document.name,
804
- revision: document.revision,
805
- documentType: document.documentType,
806
- created: document.created,
807
- lastModified: document.lastModified,
808
- operations: { global: [], local: [] },
809
- initialState: document.initialState,
810
- clipboard: [],
811
- state: state ?? document.state
812
- };
813
- await this.storage.createDocument(driveId, input.id, documentStorage);
814
-
815
- // if the document contains operations then
816
- // stores the operations in the storage
817
- const operations = Object.values(document.operations).flat();
818
- if (operations.length) {
819
- if (isDocumentDrive(document)) {
820
- await this.storage.addDriveOperations(
821
- driveId,
822
- operations as Operation<DocumentDriveAction>[],
823
- document
824
- );
825
- } else {
826
- await this.storage.addDocumentOperations(
827
- driveId,
828
- input.id,
829
- operations,
830
- document
831
- );
832
- }
833
- }
834
-
835
- return document;
855
+ const document =
856
+ syncId === "0"
857
+ ? loadedDrive || (await this.getDrive(driveId))
858
+ : await this.getDocument(driveId, syncUnit.documentId); // TODO replace with getDocumentOperations
859
+
860
+ const operations =
861
+ document.operations[syncUnit.scope as OperationScope] ?? []; // TODO filter by branch also
862
+
863
+ const filteredOperations = operations.filter(
864
+ (operation) =>
865
+ Object.keys(filter).length === 0 ||
866
+ ((filter.since === undefined ||
867
+ isBefore(filter.since, operation.timestamp)) &&
868
+ (filter.fromRevision === undefined ||
869
+ operation.index > filter.fromRevision)),
870
+ );
871
+
872
+ const limitedOperations = filter.limit
873
+ ? filteredOperations.slice(0, filter.limit)
874
+ : filteredOperations;
875
+
876
+ return limitedOperations.map((operation) => ({
877
+ hash: operation.hash,
878
+ index: operation.index,
879
+ timestamp: operation.timestamp,
880
+ type: operation.type,
881
+ input: operation.input as object,
882
+ skip: operation.skip,
883
+ context: operation.context,
884
+ id: operation.id,
885
+ }));
886
+ }
887
+
888
+ protected getDocumentModel(documentType: string) {
889
+ const documentModel = this.documentModels.find(
890
+ (model) => model.documentModel.id === documentType,
891
+ );
892
+ if (!documentModel) {
893
+ throw new Error(`Document type ${documentType} not supported`);
836
894
  }
895
+ return documentModel;
896
+ }
837
897
 
838
- async deleteDocument(driveId: string, id: string) {
839
- try {
840
- const syncUnits = await this.getSynchronizationUnitsIds(driveId, [
841
- id
842
- ]);
843
- await this.listenerStateManager.removeSyncUnits(driveId, syncUnits);
844
- } catch (error) {
845
- logger.warn('Error deleting document', error);
846
- }
847
- await this.cache.deleteDocument(driveId, id);
848
- return this.storage.deleteDocument(driveId, id);
898
+ async addDrive(drive: DriveInput): Promise<DocumentDriveDocument> {
899
+ const id = drive.global.id || generateUUID();
900
+ if (!id) {
901
+ throw new Error("Invalid Drive Id");
849
902
  }
850
903
 
851
- async _processOperations<T extends Document, A extends Action>(
852
- drive: string,
853
- documentId: string | undefined,
854
- storageDocument: DocumentStorage<T>,
855
- operations: Operation<A | BaseAction>[]
856
- ) {
857
- const operationsApplied: Operation<A | BaseAction>[] = [];
858
- const signals: SignalResult[] = [];
859
- let document: T = this._buildDocument(storageDocument);
860
-
861
- let error: OperationError | undefined; // TODO: replace with an array of errors/consistency issues
862
- const operationsByScope = groupOperationsByScope(operations);
863
-
864
- for (const scope of Object.keys(operationsByScope)) {
865
- const storageDocumentOperations =
866
- storageDocument.operations[scope as OperationScope];
867
-
868
- // TODO two equal operations done by two clients will be considered the same, ie: { type: "INCREMENT" }
869
- const branch = removeExistingOperations(
870
- operationsByScope[scope as OperationScope] || [],
871
- storageDocumentOperations
872
- );
873
-
874
- // No operations to apply
875
- if (branch.length < 1) {
876
- continue;
877
- }
878
-
879
- const trunk = garbageCollect(
880
- sortOperations(storageDocumentOperations)
881
- );
882
-
883
- const [invertedTrunk, tail] = attachBranch(trunk, branch);
884
-
885
- const newHistory =
886
- tail.length < 1
887
- ? invertedTrunk
888
- : merge(trunk, invertedTrunk, reshuffleByTimestamp);
889
-
890
- const newOperations = newHistory.filter(
891
- op => trunk.length < 1 || precedes(trunk[trunk.length - 1]!, op)
892
- );
893
-
894
- for (const nextOperation of newOperations) {
895
- let skipHashValidation = false;
896
-
897
- // when dealing with a merge (tail.length > 0) we have to skip hash validation
898
- // for the operations that were re-indexed (previous hash becomes invalid due the new position in the history)
899
- if (tail.length > 0) {
900
- const sourceOperation = operations.find(
901
- op => op.hash === nextOperation.hash
902
- );
903
-
904
- skipHashValidation =
905
- !sourceOperation ||
906
- sourceOperation.index !== nextOperation.index ||
907
- sourceOperation.skip !== nextOperation.skip;
908
- }
909
-
910
- try {
911
- const appliedResult = await this._performOperation(
912
- drive,
913
- documentId,
914
- document,
915
- nextOperation,
916
- skipHashValidation
917
- );
918
- document = appliedResult.document;
919
- signals.push(...appliedResult.signals);
920
- operationsApplied.push(...appliedResult.operation);
921
- } catch (e) {
922
- error =
923
- e instanceof OperationError
924
- ? e
925
- : new OperationError(
926
- 'ERROR',
927
- nextOperation,
928
- (e as Error).message,
929
- (e as Error).cause
930
- );
931
-
932
- // TODO: don't break on errors...
933
- break;
934
- }
935
- }
936
- }
937
-
938
- return {
939
- document,
940
- operationsApplied,
941
- signals,
942
- error
943
- } as const;
904
+ const drives = await this.storage.getDrives();
905
+ if (drives.includes(id)) {
906
+ throw new DriveAlreadyExistsError(id);
944
907
  }
945
908
 
946
- private _buildDocument<T extends Document>(
947
- documentStorage: DocumentStorage<T>,
948
- options?: GetDocumentOptions
949
- ): T {
950
- if (
951
- documentStorage.state &&
952
- (!options || options.checkHashes === false)
953
- ) {
954
- return documentStorage as T;
955
- }
909
+ const document = utils.createDocument({
910
+ state: drive,
911
+ });
956
912
 
957
- const documentModel = this._getDocumentModel(
958
- documentStorage.documentType
959
- );
913
+ await this.storage.createDrive(id, document);
960
914
 
961
- const revisionOperations =
962
- options?.revisions !== undefined
963
- ? filterOperationsByRevision(
964
- documentStorage.operations,
965
- options.revisions
966
- )
967
- : documentStorage.operations;
968
- const operations =
969
- baseUtils.documentHelpers.garbageCollectDocumentOperations(
970
- revisionOperations
971
- );
972
-
973
- return baseUtils.replayDocument(
974
- documentStorage.initialState,
975
- operations,
976
- documentModel.reducer,
977
- undefined,
978
- documentStorage,
979
- undefined,
980
- {
981
- ...options,
982
- checkHashes: options?.checkHashes ?? true,
983
- reuseOperationResultingState: options?.checkHashes ?? true
984
- }
985
- ) as T;
915
+ if (drive.global.slug) {
916
+ await this.cache.deleteDocument("drives-slug", drive.global.slug);
986
917
  }
987
918
 
988
- private async _performOperation<T extends Document>(
989
- drive: string,
990
- id: string | undefined,
991
- document: T,
992
- operation: Operation,
993
- skipHashValidation = false
994
- ) {
995
- const documentModel = this._getDocumentModel(document.documentType);
996
-
997
- const signalResults: SignalResult[] = [];
998
- let newDocument = document;
999
-
1000
- const scope = operation.scope;
1001
- const documentOperations =
1002
- DocumentUtils.documentHelpers.garbageCollectDocumentOperations({
1003
- ...document.operations,
1004
- [scope]: DocumentUtils.documentHelpers.skipHeaderOperations(
1005
- document.operations[scope],
1006
- operation
1007
- )
1008
- });
1009
-
1010
- const lastRemainingOperation = documentOperations[scope].at(-1);
1011
- // if the latest operation doesn't have a resulting state then tries
1012
- // to retrieve it from the db to avoid rerunning all the operations
1013
- if (lastRemainingOperation && !lastRemainingOperation.resultingState) {
1014
- lastRemainingOperation.resultingState = await (id
1015
- ? this.storage.getOperationResultingState?.(
1016
- drive,
1017
- id,
1018
- lastRemainingOperation.index,
1019
- lastRemainingOperation.scope,
1020
- 'main'
1021
- )
1022
- : this.storage.getDriveOperationResultingState?.(
1023
- drive,
1024
- lastRemainingOperation.index,
1025
- lastRemainingOperation.scope,
1026
- 'main'
1027
- ));
1028
- }
1029
-
1030
- const operationSignals: (() => Promise<SignalResult>)[] = [];
1031
- newDocument = documentModel.reducer(
1032
- newDocument,
1033
- operation,
1034
- signal => {
1035
- let handler: (() => Promise<unknown>) | undefined = undefined;
1036
- switch (signal.type) {
1037
- case 'CREATE_CHILD_DOCUMENT':
1038
- handler = () =>
1039
- this.createDocument(drive, signal.input);
1040
- break;
1041
- case 'DELETE_CHILD_DOCUMENT':
1042
- handler = () =>
1043
- this.deleteDocument(drive, signal.input.id);
1044
- break;
1045
- case 'COPY_CHILD_DOCUMENT':
1046
- handler = () =>
1047
- this.getDocument(drive, signal.input.id).then(
1048
- documentToCopy =>
1049
- this.createDocument(drive, {
1050
- id: signal.input.newId,
1051
- documentType:
1052
- documentToCopy.documentType,
1053
- document: documentToCopy,
1054
- synchronizationUnits:
1055
- signal.input.synchronizationUnits
1056
- })
1057
- );
1058
- break;
1059
- }
1060
- if (handler) {
1061
- operationSignals.push(() =>
1062
- handler().then(result => ({ signal, result }))
1063
- );
1064
- }
1065
- },
1066
- { skip: operation.skip, reuseOperationResultingState: true }
1067
- ) as T;
1068
-
1069
- const appliedOperation = newDocument.operations[operation.scope].filter(
1070
- op => op.index == operation.index && op.skip == operation.skip
1071
- );
1072
-
1073
- if (appliedOperation.length < 1) {
1074
- throw new OperationError(
1075
- 'ERROR',
1076
- operation,
1077
- `Operation with index ${operation.index}:${operation.skip || 0} was not applied.`
1078
- );
1079
- } else if (
1080
- appliedOperation[0]!.hash !== operation.hash &&
1081
- !skipHashValidation
1082
- ) {
1083
- throw new ConflictOperationError(operation, appliedOperation[0]!);
1084
- }
1085
-
1086
- for (const signalHandler of operationSignals) {
1087
- const result = await signalHandler();
1088
- signalResults.push(result);
1089
- }
1090
-
1091
- return {
1092
- document: newDocument,
1093
- signals: signalResults,
1094
- operation: appliedOperation
1095
- };
919
+ await this._initializeDrive(id);
920
+
921
+ return document;
922
+ }
923
+
924
+ async addRemoteDrive(
925
+ url: string,
926
+ options: RemoteDriveOptions,
927
+ ): Promise<DocumentDriveDocument> {
928
+ const { id, name, slug, icon } =
929
+ options.expectedDriveInfo || (await requestPublicDrive(url));
930
+
931
+ const {
932
+ pullFilter,
933
+ pullInterval,
934
+ availableOffline,
935
+ sharingType,
936
+ listeners,
937
+ triggers,
938
+ } = options;
939
+
940
+ const pullTrigger =
941
+ await PullResponderTransmitter.createPullResponderTrigger(id, url, {
942
+ pullFilter,
943
+ pullInterval,
944
+ });
945
+
946
+ return await this.addDrive({
947
+ global: {
948
+ id: id,
949
+ name,
950
+ slug,
951
+ icon: icon ?? null,
952
+ },
953
+ local: {
954
+ triggers: [...triggers, pullTrigger],
955
+ listeners: listeners,
956
+ availableOffline,
957
+ sharingType,
958
+ },
959
+ });
960
+ }
961
+
962
+ public async registerPullResponderTrigger(
963
+ id: string,
964
+ url: string,
965
+ options: Pick<RemoteDriveOptions, "pullFilter" | "pullInterval">,
966
+ ) {
967
+ const pullTrigger =
968
+ await PullResponderTransmitter.createPullResponderTrigger(
969
+ id,
970
+ url,
971
+ options,
972
+ );
973
+
974
+ return pullTrigger;
975
+ }
976
+
977
+ async deleteDrive(id: string) {
978
+ const result = await Promise.allSettled([
979
+ this.stopSyncRemoteDrive(id),
980
+ this.listenerStateManager.removeDrive(id),
981
+ this.cache.deleteDocument("drives", id),
982
+ this.storage.deleteDrive(id),
983
+ ]);
984
+
985
+ result.forEach((r) => {
986
+ if (r.status === "rejected") {
987
+ throw r.reason;
988
+ }
989
+ });
990
+ }
991
+
992
+ getDrives() {
993
+ return this.storage.getDrives();
994
+ }
995
+
996
+ async getDrive(drive: string, options?: GetDocumentOptions) {
997
+ try {
998
+ const document = await this.cache.getDocument("drives", drive); // TODO support GetDocumentOptions
999
+ if (document && isDocumentDrive(document)) {
1000
+ return document;
1001
+ }
1002
+ } catch (e) {
1003
+ logger.error("Error getting drive from cache", e);
1004
+ }
1005
+ const driveStorage = await this.storage.getDrive(drive);
1006
+ const document = this._buildDocument(driveStorage, options);
1007
+ if (!isDocumentDrive(document)) {
1008
+ throw new Error(`Document with id ${drive} is not a Document Drive`);
1009
+ } else {
1010
+ this.cache.setDocument("drives", drive, document).catch(logger.error);
1011
+ return document;
1096
1012
  }
1013
+ }
1097
1014
 
1098
- addOperation(
1099
- drive: string,
1100
- id: string,
1101
- operation: Operation,
1102
- forceSync = true
1103
- ): Promise<IOperationResult> {
1104
- return this.addOperations(drive, id, [operation], forceSync);
1015
+ async getDriveBySlug(slug: string, options?: GetDocumentOptions) {
1016
+ try {
1017
+ const document = await this.cache.getDocument("drives-slug", slug);
1018
+ if (document && isDocumentDrive(document)) {
1019
+ return document;
1020
+ }
1021
+ } catch (e) {
1022
+ logger.error("Error getting drive from cache", e);
1105
1023
  }
1106
1024
 
1107
- private async _addOperations(
1108
- drive: string,
1109
- id: string,
1110
- callback: (document: DocumentStorage) => Promise<{
1111
- operations: Operation[];
1112
- header: DocumentHeader;
1113
- }>
1114
- ) {
1115
- if (!this.storage.addDocumentOperationsWithTransaction) {
1116
- const documentStorage = await this.storage.getDocument(drive, id);
1117
- const result = await callback(documentStorage);
1118
- // saves the applied operations to storage
1119
- if (result.operations.length > 0) {
1120
- await this.storage.addDocumentOperations(
1121
- drive,
1122
- id,
1123
- result.operations,
1124
- result.header
1125
- );
1126
- }
1127
- } else {
1128
- await this.storage.addDocumentOperationsWithTransaction(
1129
- drive,
1130
- id,
1131
- callback
1132
- );
1133
- }
1025
+ const driveStorage = await this.storage.getDriveBySlug(slug);
1026
+ const document = this._buildDocument(driveStorage, options);
1027
+ if (!isDocumentDrive(document)) {
1028
+ throw new Error(`Document with slug ${slug} is not a Document Drive`);
1029
+ } else {
1030
+ this.cache.setDocument("drives-slug", slug, document).catch(logger.error);
1031
+ return document;
1134
1032
  }
1033
+ }
1135
1034
 
1136
- queueOperation(
1137
- drive: string,
1138
- id: string,
1139
- operation: Operation,
1140
- forceSync = true
1141
- ): Promise<IOperationResult> {
1142
- return this.queueOperations(drive, id, [operation], forceSync);
1035
+ async getDocument(drive: string, id: string, options?: GetDocumentOptions) {
1036
+ try {
1037
+ const document = await this.cache.getDocument(drive, id); // TODO support GetDocumentOptions
1038
+ if (document) {
1039
+ return document;
1040
+ }
1041
+ } catch (e) {
1042
+ logger.error("Error getting document from cache", e);
1043
+ }
1044
+ const documentStorage = await this.storage.getDocument(drive, id);
1045
+ const document = this._buildDocument(documentStorage, options);
1046
+
1047
+ this.cache.setDocument(drive, id, document).catch(logger.error);
1048
+ return document;
1049
+ }
1050
+
1051
+ getDocuments(drive: string) {
1052
+ return this.storage.getDocuments(drive);
1053
+ }
1054
+
1055
+ protected async createDocument(driveId: string, input: CreateDocumentInput) {
1056
+ // if a document was provided then checks if it's valid
1057
+ let state = undefined;
1058
+ if (input.document) {
1059
+ if (input.documentType !== input.document.documentType) {
1060
+ throw new Error(`Provided document is not ${input.documentType}`);
1061
+ }
1062
+ const doc = this._buildDocument(input.document);
1063
+ state = doc.state;
1143
1064
  }
1144
1065
 
1145
- private async resultIfExistingOperations(
1146
- drive: string,
1147
- id: string,
1148
- operations: Operation[]
1149
- ): Promise<IOperationResult | undefined> {
1150
- try {
1151
- const document = await this.getDocument(drive, id);
1152
- const newOperation = operations.find(
1153
- op =>
1154
- !op.id ||
1155
- !document.operations[op.scope].find(
1156
- existingOp =>
1157
- existingOp.id === op.id &&
1158
- existingOp.index === op.index &&
1159
- existingOp.type === op.type &&
1160
- existingOp.hash === op.hash
1161
- )
1162
- );
1163
- if (!newOperation) {
1164
- return {
1165
- status: 'SUCCESS',
1166
- document,
1167
- operations,
1168
- signals: []
1169
- };
1170
- } else {
1171
- return undefined;
1172
- }
1173
- } catch (error) {
1174
- console.error(error); // TODO error
1175
- return undefined;
1176
- }
1066
+ // if no document was provided then create a new one
1067
+ const document =
1068
+ input.document ??
1069
+ this.getDocumentModel(input.documentType).utils.createDocument();
1070
+
1071
+ // stores document information
1072
+ const documentStorage: DocumentStorage = {
1073
+ name: document.name,
1074
+ revision: document.revision,
1075
+ documentType: document.documentType,
1076
+ created: document.created,
1077
+ lastModified: document.lastModified,
1078
+ operations: { global: [], local: [] },
1079
+ initialState: document.initialState,
1080
+ clipboard: [],
1081
+ state: state ?? document.state,
1082
+ };
1083
+ await this.storage.createDocument(driveId, input.id, documentStorage);
1084
+
1085
+ // set initial state for new syncUnits
1086
+ for (const syncUnit of input.synchronizationUnits) {
1087
+ this.initSyncStatus(syncUnit.syncId, {
1088
+ pull: this.triggerMap.get(driveId) ? "INITIAL_SYNC" : undefined,
1089
+ push: this.listenerStateManager.driveHasListeners(driveId)
1090
+ ? "SUCCESS"
1091
+ : undefined,
1092
+ });
1177
1093
  }
1178
1094
 
1179
- async queueOperations(
1180
- drive: string,
1181
- id: string,
1182
- operations: Operation[],
1183
- forceSync = true
1184
- ) {
1185
- // if operations are already stored then returns cached document
1186
- const result = await this.resultIfExistingOperations(
1187
- drive,
1188
- id,
1189
- operations
1095
+ // if the document contains operations then
1096
+ // stores the operations in the storage
1097
+ const operations = Object.values(document.operations).flat();
1098
+ if (operations.length) {
1099
+ if (isDocumentDrive(document)) {
1100
+ await this.storage.addDriveOperations(
1101
+ driveId,
1102
+ operations as Operation<DocumentDriveAction>[],
1103
+ document,
1190
1104
  );
1191
- if (result) {
1192
- console.log('Duplicated operations!');
1193
- return result;
1194
- }
1195
- try {
1196
- const jobId = await this.queueManager.addJob({
1197
- driveId: drive,
1198
- documentId: id,
1199
- operations,
1200
- forceSync
1201
- });
1202
-
1203
- return new Promise<IOperationResult>((resolve, reject) => {
1204
- const unsubscribe = this.queueManager.on(
1205
- 'jobCompleted',
1206
- (job, result) => {
1207
- if (job.jobId === jobId) {
1208
- unsubscribe();
1209
- unsubscribeError();
1210
- resolve(result);
1211
- }
1212
- }
1213
- );
1214
- const unsubscribeError = this.queueManager.on(
1215
- 'jobFailed',
1216
- (job, error) => {
1217
- if (job.jobId === jobId) {
1218
- unsubscribe();
1219
- unsubscribeError();
1220
- reject(error);
1221
- }
1222
- }
1223
- );
1224
- });
1225
- } catch (error) {
1226
- logger.error('Error adding job', error);
1227
- throw error;
1228
- }
1105
+ } else {
1106
+ await this.storage.addDocumentOperations(
1107
+ driveId,
1108
+ input.id,
1109
+ operations,
1110
+ document,
1111
+ );
1112
+ }
1229
1113
  }
1230
1114
 
1231
- async queueAction(
1232
- drive: string,
1233
- id: string,
1234
- action: Action,
1235
- forceSync?: boolean | undefined
1236
- ): Promise<IOperationResult> {
1237
- return this.queueActions(drive, id, [action], forceSync);
1115
+ return document;
1116
+ }
1117
+
1118
+ async deleteDocument(driveId: string, id: string) {
1119
+ try {
1120
+ const syncUnits = await this.getSynchronizationUnitsIds(driveId, [id]);
1121
+
1122
+ // remove document sync units status when a document is deleted
1123
+ for (const syncUnit of syncUnits) {
1124
+ this.updateSyncUnitStatus(syncUnit.syncId, null);
1125
+ }
1126
+ await this.listenerStateManager.removeSyncUnits(driveId, syncUnits);
1127
+ } catch (error) {
1128
+ logger.warn("Error deleting document", error);
1238
1129
  }
1130
+ await this.cache.deleteDocument(driveId, id);
1131
+ return this.storage.deleteDocument(driveId, id);
1132
+ }
1133
+
1134
+ async _processOperations<T extends Document, A extends Action>(
1135
+ drive: string,
1136
+ documentId: string | undefined,
1137
+ documentStorage: DocumentStorage<T>,
1138
+ operations: Operation<A | BaseAction>[],
1139
+ ) {
1140
+ const operationsApplied: Operation<A | BaseAction>[] = [];
1141
+ const signals: SignalResult[] = [];
1142
+
1143
+ const documentStorageWithState = await this._addDocumentResultingStage(
1144
+ documentStorage,
1145
+ drive,
1146
+ documentId,
1147
+ );
1148
+
1149
+ let document: T = this._buildDocument(documentStorageWithState);
1150
+ let error: OperationError | undefined; // TODO: replace with an array of errors/consistency issues
1151
+ const operationsByScope = groupOperationsByScope(operations);
1152
+
1153
+ for (const scope of Object.keys(operationsByScope)) {
1154
+ const storageDocumentOperations =
1155
+ documentStorage.operations[scope as OperationScope];
1156
+
1157
+ // TODO two equal operations done by two clients will be considered the same, ie: { type: "INCREMENT" }
1158
+ const branch = removeExistingOperations(
1159
+ operationsByScope[scope as OperationScope] || [],
1160
+ storageDocumentOperations,
1161
+ );
1162
+
1163
+ // No operations to apply
1164
+ if (branch.length < 1) {
1165
+ continue;
1166
+ }
1167
+
1168
+ const trunk = garbageCollect(sortOperations(storageDocumentOperations));
1169
+
1170
+ const [invertedTrunk, tail] = attachBranch(trunk, branch);
1171
+
1172
+ const newHistory =
1173
+ tail.length < 1
1174
+ ? invertedTrunk
1175
+ : merge(trunk, invertedTrunk, reshuffleByTimestamp);
1176
+
1177
+ const newOperations = newHistory.filter(
1178
+ (op) => trunk.length < 1 || precedes(trunk[trunk.length - 1]!, op),
1179
+ );
1180
+
1181
+ for (const nextOperation of newOperations) {
1182
+ let skipHashValidation = false;
1183
+
1184
+ // when dealing with a merge (tail.length > 0) we have to skip hash validation
1185
+ // for the operations that were re-indexed (previous hash becomes invalid due the new position in the history)
1186
+ if (tail.length > 0) {
1187
+ const sourceOperation = operations.find(
1188
+ (op) => op.hash === nextOperation.hash,
1189
+ );
1190
+
1191
+ skipHashValidation =
1192
+ !sourceOperation ||
1193
+ sourceOperation.index !== nextOperation.index ||
1194
+ sourceOperation.skip !== nextOperation.skip;
1195
+ }
1239
1196
 
1240
- async queueActions(
1241
- drive: string,
1242
- id: string,
1243
- actions: Action[],
1244
- forceSync?: boolean | undefined
1245
- ): Promise<IOperationResult> {
1246
1197
  try {
1247
- const jobId = await this.queueManager.addJob({
1248
- driveId: drive,
1249
- documentId: id,
1250
- actions,
1251
- forceSync
1252
- });
1253
-
1254
- return new Promise<IOperationResult>((resolve, reject) => {
1255
- const unsubscribe = this.queueManager.on(
1256
- 'jobCompleted',
1257
- (job, result) => {
1258
- if (job.jobId === jobId) {
1259
- unsubscribe();
1260
- unsubscribeError();
1261
- resolve(result);
1262
- }
1263
- }
1264
- );
1265
- const unsubscribeError = this.queueManager.on(
1266
- 'jobFailed',
1267
- (job, error) => {
1268
- if (job.jobId === jobId) {
1269
- unsubscribe();
1270
- unsubscribeError();
1271
- reject(error);
1272
- }
1273
- }
1198
+ // runs operation on next available tick, to avoid blocking the main thread
1199
+ const taskQueueMethod = this.options.taskQueueMethod;
1200
+ const task = () =>
1201
+ this._performOperation(
1202
+ drive,
1203
+ documentId,
1204
+ document,
1205
+ nextOperation,
1206
+ skipHashValidation,
1207
+ );
1208
+ const appliedResult = await (taskQueueMethod
1209
+ ? runAsapAsync(task, taskQueueMethod)
1210
+ : task());
1211
+ document = appliedResult.document;
1212
+ signals.push(...appliedResult.signals);
1213
+ operationsApplied.push(appliedResult.operation);
1214
+
1215
+ // TODO what to do if one of the applied operations has an error?
1216
+ } catch (e) {
1217
+ error =
1218
+ e instanceof OperationError
1219
+ ? e
1220
+ : new OperationError(
1221
+ "ERROR",
1222
+ nextOperation,
1223
+ (e as Error).message,
1224
+ (e as Error).cause,
1274
1225
  );
1275
- });
1276
- } catch (error) {
1277
- logger.error('Error adding job', error);
1278
- throw error;
1226
+
1227
+ // TODO: don't break on errors...
1228
+ break;
1279
1229
  }
1230
+ }
1280
1231
  }
1281
1232
 
1282
- async queueDriveAction(
1283
- drive: string,
1284
- action: DocumentDriveAction | BaseAction,
1285
- forceSync?: boolean | undefined
1286
- ): Promise<IOperationResult<DocumentDriveDocument>> {
1287
- return this.queueDriveActions(drive, [action], forceSync);
1233
+ return {
1234
+ document,
1235
+ operationsApplied,
1236
+ signals,
1237
+ error,
1238
+ } as const;
1239
+ }
1240
+
1241
+ private async _addDocumentResultingStage<T extends Document>(
1242
+ document: DocumentStorage<T>,
1243
+ drive: string,
1244
+ documentId?: string,
1245
+ options?: GetDocumentOptions,
1246
+ ): Promise<DocumentStorage<T>> {
1247
+ // apply skip header operations to all scopes
1248
+ const operations =
1249
+ options?.revisions !== undefined
1250
+ ? filterOperationsByRevision(document.operations, options.revisions)
1251
+ : document.operations;
1252
+ const documentOperations =
1253
+ DocumentUtils.documentHelpers.garbageCollectDocumentOperations(
1254
+ operations,
1255
+ );
1256
+
1257
+ for (const scope of Object.keys(documentOperations)) {
1258
+ const lastRemainingOperation =
1259
+ documentOperations[scope as OperationScope].at(-1);
1260
+ // if the latest operation doesn't have a resulting state then tries
1261
+ // to retrieve it from the db to avoid rerunning all the operations
1262
+ if (lastRemainingOperation && !lastRemainingOperation.resultingState) {
1263
+ lastRemainingOperation.resultingState = await (documentId
1264
+ ? this.storage.getOperationResultingState?.(
1265
+ drive,
1266
+ documentId,
1267
+ lastRemainingOperation.index,
1268
+ lastRemainingOperation.scope,
1269
+ "main",
1270
+ )
1271
+ : this.storage.getDriveOperationResultingState?.(
1272
+ drive,
1273
+ lastRemainingOperation.index,
1274
+ lastRemainingOperation.scope,
1275
+ "main",
1276
+ ));
1277
+ }
1288
1278
  }
1289
1279
 
1290
- async queueDriveActions(
1291
- drive: string,
1292
- actions: (DocumentDriveAction | BaseAction)[],
1293
- forceSync?: boolean | undefined
1294
- ): Promise<IOperationResult<DocumentDriveDocument>> {
1295
- const jobId = await this.queueManager.addJob({
1296
- driveId: drive,
1297
- actions,
1298
- forceSync
1299
- });
1300
- return new Promise<IOperationResult<DocumentDriveDocument>>(
1301
- (resolve, reject) => {
1302
- const unsubscribe = this.queueManager.on(
1303
- 'jobCompleted',
1304
- (job, result) => {
1305
- if (job.jobId === jobId) {
1306
- unsubscribe();
1307
- unsubscribeError();
1308
- resolve(
1309
- result as IOperationResult<DocumentDriveDocument>
1310
- );
1311
- }
1312
- }
1313
- );
1314
- const unsubscribeError = this.queueManager.on(
1315
- 'jobFailed',
1316
- (job, error) => {
1317
- if (job.jobId === jobId) {
1318
- unsubscribe();
1319
- unsubscribeError();
1320
- reject(error);
1321
- }
1322
- }
1323
- );
1324
- }
1325
- );
1280
+ return {
1281
+ ...document,
1282
+ operations: documentOperations,
1283
+ };
1284
+ }
1285
+
1286
+ private _buildDocument<T extends Document>(
1287
+ documentStorage: DocumentStorage<T>,
1288
+ options?: GetDocumentOptions,
1289
+ ): T {
1290
+ if (documentStorage.state && (!options || options.checkHashes === false)) {
1291
+ return documentStorage as T;
1326
1292
  }
1327
1293
 
1328
- async addOperations(
1329
- drive: string,
1330
- id: string,
1331
- operations: Operation[],
1332
- forceSync = true
1333
- ) {
1334
- // if operations are already stored then returns the result
1335
- const result = await this.resultIfExistingOperations(
1294
+ const documentModel = this.getDocumentModel(documentStorage.documentType);
1295
+
1296
+ const revisionOperations =
1297
+ options?.revisions !== undefined
1298
+ ? filterOperationsByRevision(
1299
+ documentStorage.operations,
1300
+ options.revisions,
1301
+ )
1302
+ : documentStorage.operations;
1303
+ const operations =
1304
+ baseUtils.documentHelpers.garbageCollectDocumentOperations(
1305
+ revisionOperations,
1306
+ );
1307
+
1308
+ return baseUtils.replayDocument(
1309
+ documentStorage.initialState,
1310
+ operations,
1311
+ documentModel.reducer,
1312
+ undefined,
1313
+ documentStorage,
1314
+ undefined,
1315
+ {
1316
+ ...options,
1317
+ checkHashes: options?.checkHashes ?? true,
1318
+ reuseOperationResultingState: options?.checkHashes ?? true,
1319
+ },
1320
+ ) as T;
1321
+ }
1322
+
1323
+ private async _performOperation<T extends Document>(
1324
+ drive: string,
1325
+ id: string | undefined,
1326
+ document: T,
1327
+ operation: Operation,
1328
+ skipHashValidation = false,
1329
+ ) {
1330
+ const documentModel = this.getDocumentModel(document.documentType);
1331
+
1332
+ const signalResults: SignalResult[] = [];
1333
+ let newDocument = document;
1334
+
1335
+ const scope = operation.scope;
1336
+ const documentOperations =
1337
+ DocumentUtils.documentHelpers.garbageCollectDocumentOperations({
1338
+ ...document.operations,
1339
+ [scope]: DocumentUtils.documentHelpers.skipHeaderOperations(
1340
+ document.operations[scope],
1341
+ operation,
1342
+ ),
1343
+ });
1344
+
1345
+ const lastRemainingOperation = documentOperations[scope].at(-1);
1346
+ // if the latest operation doesn't have a resulting state then tries
1347
+ // to retrieve it from the db to avoid rerunning all the operations
1348
+ if (lastRemainingOperation && !lastRemainingOperation.resultingState) {
1349
+ lastRemainingOperation.resultingState = await (id
1350
+ ? this.storage.getOperationResultingState?.(
1336
1351
  drive,
1337
1352
  id,
1338
- operations
1339
- );
1340
- if (result) {
1341
- return result;
1342
- }
1343
- let document: Document | undefined;
1344
- const operationsApplied: Operation[] = [];
1345
- const signals: SignalResult[] = [];
1346
- let error: Error | undefined;
1347
-
1348
- try {
1349
- await this._addOperations(drive, id, async documentStorage => {
1350
- const result = await this._processOperations(
1351
- drive,
1352
- id,
1353
- documentStorage,
1354
- operations
1355
- );
1356
-
1357
- if (!result.document) {
1358
- logger.error('Invalid document');
1359
- throw result.error ?? new Error('Invalid document');
1360
- }
1361
-
1362
- document = result.document;
1363
- error = result.error;
1364
- signals.push(...result.signals);
1365
- operationsApplied.push(...result.operationsApplied);
1366
-
1367
- return {
1368
- operations: result.operationsApplied,
1369
- header: result.document,
1370
- newState: document.state
1371
- };
1372
- });
1373
-
1374
- if (document) {
1375
- this.cache.setDocument(drive, id, document).catch(logger.error);
1376
- }
1377
-
1378
- // gets all the different scopes and branches combinations from the operations
1379
- const { scopes, branches } = operationsApplied.reduce(
1380
- (acc, operation) => {
1381
- if (!acc.scopes.includes(operation.scope)) {
1382
- acc.scopes.push(operation.scope);
1383
- }
1384
- return acc;
1385
- },
1386
- { scopes: [] as string[], branches: ['main'] }
1387
- );
1388
-
1389
- const syncUnits = await this.getSynchronizationUnits(
1390
- drive,
1391
- [id],
1392
- scopes,
1393
- branches
1394
- );
1395
- // update listener cache
1396
- this.listenerStateManager
1397
- .updateSynchronizationRevisions(
1398
- drive,
1399
- syncUnits,
1400
- () => {
1401
- this.updateSyncStatus(drive, 'SYNCING');
1402
-
1403
- for (const syncUnit of syncUnits) {
1404
- this.updateSyncStatus(syncUnit.syncId, 'SYNCING');
1405
- }
1406
- },
1407
- this.handleListenerError.bind(this),
1408
- forceSync
1409
- )
1410
- .then(updates => {
1411
- updates.length && this.updateSyncStatus(drive, 'SUCCESS');
1412
-
1413
- for (const syncUnit of syncUnits) {
1414
- this.updateSyncStatus(syncUnit.syncId, 'SUCCESS');
1415
- }
1416
- })
1417
- .catch(error => {
1418
- logger.error(
1419
- 'Non handled error updating sync revision',
1420
- error
1421
- );
1422
- this.updateSyncStatus(drive, 'ERROR', error as Error);
1423
-
1424
- for (const syncUnit of syncUnits) {
1425
- this.updateSyncStatus(
1426
- syncUnit.syncId,
1427
- 'ERROR',
1428
- error as Error
1429
- );
1430
- }
1431
- });
1432
-
1433
- // after applying all the valid operations,throws
1434
- // an error if there was an invalid operation
1435
- if (error) {
1436
- throw error;
1437
- }
1353
+ lastRemainingOperation.index,
1354
+ lastRemainingOperation.scope,
1355
+ "main",
1356
+ )
1357
+ : this.storage.getDriveOperationResultingState?.(
1358
+ drive,
1359
+ lastRemainingOperation.index,
1360
+ lastRemainingOperation.scope,
1361
+ "main",
1362
+ ));
1363
+ }
1438
1364
 
1439
- return {
1440
- status: 'SUCCESS',
1441
- document,
1442
- operations: operationsApplied,
1443
- signals
1444
- } satisfies IOperationResult;
1445
- } catch (error) {
1446
- const operationError =
1447
- error instanceof OperationError
1448
- ? error
1449
- : new OperationError(
1450
- 'ERROR',
1451
- undefined,
1452
- (error as Error).message,
1453
- (error as Error).cause
1454
- );
1455
-
1456
- return {
1457
- status: operationError.status,
1458
- error: operationError,
1459
- document,
1460
- operations: operationsApplied,
1461
- signals
1462
- } satisfies IOperationResult;
1365
+ const operationSignals: (() => Promise<SignalResult>)[] = [];
1366
+ newDocument = documentModel.reducer(
1367
+ newDocument,
1368
+ operation,
1369
+ (signal) => {
1370
+ let handler: (() => Promise<unknown>) | undefined = undefined;
1371
+ switch (signal.type) {
1372
+ case "CREATE_CHILD_DOCUMENT":
1373
+ handler = () => this.createDocument(drive, signal.input);
1374
+ break;
1375
+ case "DELETE_CHILD_DOCUMENT":
1376
+ handler = () => this.deleteDocument(drive, signal.input.id);
1377
+ break;
1378
+ case "COPY_CHILD_DOCUMENT":
1379
+ handler = () =>
1380
+ this.getDocument(drive, signal.input.id).then((documentToCopy) =>
1381
+ this.createDocument(drive, {
1382
+ id: signal.input.newId,
1383
+ documentType: documentToCopy.documentType,
1384
+ document: documentToCopy,
1385
+ synchronizationUnits: signal.input.synchronizationUnits,
1386
+ }),
1387
+ );
1388
+ break;
1463
1389
  }
1390
+ if (handler) {
1391
+ operationSignals.push(() =>
1392
+ handler().then((result) => ({ signal, result })),
1393
+ );
1394
+ }
1395
+ },
1396
+ { skip: operation.skip, reuseOperationResultingState: true },
1397
+ ) as T;
1398
+
1399
+ const appliedOperations = newDocument.operations[operation.scope].filter(
1400
+ (op) => op.index == operation.index && op.skip == operation.skip,
1401
+ );
1402
+ const appliedOperation = appliedOperations.at(0);
1403
+
1404
+ if (!appliedOperation) {
1405
+ throw new OperationError(
1406
+ "ERROR",
1407
+ operation,
1408
+ `Operation with index ${operation.index}:${operation.skip} was not applied.`,
1409
+ );
1464
1410
  }
1465
-
1466
- addDriveOperation(
1467
- drive: string,
1468
- operation: Operation<DocumentDriveAction | BaseAction>,
1469
- forceSync = true
1411
+ if (
1412
+ !appliedOperation.error &&
1413
+ appliedOperation.hash !== operation.hash &&
1414
+ !skipHashValidation
1470
1415
  ) {
1471
- return this.addDriveOperations(drive, [operation], forceSync);
1416
+ throw new ConflictOperationError(operation, appliedOperation);
1472
1417
  }
1473
1418
 
1474
- async clearStorage() {
1475
- for (const drive of await this.getDrives()) {
1476
- await this.deleteDrive(drive);
1477
- }
1478
-
1479
- await this.storage.clearStorage?.();
1419
+ for (const signalHandler of operationSignals) {
1420
+ const result = await signalHandler();
1421
+ signalResults.push(result);
1480
1422
  }
1481
1423
 
1482
- private async _addDriveOperations(
1483
- drive: string,
1484
- callback: (document: DocumentDriveStorage) => Promise<{
1485
- operations: Operation<DocumentDriveAction | BaseAction>[];
1486
- header: DocumentHeader;
1487
- }>
1488
- ) {
1489
- if (!this.storage.addDriveOperationsWithTransaction) {
1490
- const documentStorage = await this.storage.getDrive(drive);
1491
- const result = await callback(documentStorage);
1492
- // saves the applied operations to storage
1493
- if (result.operations.length > 0) {
1494
- await this.storage.addDriveOperations(
1495
- drive,
1496
- result.operations,
1497
- result.header
1498
- );
1499
- }
1500
- return result;
1501
- } else {
1502
- return this.storage.addDriveOperationsWithTransaction(
1503
- drive,
1504
- callback
1505
- );
1506
- }
1424
+ return {
1425
+ document: newDocument,
1426
+ signals: signalResults,
1427
+ operation: appliedOperation,
1428
+ };
1429
+ }
1430
+
1431
+ addOperation(
1432
+ drive: string,
1433
+ id: string,
1434
+ operation: Operation,
1435
+ options?: AddOperationOptions,
1436
+ ): Promise<IOperationResult> {
1437
+ return this.addOperations(drive, id, [operation], options);
1438
+ }
1439
+
1440
+ private async _addOperations(
1441
+ drive: string,
1442
+ id: string,
1443
+ callback: (document: DocumentStorage) => Promise<{
1444
+ operations: Operation[];
1445
+ header: DocumentHeader;
1446
+ }>,
1447
+ ) {
1448
+ if (!this.storage.addDocumentOperationsWithTransaction) {
1449
+ const documentStorage = await this.storage.getDocument(drive, id);
1450
+ const result = await callback(documentStorage);
1451
+ // saves the applied operations to storage
1452
+ if (result.operations.length > 0) {
1453
+ await this.storage.addDocumentOperations(
1454
+ drive,
1455
+ id,
1456
+ result.operations,
1457
+ result.header,
1458
+ );
1459
+ }
1460
+ } else {
1461
+ await this.storage.addDocumentOperationsWithTransaction(
1462
+ drive,
1463
+ id,
1464
+ callback,
1465
+ );
1507
1466
  }
1508
-
1509
- queueDriveOperation(
1510
- drive: string,
1511
- operation: Operation<DocumentDriveAction | BaseAction>,
1512
- forceSync = true
1513
- ): Promise<IOperationResult<DocumentDriveDocument>> {
1514
- return this.queueDriveOperations(drive, [operation], forceSync);
1467
+ }
1468
+
1469
+ queueOperation(
1470
+ drive: string,
1471
+ id: string,
1472
+ operation: Operation,
1473
+ options?: AddOperationOptions,
1474
+ ): Promise<IOperationResult> {
1475
+ return this.queueOperations(drive, id, [operation], options);
1476
+ }
1477
+
1478
+ private async resultIfExistingOperations(
1479
+ drive: string,
1480
+ id: string,
1481
+ operations: Operation[],
1482
+ ): Promise<IOperationResult | undefined> {
1483
+ try {
1484
+ const document = await this.getDocument(drive, id);
1485
+ const newOperation = operations.find(
1486
+ (op) =>
1487
+ !op.id ||
1488
+ !document.operations[op.scope].find(
1489
+ (existingOp) =>
1490
+ existingOp.id === op.id &&
1491
+ existingOp.index === op.index &&
1492
+ existingOp.type === op.type &&
1493
+ existingOp.hash === op.hash,
1494
+ ),
1495
+ );
1496
+ if (!newOperation) {
1497
+ return {
1498
+ status: "SUCCESS",
1499
+ document,
1500
+ operations,
1501
+ signals: [],
1502
+ };
1503
+ } else {
1504
+ return undefined;
1505
+ }
1506
+ } catch (error) {
1507
+ if (
1508
+ !(error as Error).message.includes(`Document with id ${id} not found`)
1509
+ ) {
1510
+ console.error(error);
1511
+ }
1512
+ return undefined;
1515
1513
  }
1516
-
1517
- private async resultIfExistingDriveOperations(
1518
- driveId: string,
1519
- operations: Operation<DocumentDriveAction | BaseAction>[]
1520
- ): Promise<IOperationResult<DocumentDriveDocument> | undefined> {
1521
- try {
1522
- const drive = await this.getDrive(driveId);
1523
- const newOperation = operations.find(
1524
- op =>
1525
- !op.id ||
1526
- !drive.operations[op.scope].find(
1527
- existingOp =>
1528
- existingOp.id === op.id &&
1529
- existingOp.index === op.index &&
1530
- existingOp.type === op.type &&
1531
- existingOp.hash === op.hash
1532
- )
1533
- );
1534
- if (!newOperation) {
1535
- return {
1536
- status: 'SUCCESS',
1537
- document: drive,
1538
- operations: operations,
1539
- signals: []
1540
- } as IOperationResult<DocumentDriveDocument>;
1541
- } else {
1542
- return undefined;
1514
+ }
1515
+
1516
+ async queueOperations(
1517
+ drive: string,
1518
+ id: string,
1519
+ operations: Operation[],
1520
+ options?: AddOperationOptions,
1521
+ ) {
1522
+ // if operations are already stored then returns cached document
1523
+ const result = await this.resultIfExistingOperations(drive, id, operations);
1524
+ if (result) {
1525
+ return result;
1526
+ }
1527
+ try {
1528
+ const jobId = await this.queueManager.addJob({
1529
+ driveId: drive,
1530
+ documentId: id,
1531
+ operations,
1532
+ options,
1533
+ });
1534
+
1535
+ return new Promise<IOperationResult>((resolve, reject) => {
1536
+ const unsubscribe = this.queueManager.on(
1537
+ "jobCompleted",
1538
+ (job, result) => {
1539
+ if (job.jobId === jobId) {
1540
+ unsubscribe();
1541
+ unsubscribeError();
1542
+ resolve(result);
1543
1543
  }
1544
- } catch (error) {
1545
- console.error(error); // TODO error
1546
- return undefined;
1547
- }
1544
+ },
1545
+ );
1546
+ const unsubscribeError = this.queueManager.on(
1547
+ "jobFailed",
1548
+ (job, error) => {
1549
+ if (job.jobId === jobId) {
1550
+ unsubscribe();
1551
+ unsubscribeError();
1552
+ reject(error);
1553
+ }
1554
+ },
1555
+ );
1556
+ });
1557
+ } catch (error) {
1558
+ logger.error("Error adding job", error);
1559
+ throw error;
1548
1560
  }
1549
-
1550
- async queueDriveOperations(
1551
- drive: string,
1552
- operations: Operation<DocumentDriveAction | BaseAction>[],
1553
- forceSync = true
1554
- ): Promise<IOperationResult<DocumentDriveDocument>> {
1555
- // if operations are already stored then returns cached document
1556
- const result = await this.resultIfExistingDriveOperations(
1557
- drive,
1558
- operations
1561
+ }
1562
+
1563
+ async queueAction(
1564
+ drive: string,
1565
+ id: string,
1566
+ action: Action,
1567
+ options?: AddOperationOptions,
1568
+ ): Promise<IOperationResult> {
1569
+ return this.queueActions(drive, id, [action], options);
1570
+ }
1571
+
1572
+ async queueActions(
1573
+ drive: string,
1574
+ id: string,
1575
+ actions: Action[],
1576
+ options?: AddOperationOptions,
1577
+ ): Promise<IOperationResult> {
1578
+ try {
1579
+ const jobId = await this.queueManager.addJob({
1580
+ driveId: drive,
1581
+ documentId: id,
1582
+ actions,
1583
+ options,
1584
+ });
1585
+
1586
+ return new Promise<IOperationResult>((resolve, reject) => {
1587
+ const unsubscribe = this.queueManager.on(
1588
+ "jobCompleted",
1589
+ (job, result) => {
1590
+ if (job.jobId === jobId) {
1591
+ unsubscribe();
1592
+ unsubscribeError();
1593
+ resolve(result);
1594
+ }
1595
+ },
1559
1596
  );
1560
- if (result) {
1561
- return result;
1562
- }
1563
-
1564
- const jobId = await this.queueManager.addJob({
1565
- driveId: drive,
1566
- operations,
1567
- forceSync
1568
- });
1569
- return new Promise<IOperationResult<DocumentDriveDocument>>(
1570
- (resolve, reject) => {
1571
- const unsubscribe = this.queueManager.on(
1572
- 'jobCompleted',
1573
- (job, result) => {
1574
- if (job.jobId === jobId) {
1575
- unsubscribe();
1576
- unsubscribeError();
1577
- resolve(
1578
- result as IOperationResult<DocumentDriveDocument>
1579
- );
1580
- }
1581
- }
1582
- );
1583
- const unsubscribeError = this.queueManager.on(
1584
- 'jobFailed',
1585
- (job, error) => {
1586
- if (job.jobId === jobId) {
1587
- unsubscribe();
1588
- unsubscribeError();
1589
- reject(error);
1590
- }
1591
- }
1592
- );
1597
+ const unsubscribeError = this.queueManager.on(
1598
+ "jobFailed",
1599
+ (job, error) => {
1600
+ if (job.jobId === jobId) {
1601
+ unsubscribe();
1602
+ unsubscribeError();
1603
+ reject(error);
1593
1604
  }
1605
+ },
1594
1606
  );
1607
+ });
1608
+ } catch (error) {
1609
+ logger.error("Error adding job", error);
1610
+ throw error;
1595
1611
  }
1596
-
1597
- async addDriveOperations(
1598
- drive: string,
1599
- operations: Operation<DocumentDriveAction | BaseAction>[],
1600
- forceSync = true
1601
- ) {
1602
- let document: DocumentDriveDocument | undefined;
1603
- const operationsApplied: Operation<DocumentDriveAction | BaseAction>[] =
1604
- [];
1605
- const signals: SignalResult[] = [];
1606
- let error: Error | undefined;
1607
-
1608
- // if operations are already stored then returns cached drive
1609
- const result = await this.resultIfExistingDriveOperations(
1610
- drive,
1611
- operations
1612
+ }
1613
+
1614
+ async queueDriveAction(
1615
+ drive: string,
1616
+ action: DocumentDriveAction | BaseAction,
1617
+ options?: AddOperationOptions,
1618
+ ): Promise<IOperationResult<DocumentDriveDocument>> {
1619
+ return this.queueDriveActions(drive, [action], options);
1620
+ }
1621
+
1622
+ async queueDriveActions(
1623
+ drive: string,
1624
+ actions: (DocumentDriveAction | BaseAction)[],
1625
+ options?: AddOperationOptions,
1626
+ ): Promise<IOperationResult<DocumentDriveDocument>> {
1627
+ try {
1628
+ const jobId = await this.queueManager.addJob({
1629
+ driveId: drive,
1630
+ actions,
1631
+ options,
1632
+ });
1633
+ return new Promise<IOperationResult<DocumentDriveDocument>>(
1634
+ (resolve, reject) => {
1635
+ const unsubscribe = this.queueManager.on(
1636
+ "jobCompleted",
1637
+ (job, result) => {
1638
+ if (job.jobId === jobId) {
1639
+ unsubscribe();
1640
+ unsubscribeError();
1641
+ resolve(result as IOperationResult<DocumentDriveDocument>);
1642
+ }
1643
+ },
1644
+ );
1645
+ const unsubscribeError = this.queueManager.on(
1646
+ "jobFailed",
1647
+ (job, error) => {
1648
+ if (job.jobId === jobId) {
1649
+ unsubscribe();
1650
+ unsubscribeError();
1651
+ reject(error);
1652
+ }
1653
+ },
1654
+ );
1655
+ },
1656
+ );
1657
+ } catch (error) {
1658
+ logger.error("Error adding drive job", error);
1659
+ throw error;
1660
+ }
1661
+ }
1662
+
1663
+ async addOperations(
1664
+ drive: string,
1665
+ id: string,
1666
+ operations: Operation[],
1667
+ options?: AddOperationOptions,
1668
+ ) {
1669
+ // if operations are already stored then returns the result
1670
+ const result = await this.resultIfExistingOperations(drive, id, operations);
1671
+ if (result) {
1672
+ return result;
1673
+ }
1674
+ let document: Document | undefined;
1675
+ const operationsApplied: Operation[] = [];
1676
+ const signals: SignalResult[] = [];
1677
+ let error: Error | undefined;
1678
+
1679
+ try {
1680
+ await this._addOperations(drive, id, async (documentStorage) => {
1681
+ const result = await this._processOperations(
1682
+ drive,
1683
+ id,
1684
+ documentStorage,
1685
+ operations,
1612
1686
  );
1613
- if (result) {
1614
- return result;
1615
- }
1616
1687
 
1617
- const prevSyncUnits = await this.getSynchronizationUnitsIds(drive);
1688
+ if (!result.document) {
1689
+ logger.error("Invalid document");
1690
+ throw result.error ?? new Error("Invalid document");
1691
+ }
1618
1692
 
1619
- try {
1620
- await this._addDriveOperations(drive, async documentStorage => {
1621
- const result = await this._processOperations<
1622
- DocumentDriveDocument,
1623
- DocumentDriveAction
1624
- >(drive, undefined, documentStorage, operations.slice());
1625
-
1626
- document = result.document;
1627
- operationsApplied.push(...result.operationsApplied);
1628
- signals.push(...result.signals);
1629
- error = result.error;
1630
-
1631
- return {
1632
- operations: result.operationsApplied,
1633
- header: result.document
1634
- };
1635
- });
1693
+ document = result.document;
1694
+ error = result.error;
1695
+ signals.push(...result.signals);
1696
+ operationsApplied.push(...result.operationsApplied);
1636
1697
 
1637
- if (!document || !isDocumentDrive(document)) {
1638
- throw error ?? new Error('Invalid Document Drive document');
1639
- }
1698
+ return {
1699
+ operations: result.operationsApplied,
1700
+ header: result.document,
1701
+ newState: document.state,
1702
+ };
1703
+ });
1640
1704
 
1641
- this.cache
1642
- .setDocument('drives', drive, document)
1643
- .catch(logger.error);
1705
+ if (document) {
1706
+ this.cache.setDocument(drive, id, document).catch(logger.error);
1707
+ }
1708
+
1709
+ // gets all the different scopes and branches combinations from the operations
1710
+ const { scopes, branches } = operationsApplied.reduce(
1711
+ (acc, operation) => {
1712
+ if (!acc.scopes.includes(operation.scope)) {
1713
+ acc.scopes.push(operation.scope);
1714
+ }
1715
+ return acc;
1716
+ },
1717
+ { scopes: [] as string[], branches: ["main"] },
1718
+ );
1719
+
1720
+ const syncUnits = await this.getSynchronizationUnits(
1721
+ drive,
1722
+ [id],
1723
+ scopes,
1724
+ branches,
1725
+ );
1726
+
1727
+ // checks if any of the provided operations where reshufled
1728
+ const newOp = operationsApplied.find(
1729
+ (appliedOp) =>
1730
+ !operations.find(
1731
+ (o) =>
1732
+ o.id === appliedOp.id &&
1733
+ o.index === appliedOp.index &&
1734
+ o.skip === appliedOp.skip &&
1735
+ o.hash === appliedOp.hash,
1736
+ ),
1737
+ );
1738
+
1739
+ // if there are no new operations then reuses the provided source
1740
+ // otherwise sets it to local so listeners know that there were
1741
+ // new changes originating from this document drive server
1742
+ const source: StrandUpdateSource = newOp
1743
+ ? { type: "local" }
1744
+ : (options?.source ?? { type: "local" });
1745
+
1746
+ // update listener cache
1747
+
1748
+ const operationSource = this.getOperationSource(source);
1749
+
1750
+ this.listenerStateManager
1751
+ .updateSynchronizationRevisions(
1752
+ drive,
1753
+ syncUnits,
1754
+ source,
1755
+ () => {
1756
+ this.updateSyncUnitStatus(drive, {
1757
+ [operationSource]: "SYNCING",
1758
+ });
1644
1759
 
1645
- for (const operation of operationsApplied) {
1646
- switch (operation.type) {
1647
- case 'ADD_LISTENER': {
1648
- await this.addListener(drive, operation);
1649
- break;
1650
- }
1651
- case 'REMOVE_LISTENER': {
1652
- await this.removeListener(drive, operation);
1653
- break;
1654
- }
1655
- }
1760
+ for (const syncUnit of syncUnits) {
1761
+ this.updateSyncUnitStatus(syncUnit.syncId, {
1762
+ [operationSource]: "SYNCING",
1763
+ });
1656
1764
  }
1765
+ },
1766
+ this.handleListenerError.bind(this),
1767
+ options?.forceSync ?? source.type === "local",
1768
+ )
1769
+ .then((updates) => {
1770
+ if (updates.length) {
1771
+ this.updateSyncUnitStatus(drive, {
1772
+ [operationSource]: "SUCCESS",
1773
+ });
1774
+ }
1657
1775
 
1658
- const syncUnits = await this.getSynchronizationUnitsIds(drive);
1659
-
1660
- const prevSyncUnitsIds = prevSyncUnits.map(unit => unit.syncId);
1661
- const syncUnitsIds = syncUnits.map(unit => unit.syncId);
1662
-
1663
- const newSyncUnits = syncUnitsIds.filter(
1664
- syncUnitId => !prevSyncUnitsIds.includes(syncUnitId)
1776
+ for (const syncUnit of syncUnits) {
1777
+ this.updateSyncUnitStatus(syncUnit.syncId, {
1778
+ [operationSource]: "SUCCESS",
1779
+ });
1780
+ }
1781
+ })
1782
+ .catch((error) => {
1783
+ logger.error("Non handled error updating sync revision", error);
1784
+ this.updateSyncUnitStatus(
1785
+ drive,
1786
+ {
1787
+ [operationSource]: "ERROR",
1788
+ },
1789
+ error as Error,
1790
+ );
1791
+
1792
+ for (const syncUnit of syncUnits) {
1793
+ this.updateSyncUnitStatus(
1794
+ syncUnit.syncId,
1795
+ {
1796
+ [operationSource]: "ERROR",
1797
+ },
1798
+ error as Error,
1665
1799
  );
1800
+ }
1801
+ });
1666
1802
 
1667
- const removedSyncUnits = prevSyncUnitsIds.filter(
1668
- syncUnitId => !syncUnitsIds.includes(syncUnitId)
1803
+ // after applying all the valid operations,throws
1804
+ // an error if there was an invalid operation
1805
+ if (error) {
1806
+ throw error;
1807
+ }
1808
+
1809
+ return {
1810
+ status: "SUCCESS",
1811
+ document,
1812
+ operations: operationsApplied,
1813
+ signals,
1814
+ } satisfies IOperationResult;
1815
+ } catch (error) {
1816
+ const operationError =
1817
+ error instanceof OperationError
1818
+ ? error
1819
+ : new OperationError(
1820
+ "ERROR",
1821
+ undefined,
1822
+ (error as Error).message,
1823
+ (error as Error).cause,
1669
1824
  );
1670
1825
 
1671
- // update listener cache
1672
- const lastOperation = operationsApplied
1673
- .filter(op => op.scope === 'global')
1674
- .slice()
1675
- .pop();
1676
- if (lastOperation) {
1677
- this.listenerStateManager
1678
- .updateSynchronizationRevisions(
1679
- drive,
1680
- [
1681
- {
1682
- syncId: '0',
1683
- driveId: drive,
1684
- documentId: '',
1685
- scope: 'global',
1686
- branch: 'main',
1687
- documentType: 'powerhouse/document-drive',
1688
- lastUpdated: lastOperation.timestamp,
1689
- revision: lastOperation.index
1690
- }
1691
- ],
1692
- () => {
1693
- this.updateSyncStatus(drive, 'SYNCING');
1694
-
1695
- for (const syncUnitId of [
1696
- ...newSyncUnits,
1697
- ...removedSyncUnits
1698
- ]) {
1699
- this.updateSyncStatus(syncUnitId, 'SYNCING');
1700
- }
1701
- },
1702
- this.handleListenerError.bind(this),
1703
- forceSync
1704
- )
1705
- .then(updates => {
1706
- if (updates.length) {
1707
- this.updateSyncStatus(drive, 'SUCCESS');
1708
-
1709
- for (const syncUnitId of newSyncUnits) {
1710
- this.updateSyncStatus(syncUnitId, 'SUCCESS');
1711
- }
1712
-
1713
- for (const syncUnitId of removedSyncUnits) {
1714
- this.updateSyncStatus(syncUnitId, null);
1715
- }
1716
- }
1717
- })
1718
- .catch(error => {
1719
- logger.error(
1720
- 'Non handled error updating sync revision',
1721
- error
1722
- );
1723
- this.updateSyncStatus(drive, 'ERROR', error as Error);
1724
-
1725
- for (const syncUnitId of [
1726
- ...newSyncUnits,
1727
- ...removedSyncUnits
1728
- ]) {
1729
- this.updateSyncStatus(
1730
- syncUnitId,
1731
- 'ERROR',
1732
- error as Error
1733
- );
1734
- }
1735
- });
1736
- }
1737
-
1738
- if (this.shouldSyncRemoteDrive(document)) {
1739
- this.startSyncRemoteDrive(document.state.global.id);
1740
- } else {
1741
- this.stopSyncRemoteDrive(document.state.global.id);
1742
- }
1743
-
1744
- // after applying all the valid operations,throws
1745
- // an error if there was an invalid operation
1746
- if (error) {
1747
- throw error;
1748
- }
1749
-
1750
- return {
1751
- status: 'SUCCESS',
1752
- document,
1753
- operations: operationsApplied,
1754
- signals
1755
- } satisfies IOperationResult;
1756
- } catch (error) {
1757
- const operationError =
1758
- error instanceof OperationError
1759
- ? error
1760
- : new OperationError(
1761
- 'ERROR',
1762
- undefined,
1763
- (error as Error).message,
1764
- (error as Error).cause
1765
- );
1766
-
1767
- return {
1768
- status: operationError.status,
1769
- error: operationError,
1770
- document,
1771
- operations: operationsApplied,
1772
- signals
1773
- } satisfies IOperationResult;
1774
- }
1826
+ return {
1827
+ status: operationError.status,
1828
+ error: operationError,
1829
+ document,
1830
+ operations: operationsApplied,
1831
+ signals,
1832
+ } satisfies IOperationResult;
1775
1833
  }
1776
-
1777
- private _buildOperations<T extends Action>(
1778
- document: Document,
1779
- actions: (T | BaseAction)[]
1780
- ): Operation<T | BaseAction>[] {
1781
- const operations: Operation<T | BaseAction>[] = [];
1782
- const { reducer } = this._getDocumentModel(document.documentType);
1783
- for (const action of actions) {
1784
- document = reducer(document, action);
1785
- const operation = document.operations[action.scope].slice().pop();
1786
- if (!operation) {
1787
- throw new Error('Error creating operations');
1788
- }
1789
- operations.push(operation);
1790
- }
1791
- return operations;
1834
+ }
1835
+
1836
+ addDriveOperation(
1837
+ drive: string,
1838
+ operation: Operation<DocumentDriveAction | BaseAction>,
1839
+ options?: AddOperationOptions,
1840
+ ) {
1841
+ return this.addDriveOperations(drive, [operation], options);
1842
+ }
1843
+
1844
+ async clearStorage() {
1845
+ for (const drive of await this.getDrives()) {
1846
+ await this.deleteDrive(drive);
1792
1847
  }
1793
1848
 
1794
- async addAction(
1795
- drive: string,
1796
- id: string,
1797
- action: Action,
1798
- forceSync = true
1799
- ): Promise<IOperationResult> {
1800
- return this.addActions(drive, id, [action], forceSync);
1849
+ await this.storage.clearStorage?.();
1850
+ }
1851
+
1852
+ private async _addDriveOperations(
1853
+ drive: string,
1854
+ callback: (document: DocumentDriveStorage) => Promise<{
1855
+ operations: Operation<DocumentDriveAction | BaseAction>[];
1856
+ header: DocumentHeader;
1857
+ }>,
1858
+ ) {
1859
+ if (!this.storage.addDriveOperationsWithTransaction) {
1860
+ const documentStorage = await this.storage.getDrive(drive);
1861
+ const result = await callback(documentStorage);
1862
+ // saves the applied operations to storage
1863
+ if (result.operations.length > 0) {
1864
+ await this.storage.addDriveOperations(
1865
+ drive,
1866
+ result.operations,
1867
+ result.header,
1868
+ );
1869
+ }
1870
+ return result;
1871
+ } else {
1872
+ return this.storage.addDriveOperationsWithTransaction(drive, callback);
1801
1873
  }
1802
-
1803
- async addActions(
1804
- drive: string,
1805
- id: string,
1806
- actions: Action[],
1807
- forceSync = true
1808
- ): Promise<IOperationResult> {
1809
- const document = await this.getDocument(drive, id);
1810
- const operations = this._buildOperations(document, actions);
1811
- return this.addOperations(drive, id, operations, forceSync);
1874
+ }
1875
+
1876
+ queueDriveOperation(
1877
+ drive: string,
1878
+ operation: Operation<DocumentDriveAction | BaseAction>,
1879
+ options?: AddOperationOptions,
1880
+ ): Promise<IOperationResult<DocumentDriveDocument>> {
1881
+ return this.queueDriveOperations(drive, [operation], options);
1882
+ }
1883
+
1884
+ private async resultIfExistingDriveOperations(
1885
+ driveId: string,
1886
+ operations: Operation<DocumentDriveAction | BaseAction>[],
1887
+ ): Promise<IOperationResult<DocumentDriveDocument> | undefined> {
1888
+ try {
1889
+ const drive = await this.getDrive(driveId);
1890
+ const newOperation = operations.find(
1891
+ (op) =>
1892
+ !op.id ||
1893
+ !drive.operations[op.scope].find(
1894
+ (existingOp) =>
1895
+ existingOp.id === op.id &&
1896
+ existingOp.index === op.index &&
1897
+ existingOp.type === op.type &&
1898
+ existingOp.hash === op.hash,
1899
+ ),
1900
+ );
1901
+ if (!newOperation) {
1902
+ return {
1903
+ status: "SUCCESS",
1904
+ document: drive,
1905
+ operations: operations,
1906
+ signals: [],
1907
+ } as IOperationResult<DocumentDriveDocument>;
1908
+ } else {
1909
+ return undefined;
1910
+ }
1911
+ } catch (error) {
1912
+ console.error(error); // TODO error
1913
+ return undefined;
1812
1914
  }
1813
-
1814
- async addDriveAction(
1815
- drive: string,
1816
- action: DocumentDriveAction | BaseAction,
1817
- forceSync = true
1818
- ): Promise<IOperationResult<DocumentDriveDocument>> {
1819
- return this.addDriveActions(drive, [action], forceSync);
1915
+ }
1916
+
1917
+ async queueDriveOperations(
1918
+ drive: string,
1919
+ operations: Operation<DocumentDriveAction | BaseAction>[],
1920
+ options?: AddOperationOptions,
1921
+ ): Promise<IOperationResult<DocumentDriveDocument>> {
1922
+ // if operations are already stored then returns cached document
1923
+ const result = await this.resultIfExistingDriveOperations(
1924
+ drive,
1925
+ operations,
1926
+ );
1927
+ if (result) {
1928
+ return result;
1820
1929
  }
1821
-
1822
- async addDriveActions(
1823
- drive: string,
1824
- actions: (DocumentDriveAction | BaseAction)[],
1825
- forceSync = true
1826
- ): Promise<IOperationResult<DocumentDriveDocument>> {
1827
- const document = await this.getDrive(drive);
1828
- const operations = this._buildOperations(document, actions);
1829
- const result = await this.addDriveOperations(
1830
- drive,
1831
- operations,
1832
- forceSync
1833
- );
1834
- return result;
1930
+ try {
1931
+ const jobId = await this.queueManager.addJob({
1932
+ driveId: drive,
1933
+ operations,
1934
+ options,
1935
+ });
1936
+ return new Promise<IOperationResult<DocumentDriveDocument>>(
1937
+ (resolve, reject) => {
1938
+ const unsubscribe = this.queueManager.on(
1939
+ "jobCompleted",
1940
+ (job, result) => {
1941
+ if (job.jobId === jobId) {
1942
+ unsubscribe();
1943
+ unsubscribeError();
1944
+ resolve(result as IOperationResult<DocumentDriveDocument>);
1945
+ }
1946
+ },
1947
+ );
1948
+ const unsubscribeError = this.queueManager.on(
1949
+ "jobFailed",
1950
+ (job, error) => {
1951
+ if (job.jobId === jobId) {
1952
+ unsubscribe();
1953
+ unsubscribeError();
1954
+ reject(error);
1955
+ }
1956
+ },
1957
+ );
1958
+ },
1959
+ );
1960
+ } catch (error) {
1961
+ logger.error("Error adding drive job", error);
1962
+ throw error;
1963
+ }
1964
+ }
1965
+
1966
+ async addDriveOperations(
1967
+ drive: string,
1968
+ operations: Operation<DocumentDriveAction | BaseAction>[],
1969
+ options?: AddOperationOptions,
1970
+ ) {
1971
+ let document: DocumentDriveDocument | undefined;
1972
+ const operationsApplied: Operation<DocumentDriveAction | BaseAction>[] = [];
1973
+ const signals: SignalResult[] = [];
1974
+ let error: Error | undefined;
1975
+
1976
+ // if operations are already stored then returns cached drive
1977
+ const result = await this.resultIfExistingDriveOperations(
1978
+ drive,
1979
+ operations,
1980
+ );
1981
+ if (result) {
1982
+ return result;
1835
1983
  }
1836
1984
 
1837
- async addInternalListener(
1838
- driveId: string,
1839
- receiver: IReceiver,
1840
- options: {
1841
- listenerId: string;
1842
- label: string;
1843
- block: boolean;
1844
- filter: ListenerFilter;
1845
- }
1846
- ) {
1847
- const listener: AddListenerInput['listener'] = {
1848
- callInfo: {
1849
- data: '',
1850
- name: 'Interal',
1851
- transmitterType: 'Internal'
1852
- },
1853
- system: true,
1854
- ...options
1985
+ try {
1986
+ await this._addDriveOperations(drive, async (documentStorage) => {
1987
+ const result = await this._processOperations<
1988
+ DocumentDriveDocument,
1989
+ DocumentDriveAction
1990
+ >(drive, undefined, documentStorage, operations.slice());
1991
+ document = result.document;
1992
+ operationsApplied.push(...result.operationsApplied);
1993
+ signals.push(...result.signals);
1994
+ error = result.error;
1995
+
1996
+ return {
1997
+ operations: result.operationsApplied,
1998
+ header: result.document,
1855
1999
  };
1856
- await this.addDriveAction(driveId, actions.addListener({ listener }));
1857
- const transmitter = await this.getTransmitter(
1858
- driveId,
1859
- options.listenerId
1860
- );
1861
- if (!transmitter) {
1862
- logger.error('Internal listener not found');
1863
- throw new Error('Internal listener not found');
1864
- }
1865
- if (!(transmitter instanceof InternalTransmitter)) {
1866
- logger.error('Listener is not an internal transmitter');
1867
- throw new Error('Listener is not an internal transmitter');
2000
+ });
2001
+
2002
+ if (!document || !isDocumentDrive(document)) {
2003
+ throw error ?? new Error("Invalid Document Drive document");
2004
+ }
2005
+
2006
+ this.cache.setDocument("drives", drive, document).catch(logger.error);
2007
+
2008
+ for (const operation of operationsApplied) {
2009
+ switch (operation.type) {
2010
+ case "ADD_LISTENER": {
2011
+ await this.addListener(drive, operation);
2012
+ break;
2013
+ }
2014
+ case "REMOVE_LISTENER": {
2015
+ await this.removeListener(drive, operation);
2016
+ break;
2017
+ }
1868
2018
  }
2019
+ }
2020
+
2021
+ // update listener cache
2022
+ const lastOperation = operationsApplied
2023
+ .filter((op) => op.scope === "global")
2024
+ .slice()
2025
+ .pop();
2026
+
2027
+ if (lastOperation) {
2028
+ // checks if any of the provided operations where reshufled
2029
+ const newOp = operationsApplied.find(
2030
+ (appliedOp) =>
2031
+ !operations.find(
2032
+ (o) =>
2033
+ o.id === appliedOp.id &&
2034
+ o.index === appliedOp.index &&
2035
+ o.skip === appliedOp.skip &&
2036
+ o.hash === appliedOp.hash,
2037
+ ),
2038
+ );
1869
2039
 
1870
- transmitter.setReceiver(receiver);
1871
- return transmitter;
1872
- }
2040
+ // if there are no new operations then reuses the provided source
2041
+ // otherwise sets it to local so listeners know that there were
2042
+ // new changes originating from this document drive server
2043
+ const source: StrandUpdateSource = newOp
2044
+ ? { type: "local" }
2045
+ : (options?.source ?? { type: "local" });
1873
2046
 
1874
- private async addListener(
1875
- driveId: string,
1876
- operation: Operation<Action<'ADD_LISTENER', AddListenerInput>>
1877
- ) {
1878
- const { listener } = operation.input;
1879
- await this.listenerStateManager.addListener({
1880
- ...listener,
1881
- driveId,
1882
- label: listener.label ?? '',
1883
- system: listener.system ?? false,
1884
- filter: {
1885
- branch: listener.filter.branch ?? [],
1886
- documentId: listener.filter.documentId ?? [],
1887
- documentType: listener.filter.documentType ?? [],
1888
- scope: listener.filter.scope ?? []
2047
+ const operationSource = this.getOperationSource(source);
2048
+
2049
+ this.listenerStateManager
2050
+ .updateSynchronizationRevisions(
2051
+ drive,
2052
+ [
2053
+ {
2054
+ syncId: "0",
2055
+ driveId: drive,
2056
+ documentId: "",
2057
+ scope: "global",
2058
+ branch: "main",
2059
+ documentType: "powerhouse/document-drive",
2060
+ lastUpdated: lastOperation.timestamp,
2061
+ revision: lastOperation.index,
2062
+ },
2063
+ ],
2064
+ source,
2065
+ () => {
2066
+ this.updateSyncUnitStatus(drive, {
2067
+ [operationSource]: "SYNCING",
2068
+ });
1889
2069
  },
1890
- callInfo: {
1891
- data: listener.callInfo?.data ?? '',
1892
- name: listener.callInfo?.name ?? 'PullResponder',
1893
- transmitterType:
1894
- listener.callInfo?.transmitterType ?? 'PullResponder'
2070
+ this.handleListenerError.bind(this),
2071
+ options?.forceSync ?? source.type === "local",
2072
+ )
2073
+ .then((updates) => {
2074
+ if (updates.length) {
2075
+ this.updateSyncUnitStatus(drive, {
2076
+ [operationSource]: "SUCCESS",
2077
+ });
1895
2078
  }
1896
- });
1897
- }
2079
+ })
2080
+ .catch((error) => {
2081
+ logger.error("Non handled error updating sync revision", error);
2082
+ this.updateSyncUnitStatus(
2083
+ drive,
2084
+ { [operationSource]: "ERROR" },
2085
+ error as Error,
2086
+ );
2087
+ });
2088
+ }
2089
+
2090
+ if (this.shouldSyncRemoteDrive(document)) {
2091
+ this.startSyncRemoteDrive(document.state.global.id);
2092
+ } else {
2093
+ this.stopSyncRemoteDrive(document.state.global.id);
2094
+ }
2095
+
2096
+ // after applying all the valid operations,throws
2097
+ // an error if there was an invalid operation
2098
+ if (error) {
2099
+ throw error;
2100
+ }
2101
+
2102
+ return {
2103
+ status: "SUCCESS",
2104
+ document,
2105
+ operations: operationsApplied,
2106
+ signals,
2107
+ } satisfies IOperationResult;
2108
+ } catch (error) {
2109
+ const operationError =
2110
+ error instanceof OperationError
2111
+ ? error
2112
+ : new OperationError(
2113
+ "ERROR",
2114
+ undefined,
2115
+ (error as Error).message,
2116
+ (error as Error).cause,
2117
+ );
1898
2118
 
1899
- private async removeListener(
1900
- driveId: string,
1901
- operation: Operation<Action<'REMOVE_LISTENER', RemoveListenerInput>>
1902
- ) {
1903
- const { listenerId } = operation.input;
1904
- await this.listenerStateManager.removeListener(driveId, listenerId);
2119
+ return {
2120
+ status: operationError.status,
2121
+ error: operationError,
2122
+ document,
2123
+ operations: operationsApplied,
2124
+ signals,
2125
+ } satisfies IOperationResult;
1905
2126
  }
1906
-
1907
- getTransmitter(
1908
- driveId: string,
1909
- listenerId: string
1910
- ): Promise<ITransmitter | undefined> {
1911
- return this.listenerStateManager.getTransmitter(driveId, listenerId);
2127
+ }
2128
+
2129
+ private _buildOperations<T extends Action>(
2130
+ document: Document,
2131
+ actions: (T | BaseAction)[],
2132
+ ): Operation<T | BaseAction>[] {
2133
+ const operations: Operation<T | BaseAction>[] = [];
2134
+ const { reducer } = this.getDocumentModel(document.documentType);
2135
+ for (const action of actions) {
2136
+ document = reducer(document, action);
2137
+ const operation = document.operations[action.scope].slice().pop();
2138
+ if (!operation) {
2139
+ throw new Error("Error creating operations");
2140
+ }
2141
+ operations.push(operation);
1912
2142
  }
1913
-
1914
- getListener(
1915
- driveId: string,
1916
- listenerId: string
1917
- ): Promise<ListenerState | undefined> {
1918
- return this.listenerStateManager.getListener(driveId, listenerId);
2143
+ return operations;
2144
+ }
2145
+
2146
+ async addAction(
2147
+ drive: string,
2148
+ id: string,
2149
+ action: Action,
2150
+ options?: AddOperationOptions,
2151
+ ): Promise<IOperationResult> {
2152
+ return this.addActions(drive, id, [action], options);
2153
+ }
2154
+
2155
+ async addActions(
2156
+ drive: string,
2157
+ id: string,
2158
+ actions: Action[],
2159
+ options?: AddOperationOptions,
2160
+ ): Promise<IOperationResult> {
2161
+ const document = await this.getDocument(drive, id);
2162
+ const operations = this._buildOperations(document, actions);
2163
+ return this.addOperations(drive, id, operations, options);
2164
+ }
2165
+
2166
+ async addDriveAction(
2167
+ drive: string,
2168
+ action: DocumentDriveAction | BaseAction,
2169
+ options?: AddOperationOptions,
2170
+ ): Promise<IOperationResult<DocumentDriveDocument>> {
2171
+ return this.addDriveActions(drive, [action], options);
2172
+ }
2173
+
2174
+ async addDriveActions(
2175
+ drive: string,
2176
+ actions: (DocumentDriveAction | BaseAction)[],
2177
+ options?: AddOperationOptions,
2178
+ ): Promise<IOperationResult<DocumentDriveDocument>> {
2179
+ const document = await this.getDrive(drive);
2180
+ const operations = this._buildOperations(document, actions);
2181
+ const result = await this.addDriveOperations(drive, operations, options);
2182
+ return result;
2183
+ }
2184
+
2185
+ async addInternalListener(
2186
+ driveId: string,
2187
+ receiver: IReceiver,
2188
+ options: {
2189
+ listenerId: string;
2190
+ label: string;
2191
+ block: boolean;
2192
+ filter: ListenerFilter;
2193
+ },
2194
+ ) {
2195
+ const listener: AddListenerInput["listener"] = {
2196
+ callInfo: {
2197
+ data: "",
2198
+ name: "Interal",
2199
+ transmitterType: "Internal",
2200
+ },
2201
+ system: true,
2202
+ ...options,
2203
+ };
2204
+ await this.addDriveAction(driveId, actions.addListener({ listener }));
2205
+ const transmitter = await this.getTransmitter(driveId, options.listenerId);
2206
+ if (!transmitter) {
2207
+ logger.error("Internal listener not found");
2208
+ throw new Error("Internal listener not found");
2209
+ }
2210
+ if (!(transmitter instanceof InternalTransmitter)) {
2211
+ logger.error("Listener is not an internal transmitter");
2212
+ throw new Error("Listener is not an internal transmitter");
1919
2213
  }
1920
2214
 
1921
- getSyncStatus(drive: string): SyncStatus {
1922
- const status = this.syncStatus.get(drive);
1923
- if (!status) {
1924
- logger.error(`Sync status not found for drive ${drive}`);
1925
- throw new Error(`Sync status not found for drive ${drive}`);
1926
- }
1927
- return status;
2215
+ transmitter.setReceiver(receiver);
2216
+ return transmitter;
2217
+ }
2218
+
2219
+ async detachDrive(driveId: string) {
2220
+ const documentDrive = await this.getDrive(driveId);
2221
+ const listeners = documentDrive.state.local.listeners || [];
2222
+ const triggers = documentDrive.state.local.triggers || [];
2223
+
2224
+ for (const listener of listeners) {
2225
+ await this.addDriveAction(
2226
+ driveId,
2227
+ actions.removeListener({ listenerId: listener.listenerId }),
2228
+ );
1928
2229
  }
1929
2230
 
1930
- on<K extends keyof DriveEvents>(event: K, cb: DriveEvents[K]): Unsubscribe {
1931
- return this.emitter.on(event, cb);
2231
+ for (const trigger of triggers) {
2232
+ await this.addDriveAction(
2233
+ driveId,
2234
+ actions.removeTrigger({ triggerId: trigger.id }),
2235
+ );
1932
2236
  }
1933
2237
 
1934
- protected emit<K extends keyof DriveEvents>(
1935
- event: K,
1936
- ...args: Parameters<DriveEvents[K]>
1937
- ): void {
1938
- logger.debug(`Emitting event ${event}`, args);
1939
- return this.emitter.emit(event, ...args);
2238
+ await this.addDriveAction(
2239
+ driveId,
2240
+ actions.setSharingType({ type: "LOCAL" }),
2241
+ );
2242
+ }
2243
+
2244
+ private async addListener(
2245
+ driveId: string,
2246
+ operation: Operation<Action<"ADD_LISTENER", AddListenerInput>>,
2247
+ ) {
2248
+ const { listener } = operation.input;
2249
+ await this.listenerStateManager.addListener({
2250
+ ...listener,
2251
+ driveId,
2252
+ label: listener.label ?? "",
2253
+ system: listener.system ?? false,
2254
+ filter: {
2255
+ branch: listener.filter.branch ?? [],
2256
+ documentId: listener.filter.documentId ?? [],
2257
+ documentType: listener.filter.documentType ?? [],
2258
+ scope: listener.filter.scope ?? [],
2259
+ },
2260
+ callInfo: {
2261
+ data: listener.callInfo?.data ?? "",
2262
+ name: listener.callInfo?.name ?? "PullResponder",
2263
+ transmitterType: listener.callInfo?.transmitterType ?? "PullResponder",
2264
+ },
2265
+ });
2266
+ }
2267
+
2268
+ private async removeListener(
2269
+ driveId: string,
2270
+ operation: Operation<Action<"REMOVE_LISTENER", RemoveListenerInput>>,
2271
+ ) {
2272
+ const { listenerId } = operation.input;
2273
+ await this.listenerStateManager.removeListener(driveId, listenerId);
2274
+ }
2275
+
2276
+ getTransmitter(
2277
+ driveId: string,
2278
+ listenerId: string,
2279
+ ): Promise<ITransmitter | undefined> {
2280
+ return this.listenerStateManager.getTransmitter(driveId, listenerId);
2281
+ }
2282
+
2283
+ getListener(
2284
+ driveId: string,
2285
+ listenerId: string,
2286
+ ): Promise<ListenerState | undefined> {
2287
+ return this.listenerStateManager.getListener(driveId, listenerId);
2288
+ }
2289
+
2290
+ getSyncStatus(
2291
+ syncUnitId: string,
2292
+ ): SyncStatus | SynchronizationUnitNotFoundError {
2293
+ const status = this.syncStatus.get(syncUnitId);
2294
+ if (!status) {
2295
+ return new SynchronizationUnitNotFoundError(
2296
+ `Sync status not found for syncUnitId: ${syncUnitId}`,
2297
+ syncUnitId,
2298
+ );
1940
2299
  }
2300
+ return this.getCombinedSyncUnitStatus(status);
2301
+ }
2302
+
2303
+ on<K extends keyof DriveEvents>(event: K, cb: DriveEvents[K]): Unsubscribe {
2304
+ return this.emitter.on(event, cb);
2305
+ }
2306
+
2307
+ protected emit<K extends keyof DriveEvents>(
2308
+ event: K,
2309
+ ...args: Parameters<DriveEvents[K]>
2310
+ ): void {
2311
+ logger.debug(`Emitting event ${event}`, args);
2312
+ return this.emitter.emit(event, ...args);
2313
+ }
1941
2314
  }
2315
+
2316
+ export const DocumentDriveServer = ReadModeServer(BaseDocumentDriveServer);