document-drive 1.0.0-experimental.8 → 1.0.0-websockets
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +148 -0
- package/package.json +15 -10
- package/src/queue/base.ts +19 -34
- package/src/queue/redis.ts +15 -25
- package/src/queue/types.ts +32 -16
- package/src/server/index.ts +773 -197
- package/src/server/listener/manager.ts +60 -22
- package/src/server/listener/transmitter/index.ts +1 -0
- package/src/server/listener/transmitter/internal.ts +5 -0
- package/src/server/listener/transmitter/pull-responder.ts +11 -15
- package/src/server/listener/transmitter/subscription.ts +364 -0
- package/src/server/listener/transmitter/types.ts +15 -0
- package/src/server/types.ts +55 -6
- package/src/storage/browser.ts +69 -5
- package/src/storage/filesystem.ts +60 -2
- package/src/storage/memory.ts +67 -4
- package/src/storage/prisma.ts +316 -81
- package/src/storage/sequelize.ts +69 -7
- package/src/storage/types.ts +19 -5
- package/src/queue/manager.ts +0 -22
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,151 @@
|
|
|
1
|
+
# [1.0.0-alpha.75](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.74...v1.0.0-alpha.75) (2024-06-18)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* check if operations being added are already stored ([5de4d74](https://github.com/powerhouse-inc/document-drive/commit/5de4d7482e2e6fa60a4e2ca43734771ff501f4d6))
|
|
7
|
+
|
|
8
|
+
# [1.0.0-alpha.74](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.73...v1.0.0-alpha.74) (2024-06-12)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Features
|
|
12
|
+
|
|
13
|
+
* cleanup listeners when drive is removed ([7c33166](https://github.com/powerhouse-inc/document-drive/commit/7c33166141ee65d7776e5281be0e808bd8936563))
|
|
14
|
+
|
|
15
|
+
# [1.0.0-alpha.73](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.72...v1.0.0-alpha.73) (2024-06-10)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
### Features
|
|
19
|
+
|
|
20
|
+
* add storage method to fetch sync units revision ([aa9c16f](https://github.com/powerhouse-inc/document-drive/commit/aa9c16f6b8b90273dc6df007e806274fa2d30f76))
|
|
21
|
+
* implemented getSynchronizationUnitsRevision ([4e8f899](https://github.com/powerhouse-inc/document-drive/commit/4e8f89919d1098dbb4376a9e3be137f581b565d2))
|
|
22
|
+
* removed unnecessary sync units call on pullResponder ([972d2ec](https://github.com/powerhouse-inc/document-drive/commit/972d2ec15cf0f150a4e2b6e88a09e6df6cb7dbc7))
|
|
23
|
+
* removed unnecessary sync units queries ([1c0d98f](https://github.com/powerhouse-inc/document-drive/commit/1c0d98f65dd0e7a52fbb7ad1ad394c256419dc68))
|
|
24
|
+
|
|
25
|
+
# [1.0.0-alpha.72](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.71...v1.0.0-alpha.72) (2024-06-07)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
### Features
|
|
29
|
+
|
|
30
|
+
* add sync status to file when add drive operations ([a6c5380](https://github.com/powerhouse-inc/document-drive/commit/a6c5380aafb383c467fc765ee2c2690bdc20cd7c))
|
|
31
|
+
* added sync status to files when add document operations ([c89b02c](https://github.com/powerhouse-inc/document-drive/commit/c89b02c01c3589c725ffa700fbf38812ea700259))
|
|
32
|
+
* remove related sync status when stopSyncRemoteDrive ([c6577e2](https://github.com/powerhouse-inc/document-drive/commit/c6577e2df3846f15248bacbc8b42f358949db3c8))
|
|
33
|
+
* use syncId instead of documentId ([bd4073a](https://github.com/powerhouse-inc/document-drive/commit/bd4073a3d741e03bf99ae9651ef57c5192a1a204))
|
|
34
|
+
|
|
35
|
+
# [1.0.0-alpha.71](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.70...v1.0.0-alpha.71) (2024-06-07)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
### Features
|
|
39
|
+
|
|
40
|
+
* added queue actions implementations ([6741a59](https://github.com/powerhouse-inc/document-drive/commit/6741a59d191c5cdd1bfb253c3d5ee28d7f2a0779))
|
|
41
|
+
* support operation and action jobs ([7e6c878](https://github.com/powerhouse-inc/document-drive/commit/7e6c878560ce3132d8dc1a71913a83ad5d138da9))
|
|
42
|
+
|
|
43
|
+
# [1.0.0-alpha.70](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.69...v1.0.0-alpha.70) (2024-06-07)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
### Bug Fixes
|
|
47
|
+
|
|
48
|
+
* slug caching ([1465ef0](https://github.com/powerhouse-inc/document-drive/commit/1465ef028d825ed2dfe50126c6dc64e8ba4445d5))
|
|
49
|
+
* slug caching ([1134dc6](https://github.com/powerhouse-inc/document-drive/commit/1134dc6c1384648f6af5c871d0e51e6acb4e63fb))
|
|
50
|
+
|
|
51
|
+
# [1.0.0-alpha.69](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.68...v1.0.0-alpha.69) (2024-06-07)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
### Bug Fixes
|
|
55
|
+
|
|
56
|
+
* optimize get documents ([f6f13ab](https://github.com/powerhouse-inc/document-drive/commit/f6f13abb65d8c34a7ba64787fa4ae4ee024956fa))
|
|
57
|
+
|
|
58
|
+
# [1.0.0-alpha.68](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.67...v1.0.0-alpha.68) (2024-06-04)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
### Bug Fixes
|
|
62
|
+
|
|
63
|
+
* redis queue ([e99d1b9](https://github.com/powerhouse-inc/document-drive/commit/e99d1b9f0c704836cc0752e36e0f905dc192e5e1))
|
|
64
|
+
|
|
65
|
+
# [1.0.0-alpha.67](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.66...v1.0.0-alpha.67) (2024-06-04)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
### Features
|
|
69
|
+
|
|
70
|
+
* get operations from cache on storage ([8416464](https://github.com/powerhouse-inc/document-drive/commit/84164646609045b2cd8e9e8747e364c746d9e3ec))
|
|
71
|
+
|
|
72
|
+
# [1.0.0-alpha.66](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.65...v1.0.0-alpha.66) (2024-06-03)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
### Features
|
|
76
|
+
|
|
77
|
+
* updated libs ([5e47758](https://github.com/powerhouse-inc/document-drive/commit/5e477580524a1defd92dd0bef9b160e182436ec9))
|
|
78
|
+
|
|
79
|
+
# [1.0.0-alpha.65](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.64...v1.0.0-alpha.65) (2024-05-30)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
### Features
|
|
83
|
+
|
|
84
|
+
* fetch resulting state for last unskipped operation ([8297353](https://github.com/powerhouse-inc/document-drive/commit/8297353dc8eaca107d8134580dee67c3265b05b5))
|
|
85
|
+
|
|
86
|
+
# [1.0.0-alpha.64](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.63...v1.0.0-alpha.64) (2024-05-30)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
### Features
|
|
90
|
+
|
|
91
|
+
* update document-model lib version ([#180](https://github.com/powerhouse-inc/document-drive/issues/180)) ([83cec58](https://github.com/powerhouse-inc/document-drive/commit/83cec58cb02388a3b2a643dd5af6c31cd850242d))
|
|
92
|
+
|
|
93
|
+
# [1.0.0-alpha.63](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.62...v1.0.0-alpha.63) (2024-05-30)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
### Bug Fixes
|
|
97
|
+
|
|
98
|
+
* operations query ([8dd11c7](https://github.com/powerhouse-inc/document-drive/commit/8dd11c72058452f3e8febac472d01ad56d959e17))
|
|
99
|
+
|
|
100
|
+
# [1.0.0-alpha.62](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.61...v1.0.0-alpha.62) (2024-05-30)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
### Bug Fixes
|
|
104
|
+
|
|
105
|
+
* added drive and documentId filter ([718405f](https://github.com/powerhouse-inc/document-drive/commit/718405febbe8e53a3fd392cdd4acf2b041347eaf))
|
|
106
|
+
|
|
107
|
+
# [1.0.0-alpha.61](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.60...v1.0.0-alpha.61) (2024-05-30)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
### Features
|
|
111
|
+
|
|
112
|
+
* store resulting state as bytes and only retrieve state for last op of each scope ([c6a5004](https://github.com/powerhouse-inc/document-drive/commit/c6a5004a3dd07ee1ce2f5521bb7a6aa5e970465e))
|
|
113
|
+
|
|
114
|
+
# [1.0.0-alpha.60](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.59...v1.0.0-alpha.60) (2024-05-29)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
### Features
|
|
118
|
+
|
|
119
|
+
* avoid duplicated getDocument call ([c0684bc](https://github.com/powerhouse-inc/document-drive/commit/c0684bc0746d8bafb9393cdaeebfb60b38a8a32f))
|
|
120
|
+
|
|
121
|
+
# [1.0.0-alpha.59](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.58...v1.0.0-alpha.59) (2024-05-28)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
### Features
|
|
125
|
+
|
|
126
|
+
* enable operation id ([#174](https://github.com/powerhouse-inc/document-drive/issues/174)) ([1d77fd2](https://github.com/powerhouse-inc/document-drive/commit/1d77fd2f6a4618371c6fc4c072d4eab7d27a662a))
|
|
127
|
+
|
|
128
|
+
# [1.0.0-alpha.58](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.57...v1.0.0-alpha.58) (2024-05-28)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
### Features
|
|
132
|
+
|
|
133
|
+
* don't save queue job result ([efbf239](https://github.com/powerhouse-inc/document-drive/commit/efbf239e5f77592267446d5c11d670a82f1f6e58))
|
|
134
|
+
|
|
135
|
+
# [1.0.0-alpha.57](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.56...v1.0.0-alpha.57) (2024-05-22)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
### Features
|
|
139
|
+
|
|
140
|
+
* new release ([17796e8](https://github.com/powerhouse-inc/document-drive/commit/17796e8577d16d3095a8adf3c40e7bd7146b6142))
|
|
141
|
+
|
|
142
|
+
# [1.0.0-alpha.56](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.55...v1.0.0-alpha.56) (2024-05-22)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
### Features
|
|
146
|
+
|
|
147
|
+
* add queues and append only conflict resolution ([d17abd6](https://github.com/powerhouse-inc/document-drive/commit/d17abd664a7381f80faa6530f83ca9e224282ba1)), closes [#153](https://github.com/powerhouse-inc/document-drive/issues/153)
|
|
148
|
+
|
|
1
149
|
# 1.0.0-experimental.1 (2024-05-15)
|
|
2
150
|
|
|
3
151
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "document-drive",
|
|
3
|
-
"version": "1.0.0-
|
|
3
|
+
"version": "1.0.0-websockets",
|
|
4
4
|
"license": "AGPL-3.0-only",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"module": "./src/index.ts",
|
|
@@ -34,15 +34,18 @@
|
|
|
34
34
|
"test:watch": "vitest watch"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
|
-
"document-model": "1.
|
|
38
|
-
"document-model-libs": "^1.
|
|
37
|
+
"document-model": "^1.4.0",
|
|
38
|
+
"document-model-libs": "^1.60.1"
|
|
39
39
|
},
|
|
40
40
|
"optionalDependencies": {
|
|
41
|
-
"@prisma/client": "5.
|
|
41
|
+
"@prisma/client": "5.14.0",
|
|
42
|
+
"graphql-ws": "^5.16.0",
|
|
43
|
+
"isomorphic-ws": "^5.0.0",
|
|
42
44
|
"localforage": "^1.10.0",
|
|
43
45
|
"redis": "^4.6.13",
|
|
44
46
|
"sequelize": "^6.35.2",
|
|
45
|
-
"sqlite3": "^5.1.7"
|
|
47
|
+
"sqlite3": "^5.1.7",
|
|
48
|
+
"ws": "^8.17.0"
|
|
46
49
|
},
|
|
47
50
|
"dependencies": {
|
|
48
51
|
"exponential-backoff": "^3.1.1",
|
|
@@ -62,11 +65,12 @@
|
|
|
62
65
|
"@total-typescript/ts-reset": "^0.5.1",
|
|
63
66
|
"@types/node": "^20.12.7",
|
|
64
67
|
"@types/uuid": "^9.0.8",
|
|
68
|
+
"@types/ws": "^8.5.10",
|
|
65
69
|
"@typescript-eslint/eslint-plugin": "^6.21.0",
|
|
66
70
|
"@typescript-eslint/parser": "^6.21.0",
|
|
67
71
|
"@vitest/coverage-v8": "^1.4.0",
|
|
68
|
-
"document-model": "1.
|
|
69
|
-
"document-model-libs": "1.
|
|
72
|
+
"document-model": "^1.4.0",
|
|
73
|
+
"document-model-libs": "^1.60.1",
|
|
70
74
|
"eslint": "^8.57.0",
|
|
71
75
|
"eslint-config-prettier": "^9.1.0",
|
|
72
76
|
"fake-indexeddb": "^5.0.2",
|
|
@@ -74,11 +78,12 @@
|
|
|
74
78
|
"msw": "^2.2.13",
|
|
75
79
|
"prettier": "^3.2.5",
|
|
76
80
|
"prettier-plugin-organize-imports": "^3.2.4",
|
|
77
|
-
"prisma": "^5.
|
|
81
|
+
"prisma": "^5.14.0",
|
|
78
82
|
"semantic-release": "^23.0.8",
|
|
79
83
|
"sequelize": "^6.37.2",
|
|
80
84
|
"sqlite3": "^5.1.7",
|
|
81
85
|
"typescript": "^5.4.4",
|
|
82
|
-
"vitest": "^1.
|
|
83
|
-
}
|
|
86
|
+
"vitest": "^1.6.0"
|
|
87
|
+
},
|
|
88
|
+
"packageManager": "pnpm@9.1.4+sha256.30a1801ac4e723779efed13a21f4c39f9eb6c9fbb4ced101bce06b422593d7c9"
|
|
84
89
|
}
|
package/src/queue/base.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { IJob, IJobQueue, IQueue, IQueueManager, IServerDelegate, JobId,
|
|
1
|
+
import { IJob, IJobQueue, IQueue, IQueueManager, IServerDelegate, JobId, Job, QueueEvents, isOperationJob } from "./types";
|
|
2
2
|
import { generateUUID } from "../utils";
|
|
3
|
-
import { IOperationResult } from "../server";
|
|
4
3
|
import { createNanoEvents, Unsubscribe } from 'nanoevents';
|
|
5
|
-
import {
|
|
4
|
+
import { Action } from "document-model/document";
|
|
6
5
|
import { AddFileInput, DeleteNodeInput } from "document-model-libs/document-drive";
|
|
7
6
|
|
|
8
7
|
export class MemoryQueue<T, R> implements IQueue<T, R> {
|
|
@@ -10,8 +9,7 @@ export class MemoryQueue<T, R> implements IQueue<T, R> {
|
|
|
10
9
|
private blocked = false;
|
|
11
10
|
private deleted = false;
|
|
12
11
|
private items: IJob<T>[] = [];
|
|
13
|
-
private
|
|
14
|
-
private dependencies = new Array<IJob<OperationJob>>();
|
|
12
|
+
private dependencies = new Array<IJob<Job>>();
|
|
15
13
|
|
|
16
14
|
constructor(id: string) {
|
|
17
15
|
this.id = id;
|
|
@@ -25,15 +23,6 @@ export class MemoryQueue<T, R> implements IQueue<T, R> {
|
|
|
25
23
|
return this.deleted;
|
|
26
24
|
}
|
|
27
25
|
|
|
28
|
-
async setResult(jobId: string, result: R): Promise<void> {
|
|
29
|
-
this.results.set(jobId, result);
|
|
30
|
-
return Promise.resolve();
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
async getResult(jobId: string): Promise<R | undefined> {
|
|
34
|
-
return Promise.resolve(this.results.get(jobId));
|
|
35
|
-
}
|
|
36
|
-
|
|
37
26
|
async addJob(data: IJob<T>) {
|
|
38
27
|
this.items.push(data);
|
|
39
28
|
return Promise.resolve();
|
|
@@ -64,7 +53,7 @@ export class MemoryQueue<T, R> implements IQueue<T, R> {
|
|
|
64
53
|
return this.items;
|
|
65
54
|
}
|
|
66
55
|
|
|
67
|
-
async addDependencies(job: IJob<
|
|
56
|
+
async addDependencies(job: IJob<Job>) {
|
|
68
57
|
if (!this.dependencies.find(j => j.jobId === job.jobId)) {
|
|
69
58
|
this.dependencies.push(job);
|
|
70
59
|
}
|
|
@@ -73,7 +62,7 @@ export class MemoryQueue<T, R> implements IQueue<T, R> {
|
|
|
73
62
|
}
|
|
74
63
|
}
|
|
75
64
|
|
|
76
|
-
async removeDependencies(job: IJob<
|
|
65
|
+
async removeDependencies(job: IJob<Job>) {
|
|
77
66
|
this.dependencies = this.dependencies.filter((j) => j.jobId !== job.jobId && j.driveId !== job.driveId);
|
|
78
67
|
if (this.dependencies.length === 0) {
|
|
79
68
|
await this.setBlocked(false);
|
|
@@ -102,7 +91,7 @@ export class BaseQueueManager implements IQueueManager {
|
|
|
102
91
|
return Promise.resolve()
|
|
103
92
|
}
|
|
104
93
|
|
|
105
|
-
async addJob(job:
|
|
94
|
+
async addJob(job: Job): Promise<JobId> {
|
|
106
95
|
if (!this.delegate) {
|
|
107
96
|
throw new Error("No server delegate defined");
|
|
108
97
|
}
|
|
@@ -125,7 +114,8 @@ export class BaseQueueManager implements IQueueManager {
|
|
|
125
114
|
const driveQueue = this.getQueue(job.driveId);
|
|
126
115
|
const jobs = await driveQueue.getJobs();
|
|
127
116
|
for (const driveJob of jobs) {
|
|
128
|
-
const
|
|
117
|
+
const actions = isOperationJob(driveJob) ? driveJob.operations : driveJob.actions;
|
|
118
|
+
const op = actions.find((j: Action) => {
|
|
129
119
|
const input = j.input as AddFileInput;
|
|
130
120
|
return j.type === "ADD_FILE" && input.id === job.documentId
|
|
131
121
|
})
|
|
@@ -137,7 +127,8 @@ export class BaseQueueManager implements IQueueManager {
|
|
|
137
127
|
|
|
138
128
|
// if it has ADD_FILE operations then adds the job as
|
|
139
129
|
// a dependency to the corresponding document queues
|
|
140
|
-
const
|
|
130
|
+
const actions = isOperationJob(job) ? job.operations : job.actions;
|
|
131
|
+
const addFileOps = actions.filter((j: Action) => j.type === "ADD_FILE");
|
|
141
132
|
for (const addFileOp of addFileOps) {
|
|
142
133
|
const input = addFileOp.input as AddFileInput;
|
|
143
134
|
const q = this.getQueue(job.driveId, input.id)
|
|
@@ -145,7 +136,7 @@ export class BaseQueueManager implements IQueueManager {
|
|
|
145
136
|
}
|
|
146
137
|
|
|
147
138
|
// remove document if operations contains delete_node
|
|
148
|
-
const removeFileOps =
|
|
139
|
+
const removeFileOps = actions.filter((j: Action) => j.type === "DELETE_NODE");
|
|
149
140
|
for (const removeFileOp of removeFileOps) {
|
|
150
141
|
const input = removeFileOp.input as DeleteNodeInput;
|
|
151
142
|
const queue = this.getQueue(job.driveId, input.id);
|
|
@@ -157,11 +148,6 @@ export class BaseQueueManager implements IQueueManager {
|
|
|
157
148
|
return jobId;
|
|
158
149
|
}
|
|
159
150
|
|
|
160
|
-
async getResult(driveId: string, documentId: string, jobId: JobId): Promise<IOperationResult | undefined> {
|
|
161
|
-
const queue = this.getQueue(driveId, documentId);
|
|
162
|
-
return queue.getResult(jobId);
|
|
163
|
-
}
|
|
164
|
-
|
|
165
151
|
getQueue(driveId: string, documentId?: string) {
|
|
166
152
|
const queueId = this.getQueueId(driveId, documentId);
|
|
167
153
|
let queue = this.queues.find((q) => q.getId() === queueId);
|
|
@@ -236,23 +222,22 @@ export class BaseQueueManager implements IQueueManager {
|
|
|
236
222
|
}
|
|
237
223
|
|
|
238
224
|
try {
|
|
239
|
-
const result = await this.delegate.
|
|
240
|
-
await queue.setResult(nextJob.jobId, result);
|
|
225
|
+
const result = await this.delegate.processJob(nextJob);
|
|
241
226
|
|
|
242
227
|
// unblock the document queues of each add_file operation
|
|
243
|
-
const
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
228
|
+
const actions = isOperationJob(nextJob) ? nextJob.operations : nextJob.actions;
|
|
229
|
+
const addFileActions = actions.filter((op) => op.type === "ADD_FILE");
|
|
230
|
+
if (addFileActions.length > 0) {
|
|
231
|
+
for (const addFile of addFileActions) {
|
|
232
|
+
const documentQueue = this.getQueue(nextJob.driveId, (addFile.input as AddFileInput).id);
|
|
247
233
|
await documentQueue.removeDependencies(nextJob);
|
|
248
|
-
}
|
|
234
|
+
};
|
|
249
235
|
}
|
|
250
|
-
|
|
251
236
|
this.emit("jobCompleted", nextJob, result);
|
|
252
237
|
} catch (e) {
|
|
253
238
|
this.emit("jobFailed", nextJob, e as Error);
|
|
254
239
|
} finally {
|
|
255
|
-
queue.setBlocked(false);
|
|
240
|
+
await queue.setBlocked(false);
|
|
256
241
|
await this.processNextJob();
|
|
257
242
|
}
|
|
258
243
|
}
|
package/src/queue/redis.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { RedisClientType } from "redis";
|
|
2
|
-
import { IJob, IQueue, IQueueManager, OperationJob } from "./types";
|
|
2
|
+
import { IJob, IQueue, IQueueManager, IServerDelegate, OperationJob } from "./types";
|
|
3
3
|
import { BaseQueueManager } from "./base";
|
|
4
4
|
|
|
5
5
|
export class RedisQueue<T, R> implements IQueue<T, R> {
|
|
@@ -10,18 +10,7 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
|
|
|
10
10
|
this.client = client;
|
|
11
11
|
this.id = id;
|
|
12
12
|
this.client.hSet("queues", id, "true");
|
|
13
|
-
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
async setResult(jobId: string, result: any): Promise<void> {
|
|
17
|
-
await this.client.hSet(this.id + "-results", jobId, JSON.stringify(result));
|
|
18
|
-
}
|
|
19
|
-
async getResult(jobId: string): Promise<any> {
|
|
20
|
-
const results = await this.client.hGet(this.id + "-results", jobId);
|
|
21
|
-
if (!results) {
|
|
22
|
-
return null;
|
|
23
|
-
}
|
|
24
|
-
return JSON.parse(results);
|
|
13
|
+
this.client.hSet(this.id, "blocked", "false");
|
|
25
14
|
}
|
|
26
15
|
|
|
27
16
|
async addJob(data: any) {
|
|
@@ -31,9 +20,9 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
|
|
|
31
20
|
async getNextJob() {
|
|
32
21
|
const job = await this.client.rPop(this.id + "-jobs");
|
|
33
22
|
if (!job) {
|
|
34
|
-
return
|
|
23
|
+
return undefined;
|
|
35
24
|
}
|
|
36
|
-
return JSON.parse(job)
|
|
25
|
+
return JSON.parse(job) as IJob<T>;
|
|
37
26
|
}
|
|
38
27
|
|
|
39
28
|
async amountOfJobs() {
|
|
@@ -44,13 +33,13 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
|
|
|
44
33
|
if (blocked) {
|
|
45
34
|
await this.client.hSet(this.id, "blocked", "true");
|
|
46
35
|
} else {
|
|
47
|
-
await this.client.
|
|
36
|
+
await this.client.hSet(this.id, "blocked", "false");
|
|
48
37
|
}
|
|
49
38
|
}
|
|
50
39
|
|
|
51
40
|
async isBlocked() {
|
|
52
41
|
const blockedResult = await this.client.hGet(this.id, "blocked");
|
|
53
|
-
if (blockedResult) {
|
|
42
|
+
if (blockedResult === "true") {
|
|
54
43
|
return true;
|
|
55
44
|
}
|
|
56
45
|
|
|
@@ -63,7 +52,7 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
|
|
|
63
52
|
|
|
64
53
|
async getJobs() {
|
|
65
54
|
const entries = await this.client.lRange(this.id + "-jobs", 0, -1)
|
|
66
|
-
return entries.map(e => JSON.parse(e));
|
|
55
|
+
return entries.map(e => JSON.parse(e) as IJob<T>);
|
|
67
56
|
}
|
|
68
57
|
|
|
69
58
|
async addDependencies(job: IJob<OperationJob>) {
|
|
@@ -80,7 +69,6 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
|
|
|
80
69
|
}
|
|
81
70
|
|
|
82
71
|
async removeDependencies(job: IJob<OperationJob>) {
|
|
83
|
-
const allDeps1 = await this.client.lLen(this.id + "-deps");
|
|
84
72
|
await this.client.lRem(this.id + "-deps", 1, JSON.stringify(job));
|
|
85
73
|
const allDeps = await this.client.lLen(this.id + "-deps");
|
|
86
74
|
if (allDeps > 0) {
|
|
@@ -91,15 +79,15 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
|
|
|
91
79
|
}
|
|
92
80
|
|
|
93
81
|
async isDeleted() {
|
|
94
|
-
const
|
|
95
|
-
return
|
|
82
|
+
const active = await this.client.hGet("queues", this.id);
|
|
83
|
+
return active === "false";
|
|
96
84
|
}
|
|
97
85
|
|
|
98
86
|
async setDeleted(deleted: boolean) {
|
|
99
87
|
if (deleted) {
|
|
100
|
-
await this.client.hSet(this.id, "
|
|
88
|
+
await this.client.hSet("queues", this.id, "false");
|
|
101
89
|
} else {
|
|
102
|
-
await this.client.
|
|
90
|
+
await this.client.hSet("queues", this.id, "true");
|
|
103
91
|
}
|
|
104
92
|
}
|
|
105
93
|
}
|
|
@@ -115,10 +103,12 @@ export class RedisQueueManager extends BaseQueueManager implements IQueueManager
|
|
|
115
103
|
|
|
116
104
|
async init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void> {
|
|
117
105
|
await super.init(delegate, onError);
|
|
118
|
-
// load all queues
|
|
119
106
|
const queues = await this.client.hGetAll("queues");
|
|
120
107
|
for (const queueId in queues) {
|
|
121
|
-
this.
|
|
108
|
+
const active = await this.client.hGet("queues", queueId);
|
|
109
|
+
if (active === "true") {
|
|
110
|
+
this.queues.push(new RedisQueue(queueId, this.client));
|
|
111
|
+
}
|
|
122
112
|
}
|
|
123
113
|
}
|
|
124
114
|
|
package/src/queue/types.ts
CHANGED
|
@@ -1,32 +1,42 @@
|
|
|
1
|
-
import { Operation } from "document-model/document";
|
|
1
|
+
import { Action, Operation } from "document-model/document";
|
|
2
2
|
import { IOperationResult } from "../server";
|
|
3
3
|
import type { Unsubscribe } from "nanoevents";
|
|
4
4
|
|
|
5
|
-
export
|
|
5
|
+
export interface BaseJob {
|
|
6
6
|
driveId: string;
|
|
7
7
|
documentId?: string
|
|
8
|
-
|
|
8
|
+
actions?: Action[]
|
|
9
9
|
forceSync?: boolean
|
|
10
10
|
}
|
|
11
11
|
|
|
12
|
+
export interface OperationJob extends BaseJob {
|
|
13
|
+
operations: Operation[]
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface ActionJob extends BaseJob {
|
|
17
|
+
actions: Action[]
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export type Job = OperationJob | ActionJob;
|
|
21
|
+
|
|
12
22
|
export type JobId = string;
|
|
13
23
|
|
|
14
24
|
export interface QueueEvents {
|
|
15
|
-
jobCompleted: (job: IJob<
|
|
16
|
-
jobFailed: (job: IJob<
|
|
25
|
+
jobCompleted: (job: IJob<Job>, result: IOperationResult) => void;
|
|
26
|
+
jobFailed: (job: IJob<Job>, error: Error) => void;
|
|
27
|
+
queueRemoved: (queueId: string) => void;
|
|
17
28
|
}
|
|
18
29
|
|
|
19
30
|
export interface IServerDelegate {
|
|
20
31
|
checkDocumentExists: (driveId: string, documentId: string) => Promise<boolean>;
|
|
21
|
-
|
|
22
|
-
}
|
|
32
|
+
processJob: (job: Job) => Promise<IOperationResult>;
|
|
33
|
+
};
|
|
23
34
|
|
|
24
35
|
export interface IQueueManager {
|
|
25
|
-
addJob(job:
|
|
26
|
-
|
|
27
|
-
getQueue(driveId: string, document?: string): IQueue<OperationJob, IOperationResult>;
|
|
36
|
+
addJob(job: Job): Promise<JobId>;
|
|
37
|
+
getQueue(driveId: string, document?: string): IQueue<Job, IOperationResult>;
|
|
28
38
|
removeQueue(driveId: string, documentId?: string): void;
|
|
29
|
-
getQueueByIndex(index: number): IQueue<
|
|
39
|
+
getQueueByIndex(index: number): IQueue<Job, IOperationResult> | null;
|
|
30
40
|
getQueues(): string[];
|
|
31
41
|
init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void>;
|
|
32
42
|
on<K extends keyof QueueEvents>(
|
|
@@ -47,11 +57,17 @@ export interface IQueue<T, R> {
|
|
|
47
57
|
isBlocked(): Promise<boolean>;
|
|
48
58
|
isDeleted(): Promise<boolean>;
|
|
49
59
|
setDeleted(deleted: boolean): Promise<void>;
|
|
50
|
-
setResult(jobId: JobId, result: R): Promise<void>;
|
|
51
|
-
getResult(jobId: JobId): Promise<R | undefined>;
|
|
52
60
|
getJobs(): Promise<IJob<T>[]>;
|
|
53
|
-
addDependencies(job: IJob<
|
|
54
|
-
removeDependencies(job: IJob<
|
|
61
|
+
addDependencies(job: IJob<Job>): Promise<void>;
|
|
62
|
+
removeDependencies(job: IJob<Job>): Promise<void>;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export type IJobQueue = IQueue<Job, IOperationResult>;
|
|
66
|
+
|
|
67
|
+
export function isOperationJob(job: Job): job is OperationJob {
|
|
68
|
+
return "operations" in job;
|
|
55
69
|
}
|
|
56
70
|
|
|
57
|
-
export
|
|
71
|
+
export function isActionJob(job: Job): job is ActionJob {
|
|
72
|
+
return "actions" in job;
|
|
73
|
+
}
|