@nocobase/plugin-async-task-manager 1.9.0-beta.4 → 1.9.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ISSUES.md +31 -0
- package/dist/client/index.d.ts +17 -4
- package/dist/client/index.js +1 -1
- package/dist/client/locale.d.ts +9 -1
- package/dist/common/collections/asyncTasks.d.ts +25 -0
- package/dist/common/collections/asyncTasks.js +97 -0
- package/dist/common/constants.d.ts +73 -0
- package/dist/common/constants.js +86 -0
- package/dist/common/types.d.ts +11 -0
- package/dist/common/types.js +24 -0
- package/dist/externalVersion.js +7 -5
- package/dist/server/base-task-manager.d.ts +21 -15
- package/dist/server/base-task-manager.js +230 -101
- package/dist/server/collections/asyncTasks.d.ts +11 -0
- package/dist/server/collections/asyncTasks.js +43 -0
- package/dist/server/command-task-type.d.ts +9 -0
- package/dist/server/command-task-type.js +19 -8
- package/dist/server/interfaces/async-task-manager.d.ts +11 -48
- package/dist/server/interfaces/task.d.ts +25 -23
- package/dist/server/interfaces/task.js +28 -0
- package/dist/server/plugin.d.ts +2 -5
- package/dist/server/plugin.js +15 -129
- package/dist/server/resourcers/async-tasks.d.ts +3 -2
- package/dist/server/resourcers/async-tasks.js +44 -27
- package/dist/server/task-type.d.ts +10 -62
- package/dist/server/task-type.js +66 -121
- package/package.json +2 -2
- package/dist/client/TaskResultRendererManager.d.ts +0 -6
- package/dist/node_modules/p-queue/dist/index.d.ts +0 -101
- package/dist/node_modules/p-queue/dist/index.js +0 -1
- package/dist/node_modules/p-queue/dist/lower-bound.d.ts +0 -1
- package/dist/node_modules/p-queue/dist/lower-bound.js +0 -21
- package/dist/node_modules/p-queue/dist/options.d.ts +0 -64
- package/dist/node_modules/p-queue/dist/options.js +0 -2
- package/dist/node_modules/p-queue/dist/priority-queue.d.ts +0 -12
- package/dist/node_modules/p-queue/dist/priority-queue.js +0 -32
- package/dist/node_modules/p-queue/dist/queue.d.ts +0 -7
- package/dist/node_modules/p-queue/dist/queue.js +0 -2
- package/dist/node_modules/p-queue/license +0 -9
- package/dist/node_modules/p-queue/node_modules/eventemitter3/index.d.ts +0 -134
- package/dist/node_modules/p-queue/node_modules/eventemitter3/index.js +0 -336
- package/dist/node_modules/p-queue/node_modules/eventemitter3/package.json +0 -56
- package/dist/node_modules/p-queue/node_modules/eventemitter3/umd/eventemitter3.js +0 -340
- package/dist/node_modules/p-queue/node_modules/eventemitter3/umd/eventemitter3.min.js +0 -1
- package/dist/node_modules/p-queue/package.json +0 -1
- package/dist/node_modules/uuid/dist/bin/uuid +0 -2
- package/dist/node_modules/uuid/dist/commonjs-browser/index.js +0 -79
- package/dist/node_modules/uuid/dist/commonjs-browser/md5.js +0 -223
- package/dist/node_modules/uuid/dist/commonjs-browser/native.js +0 -11
- package/dist/node_modules/uuid/dist/commonjs-browser/nil.js +0 -8
- package/dist/node_modules/uuid/dist/commonjs-browser/parse.js +0 -45
- package/dist/node_modules/uuid/dist/commonjs-browser/regex.js +0 -8
- package/dist/node_modules/uuid/dist/commonjs-browser/rng.js +0 -25
- package/dist/node_modules/uuid/dist/commonjs-browser/sha1.js +0 -104
- package/dist/node_modules/uuid/dist/commonjs-browser/stringify.js +0 -44
- package/dist/node_modules/uuid/dist/commonjs-browser/v1.js +0 -107
- package/dist/node_modules/uuid/dist/commonjs-browser/v3.js +0 -16
- package/dist/node_modules/uuid/dist/commonjs-browser/v35.js +0 -80
- package/dist/node_modules/uuid/dist/commonjs-browser/v4.js +0 -43
- package/dist/node_modules/uuid/dist/commonjs-browser/v5.js +0 -16
- package/dist/node_modules/uuid/dist/commonjs-browser/validate.js +0 -17
- package/dist/node_modules/uuid/dist/commonjs-browser/version.js +0 -21
- package/dist/node_modules/uuid/dist/esm-browser/index.js +0 -9
- package/dist/node_modules/uuid/dist/esm-browser/md5.js +0 -215
- package/dist/node_modules/uuid/dist/esm-browser/native.js +0 -4
- package/dist/node_modules/uuid/dist/esm-browser/nil.js +0 -1
- package/dist/node_modules/uuid/dist/esm-browser/parse.js +0 -35
- package/dist/node_modules/uuid/dist/esm-browser/regex.js +0 -1
- package/dist/node_modules/uuid/dist/esm-browser/rng.js +0 -18
- package/dist/node_modules/uuid/dist/esm-browser/sha1.js +0 -96
- package/dist/node_modules/uuid/dist/esm-browser/stringify.js +0 -33
- package/dist/node_modules/uuid/dist/esm-browser/v1.js +0 -95
- package/dist/node_modules/uuid/dist/esm-browser/v3.js +0 -4
- package/dist/node_modules/uuid/dist/esm-browser/v35.js +0 -66
- package/dist/node_modules/uuid/dist/esm-browser/v4.js +0 -29
- package/dist/node_modules/uuid/dist/esm-browser/v5.js +0 -4
- package/dist/node_modules/uuid/dist/esm-browser/validate.js +0 -7
- package/dist/node_modules/uuid/dist/esm-browser/version.js +0 -11
- package/dist/node_modules/uuid/dist/esm-node/index.js +0 -9
- package/dist/node_modules/uuid/dist/esm-node/md5.js +0 -13
- package/dist/node_modules/uuid/dist/esm-node/native.js +0 -4
- package/dist/node_modules/uuid/dist/esm-node/nil.js +0 -1
- package/dist/node_modules/uuid/dist/esm-node/parse.js +0 -35
- package/dist/node_modules/uuid/dist/esm-node/regex.js +0 -1
- package/dist/node_modules/uuid/dist/esm-node/rng.js +0 -12
- package/dist/node_modules/uuid/dist/esm-node/sha1.js +0 -13
- package/dist/node_modules/uuid/dist/esm-node/stringify.js +0 -33
- package/dist/node_modules/uuid/dist/esm-node/v1.js +0 -95
- package/dist/node_modules/uuid/dist/esm-node/v3.js +0 -4
- package/dist/node_modules/uuid/dist/esm-node/v35.js +0 -66
- package/dist/node_modules/uuid/dist/esm-node/v4.js +0 -29
- package/dist/node_modules/uuid/dist/esm-node/v5.js +0 -4
- package/dist/node_modules/uuid/dist/esm-node/validate.js +0 -7
- package/dist/node_modules/uuid/dist/esm-node/version.js +0 -11
- package/dist/node_modules/uuid/dist/index.js +0 -1
- package/dist/node_modules/uuid/dist/md5-browser.js +0 -223
- package/dist/node_modules/uuid/dist/md5.js +0 -23
- package/dist/node_modules/uuid/dist/native-browser.js +0 -11
- package/dist/node_modules/uuid/dist/native.js +0 -15
- package/dist/node_modules/uuid/dist/nil.js +0 -8
- package/dist/node_modules/uuid/dist/parse.js +0 -45
- package/dist/node_modules/uuid/dist/regex.js +0 -8
- package/dist/node_modules/uuid/dist/rng-browser.js +0 -25
- package/dist/node_modules/uuid/dist/rng.js +0 -24
- package/dist/node_modules/uuid/dist/sha1-browser.js +0 -104
- package/dist/node_modules/uuid/dist/sha1.js +0 -23
- package/dist/node_modules/uuid/dist/stringify.js +0 -44
- package/dist/node_modules/uuid/dist/uuid-bin.js +0 -85
- package/dist/node_modules/uuid/dist/v1.js +0 -107
- package/dist/node_modules/uuid/dist/v3.js +0 -16
- package/dist/node_modules/uuid/dist/v35.js +0 -80
- package/dist/node_modules/uuid/dist/v4.js +0 -43
- package/dist/node_modules/uuid/dist/v5.js +0 -16
- package/dist/node_modules/uuid/dist/validate.js +0 -17
- package/dist/node_modules/uuid/dist/version.js +0 -21
- package/dist/node_modules/uuid/package.json +0 -1
|
@@ -39,119 +39,227 @@ __export(base_task_manager_exports, {
|
|
|
39
39
|
BaseTaskManager: () => BaseTaskManager
|
|
40
40
|
});
|
|
41
41
|
module.exports = __toCommonJS(base_task_manager_exports);
|
|
42
|
-
var
|
|
43
|
-
var
|
|
44
|
-
|
|
42
|
+
var import_lodash = require("lodash");
|
|
43
|
+
var import_constants = require("../common/constants");
|
|
44
|
+
var import_crypto = require("crypto");
|
|
45
|
+
var import_plugin = __toESM(require("./plugin"));
|
|
46
|
+
const WORKER_JOB_ASYNC_TASK_PROCESS = "async-task:process";
|
|
47
|
+
class BaseTaskManager {
|
|
45
48
|
taskTypes = /* @__PURE__ */ new Map();
|
|
46
49
|
tasks = /* @__PURE__ */ new Map();
|
|
47
50
|
// Clean up completed tasks after 30 minutes by default
|
|
48
51
|
cleanupDelay = 30 * 60 * 1e3;
|
|
52
|
+
cleanupTimer;
|
|
49
53
|
logger;
|
|
50
54
|
app;
|
|
51
|
-
|
|
52
|
-
|
|
55
|
+
progressThrottles = /* @__PURE__ */ new Map();
|
|
56
|
+
concurrency = process.env.ASYNC_TASK_MAX_CONCURRENCY ? Number.parseInt(process.env.ASYNC_TASK_MAX_CONCURRENCY, 10) : 3;
|
|
57
|
+
idle = () => {
|
|
58
|
+
return this.app.serving(WORKER_JOB_ASYNC_TASK_PROCESS) && this.tasks.size < this.concurrency;
|
|
59
|
+
};
|
|
60
|
+
onQueueTask = async ({ id }) => {
|
|
61
|
+
const task = await this.prepareTask(id);
|
|
62
|
+
await this.runTask(task);
|
|
63
|
+
};
|
|
64
|
+
onTaskProgress = (item) => {
|
|
65
|
+
const userId = item.createdById;
|
|
66
|
+
this.logger.trace(`Task ${item.id} of user(${userId}) progress: ${item.progressCurrent} / ${item.progressTotal}`);
|
|
67
|
+
if (userId) {
|
|
68
|
+
const throttledEmit = this.getThrottledProgressEmitter(item.id, userId);
|
|
69
|
+
throttledEmit(item);
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
onTaskAfterCreate = (task) => {
|
|
73
|
+
const userId = task.createdById;
|
|
74
|
+
if (userId) {
|
|
75
|
+
this.app.emit("ws:sendToUser", {
|
|
76
|
+
userId,
|
|
77
|
+
message: {
|
|
78
|
+
type: "async-tasks:created",
|
|
79
|
+
payload: task.toJSON()
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
onTaskStatusChanged = (task) => {
|
|
85
|
+
if (!task.changed("status")) return;
|
|
86
|
+
const userId = task.createdById;
|
|
87
|
+
if (!userId) return;
|
|
88
|
+
this.app.emit("ws:sendToUser", {
|
|
89
|
+
userId,
|
|
90
|
+
message: {
|
|
91
|
+
type: "async-tasks:status",
|
|
92
|
+
payload: {
|
|
93
|
+
taskId: task.id,
|
|
94
|
+
status: task.status
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
if ([import_constants.TASK_STATUS.RUNNING, import_constants.TASK_STATUS.PENDING].includes(task.status)) {
|
|
99
|
+
const throttled = this.progressThrottles.get(task.id);
|
|
100
|
+
if (throttled) {
|
|
101
|
+
throttled.cancel();
|
|
102
|
+
this.progressThrottles.delete(task.id);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (task.doneAt) {
|
|
106
|
+
this.progressThrottles.delete(task.id);
|
|
107
|
+
this.tasks.delete(task.id);
|
|
108
|
+
}
|
|
109
|
+
if (task.status === import_constants.TASK_STATUS.SUCCEEDED) {
|
|
110
|
+
this.app.emit("workflow:dispatch");
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
onTaskAfterDelete = (task) => {
|
|
114
|
+
this.tasks.delete(task.id);
|
|
115
|
+
this.progressThrottles.delete(task.id);
|
|
116
|
+
const userId = task.createdById;
|
|
117
|
+
if (userId) {
|
|
118
|
+
this.app.emit("ws:sendToUser", {
|
|
119
|
+
userId,
|
|
120
|
+
message: {
|
|
121
|
+
type: "async-tasks:deleted",
|
|
122
|
+
payload: { id: task.id }
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
onTaskCancelSignal = async (data) => {
|
|
128
|
+
return this.cancelTask(data.id, true);
|
|
129
|
+
};
|
|
130
|
+
cleanup = async () => {
|
|
131
|
+
this.logger.debug("Running cleanup for completed tasks...");
|
|
132
|
+
const TaskRepo = this.app.db.getRepository("asyncTasks");
|
|
133
|
+
const tasksToCleanup = await TaskRepo.find({
|
|
134
|
+
fields: ["id"],
|
|
135
|
+
filter: {
|
|
136
|
+
$or: [
|
|
137
|
+
{
|
|
138
|
+
status: [import_constants.TASK_STATUS.SUCCEEDED, import_constants.TASK_STATUS.FAILED],
|
|
139
|
+
doneAt: {
|
|
140
|
+
$lt: new Date(Date.now() - this.cleanupDelay)
|
|
141
|
+
}
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
status: import_constants.TASK_STATUS.CANCELED
|
|
145
|
+
}
|
|
146
|
+
]
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
this.logger.debug(`Found ${tasksToCleanup.length} tasks to cleanup`);
|
|
150
|
+
for (const task of tasksToCleanup) {
|
|
151
|
+
this.tasks.delete(task.id);
|
|
152
|
+
this.progressThrottles.delete(task.id);
|
|
153
|
+
}
|
|
154
|
+
await TaskRepo.destroy({
|
|
155
|
+
filterByTk: tasksToCleanup.map((task) => task.id),
|
|
156
|
+
individualHooks: true
|
|
157
|
+
});
|
|
158
|
+
};
|
|
159
|
+
getThrottledProgressEmitter(taskId, userId) {
|
|
160
|
+
if (!this.progressThrottles.has(taskId)) {
|
|
161
|
+
this.progressThrottles.set(
|
|
162
|
+
taskId,
|
|
163
|
+
(0, import_lodash.throttle)(
|
|
164
|
+
(record) => {
|
|
165
|
+
this.app.emit("ws:sendToUser", {
|
|
166
|
+
userId,
|
|
167
|
+
message: {
|
|
168
|
+
type: "async-tasks:progress",
|
|
169
|
+
payload: record.toJSON()
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
},
|
|
173
|
+
500,
|
|
174
|
+
{ leading: true, trailing: true }
|
|
175
|
+
)
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
return this.progressThrottles.get(taskId);
|
|
179
|
+
}
|
|
53
180
|
setLogger(logger) {
|
|
54
181
|
this.logger = logger;
|
|
55
182
|
}
|
|
56
183
|
setApp(app) {
|
|
57
184
|
this.app = app;
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
autoStart: true
|
|
70
|
-
};
|
|
71
|
-
this.queue = new import_p_queue.default(this.queueOptions);
|
|
72
|
-
this.queue.on("idle", () => {
|
|
73
|
-
this.logger.debug("Task queue is idle");
|
|
74
|
-
this.emit("queueIdle");
|
|
75
|
-
});
|
|
76
|
-
this.queue.on("add", () => {
|
|
77
|
-
this.logger.debug(`Task added to queue. Size: ${this.queue.size}, Pending: ${this.queue.pending}`);
|
|
78
|
-
this.emit("queueAdd", { size: this.queue.size, pending: this.queue.pending });
|
|
185
|
+
const plugin = this.app.pm.get(import_plugin.default);
|
|
186
|
+
this.app.on("afterStart", () => {
|
|
187
|
+
this.app.eventQueue.subscribe(`${plugin.name}.task`, {
|
|
188
|
+
idle: this.idle,
|
|
189
|
+
process: this.onQueueTask,
|
|
190
|
+
concurrency: this.concurrency
|
|
191
|
+
});
|
|
192
|
+
this.app.pubSubManager.subscribe(`${plugin.name}.task.cancel`, this.onTaskCancelSignal);
|
|
193
|
+
if (!this.cleanupTimer) {
|
|
194
|
+
this.cleanupTimer = setInterval(this.cleanup, 60 * 1e3);
|
|
195
|
+
}
|
|
79
196
|
});
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
try {
|
|
85
|
-
this.logger.debug(`Starting execution of task ${task.taskId} from queue`);
|
|
86
|
-
await task.run();
|
|
87
|
-
} catch (error) {
|
|
88
|
-
this.logger.error(`Error executing task ${task.taskId} from queue: ${error.message}`);
|
|
89
|
-
}
|
|
197
|
+
this.app.on("beforeStop", () => {
|
|
198
|
+
if (this.cleanupTimer) {
|
|
199
|
+
clearInterval(this.cleanupTimer);
|
|
200
|
+
this.cleanupTimer = null;
|
|
90
201
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
this.
|
|
96
|
-
this.
|
|
97
|
-
this.emit("queuePaused");
|
|
202
|
+
this.app.eventQueue.unsubscribe(`${plugin.name}.task`);
|
|
203
|
+
this.app.pubSubManager.unsubscribe(`${plugin.name}.task.cancel`, this.onTaskCancelSignal);
|
|
204
|
+
});
|
|
205
|
+
this.app.db.on("asyncTasks.afterCreate", this.onTaskAfterCreate);
|
|
206
|
+
this.app.db.on("asyncTasks.afterUpdate", this.onTaskStatusChanged);
|
|
207
|
+
this.app.db.on("asyncTasks.afterDestroy", this.onTaskAfterDelete);
|
|
98
208
|
}
|
|
99
|
-
|
|
100
|
-
this.
|
|
101
|
-
this.
|
|
102
|
-
this.emit("queueResumed");
|
|
209
|
+
enqueueTask(task, queueOptions) {
|
|
210
|
+
const plugin = this.app.pm.get(import_plugin.default);
|
|
211
|
+
this.app.eventQueue.publish(`${plugin.name}.task`, { id: task.record.id }, queueOptions);
|
|
103
212
|
}
|
|
104
|
-
async cancelTask(taskId) {
|
|
213
|
+
async cancelTask(taskId, externally = false) {
|
|
105
214
|
const task = this.tasks.get(taskId);
|
|
106
|
-
if (
|
|
107
|
-
this.logger.
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
215
|
+
if (task) {
|
|
216
|
+
this.logger.info(`Cancelling task ${taskId}, type: ${task.constructor.name}`);
|
|
217
|
+
this.progressThrottles.delete(taskId);
|
|
218
|
+
task.cancel();
|
|
219
|
+
await task.record.update({
|
|
220
|
+
status: import_constants.TASK_STATUS.CANCELED,
|
|
221
|
+
doneAt: /* @__PURE__ */ new Date()
|
|
222
|
+
});
|
|
223
|
+
} else {
|
|
224
|
+
if (externally) {
|
|
225
|
+
this.logger.info(`Task ${taskId} not found on this instance, skip`);
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
this.logger.warn(`Task ${taskId} not found on this instance, broadcasting to other instances...`);
|
|
229
|
+
const plugin = this.app.pm.get(import_plugin.default);
|
|
230
|
+
this.app.pubSubManager.publish(`${plugin.name}.task.cancel`, { id: taskId }, { skipSelf: true });
|
|
231
|
+
const TaskRepo = this.app.db.getRepository("asyncTasks");
|
|
232
|
+
await TaskRepo.update({
|
|
233
|
+
filterByTk: taskId,
|
|
234
|
+
values: {
|
|
235
|
+
status: import_constants.TASK_STATUS.CANCELED,
|
|
236
|
+
doneAt: /* @__PURE__ */ new Date()
|
|
237
|
+
}
|
|
238
|
+
});
|
|
114
239
|
}
|
|
115
|
-
return task.cancel();
|
|
116
240
|
}
|
|
117
|
-
createTask(options) {
|
|
118
|
-
const taskType = this.taskTypes.get(
|
|
241
|
+
async createTask(data, { useQueue, ...options } = {}) {
|
|
242
|
+
const taskType = this.taskTypes.get(data.type);
|
|
119
243
|
if (!taskType) {
|
|
120
|
-
this.logger.error(`Task type not found: ${
|
|
121
|
-
throw new Error(`Task type ${
|
|
244
|
+
this.logger.error(`Task type not found: ${data.type}, params: ${JSON.stringify(data.params)}`);
|
|
245
|
+
throw new Error(`Task type ${data.type} not found`);
|
|
122
246
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
);
|
|
128
|
-
const task = new taskType(options.params, options.tags);
|
|
129
|
-
task.title = options.title;
|
|
130
|
-
task.setLogger(this.logger);
|
|
131
|
-
task.setApp(this.app);
|
|
132
|
-
task.setContext(options.context);
|
|
133
|
-
this.tasks.set(task.taskId, task);
|
|
134
|
-
this.logger.info(
|
|
135
|
-
`Created new task ${task.taskId} of type ${options.type}, params: ${JSON.stringify(
|
|
136
|
-
options.params
|
|
137
|
-
)}, tags: ${JSON.stringify(options.tags)}, title: ${task.title}`
|
|
138
|
-
);
|
|
139
|
-
this.emit("taskCreated", { task });
|
|
140
|
-
if (options.useQueue) {
|
|
141
|
-
this.enqueueTask(task);
|
|
142
|
-
}
|
|
143
|
-
task.on("progress", (progress) => {
|
|
144
|
-
this.logger.debug(`Task ${task.taskId} progress: ${progress}`);
|
|
145
|
-
this.emit("taskProgress", { task, progress });
|
|
146
|
-
});
|
|
147
|
-
task.on("statusChange", (status) => {
|
|
148
|
-
if (["success", "failed"].includes(status.type)) {
|
|
149
|
-
this.scheduleCleanup(task.taskId);
|
|
150
|
-
} else if (status.type === "cancelled") {
|
|
151
|
-
this.tasks.delete(task.taskId);
|
|
152
|
-
}
|
|
153
|
-
this.emit("taskStatusChange", { task, status });
|
|
247
|
+
const values = taskType.defaults({
|
|
248
|
+
id: (0, import_crypto.randomUUID)(),
|
|
249
|
+
status: import_constants.TASK_STATUS.PENDING,
|
|
250
|
+
...data
|
|
154
251
|
});
|
|
252
|
+
const DBTaskModel = this.app.db.getModel("asyncTasks");
|
|
253
|
+
const record = await DBTaskModel.create(values, options);
|
|
254
|
+
this.logger.debug(`Creating task of type: ${data.type}`);
|
|
255
|
+
this.logger.debug(`Task data: ${JSON.stringify(data)}`);
|
|
256
|
+
const task = new taskType(record);
|
|
257
|
+
this.logger.info(`New task of type: ${data.type} created as ${record.id}`);
|
|
258
|
+
if (useQueue) {
|
|
259
|
+
this.logger.debug(`New task ${record.id} will be sent to queue for processing`);
|
|
260
|
+
const queueOptions = typeof useQueue === "object" ? useQueue : {};
|
|
261
|
+
this.enqueueTask(task, queueOptions);
|
|
262
|
+
}
|
|
155
263
|
return task;
|
|
156
264
|
}
|
|
157
265
|
getTask(taskId) {
|
|
@@ -160,7 +268,7 @@ class BaseTaskManager extends import_events.EventEmitter {
|
|
|
160
268
|
this.logger.debug(`Task not found: ${taskId}`);
|
|
161
269
|
return void 0;
|
|
162
270
|
}
|
|
163
|
-
this.logger.debug(`Retrieved task ${taskId}, type: ${task.constructor.name}, status: ${task.status
|
|
271
|
+
this.logger.debug(`Retrieved task ${taskId}, type: ${task.constructor.name}, status: ${task.record.status}`);
|
|
164
272
|
return task;
|
|
165
273
|
}
|
|
166
274
|
async getTaskStatus(taskId) {
|
|
@@ -169,20 +277,41 @@ class BaseTaskManager extends import_events.EventEmitter {
|
|
|
169
277
|
this.logger.warn(`Attempted to get status of non-existent task ${taskId}`);
|
|
170
278
|
throw new Error(`Task ${taskId} not found`);
|
|
171
279
|
}
|
|
172
|
-
this.logger.debug(`Getting status for task ${taskId}, current status: ${task.status
|
|
173
|
-
return task.status;
|
|
280
|
+
this.logger.debug(`Getting status for task ${taskId}, current status: ${task.record.status}`);
|
|
281
|
+
return task.record.status;
|
|
174
282
|
}
|
|
175
283
|
registerTaskType(taskType) {
|
|
176
284
|
this.logger.debug(`Registering task type: ${taskType.type}`);
|
|
177
285
|
this.taskTypes.set(taskType.type, taskType);
|
|
178
286
|
}
|
|
179
|
-
async
|
|
180
|
-
this.
|
|
181
|
-
const
|
|
182
|
-
|
|
287
|
+
async prepareTask(id) {
|
|
288
|
+
const TaskRepo = this.app.db.getRepository("asyncTasks");
|
|
289
|
+
const record = await TaskRepo.findOne({
|
|
290
|
+
filterByTk: id
|
|
183
291
|
});
|
|
184
|
-
this.
|
|
185
|
-
|
|
292
|
+
const taskType = this.taskTypes.get(record.type);
|
|
293
|
+
if (!taskType) {
|
|
294
|
+
this.logger.error(`Task type not found: ${record.type} (#${id})`);
|
|
295
|
+
throw new Error(`Task type ${record.type} not found`);
|
|
296
|
+
}
|
|
297
|
+
const task = new taskType(record);
|
|
298
|
+
return task;
|
|
299
|
+
}
|
|
300
|
+
async runTask(task) {
|
|
301
|
+
if (task.record.status === import_constants.TASK_STATUS.PENDING) {
|
|
302
|
+
task.setLogger(this.logger);
|
|
303
|
+
task.setApp(this.app);
|
|
304
|
+
task.onProgress = this.onTaskProgress;
|
|
305
|
+
this.tasks.set(task.record.id, task);
|
|
306
|
+
try {
|
|
307
|
+
this.logger.debug(`Starting execution of task ${task.record.id} from queue`);
|
|
308
|
+
await task.run();
|
|
309
|
+
} catch (error) {
|
|
310
|
+
this.logger.error(`Error executing task ${task.record.id} from queue: ${error.message}`);
|
|
311
|
+
} finally {
|
|
312
|
+
this.tasks.delete(task.record.id);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
186
315
|
}
|
|
187
316
|
}
|
|
188
317
|
// Annotate the CommonJS export names for ESM import in node:
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This file is part of the NocoBase (R) project.
|
|
3
|
+
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
|
|
4
|
+
* Authors: NocoBase Team.
|
|
5
|
+
*
|
|
6
|
+
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
|
|
7
|
+
* For more information, please refer to: https://www.nocobase.com/agreement.
|
|
8
|
+
*/
|
|
9
|
+
import { CollectionOptions } from '@nocobase/database';
|
|
10
|
+
declare const _default: CollectionOptions;
|
|
11
|
+
export default _default;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This file is part of the NocoBase (R) project.
|
|
3
|
+
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
|
|
4
|
+
* Authors: NocoBase Team.
|
|
5
|
+
*
|
|
6
|
+
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
|
|
7
|
+
* For more information, please refer to: https://www.nocobase.com/agreement.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
var __create = Object.create;
|
|
11
|
+
var __defProp = Object.defineProperty;
|
|
12
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
13
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
14
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
15
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
16
|
+
var __export = (target, all) => {
|
|
17
|
+
for (var name in all)
|
|
18
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
19
|
+
};
|
|
20
|
+
var __copyProps = (to, from, except, desc) => {
|
|
21
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
22
|
+
for (let key of __getOwnPropNames(from))
|
|
23
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
24
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
25
|
+
}
|
|
26
|
+
return to;
|
|
27
|
+
};
|
|
28
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
29
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
30
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
31
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
32
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
33
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
34
|
+
mod
|
|
35
|
+
));
|
|
36
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
37
|
+
var asyncTasks_exports = {};
|
|
38
|
+
__export(asyncTasks_exports, {
|
|
39
|
+
default: () => asyncTasks_default
|
|
40
|
+
});
|
|
41
|
+
module.exports = __toCommonJS(asyncTasks_exports);
|
|
42
|
+
var import_asyncTasks = __toESM(require("../../common/collections/asyncTasks"));
|
|
43
|
+
var asyncTasks_default = import_asyncTasks.default;
|
|
@@ -1,8 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This file is part of the NocoBase (R) project.
|
|
3
|
+
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
|
|
4
|
+
* Authors: NocoBase Team.
|
|
5
|
+
*
|
|
6
|
+
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
|
|
7
|
+
* For more information, please refer to: https://www.nocobase.com/agreement.
|
|
8
|
+
*/
|
|
1
9
|
/// <reference types="node" />
|
|
2
10
|
import { Worker } from 'worker_threads';
|
|
3
11
|
import { TaskType } from './task-type';
|
|
4
12
|
export declare class CommandTaskType extends TaskType {
|
|
5
13
|
static type: string;
|
|
6
14
|
workerThread: Worker;
|
|
15
|
+
static defaults(data: any): any;
|
|
7
16
|
execute(): Promise<unknown>;
|
|
8
17
|
}
|
|
@@ -47,9 +47,14 @@ var import_task_type = require("./task-type");
|
|
|
47
47
|
class CommandTaskType extends import_task_type.TaskType {
|
|
48
48
|
static type = "command";
|
|
49
49
|
workerThread;
|
|
50
|
+
static defaults(data) {
|
|
51
|
+
var _a;
|
|
52
|
+
(_a = data.argv) == null ? void 0 : _a.push(`--taskId=${data.id}`);
|
|
53
|
+
return data;
|
|
54
|
+
}
|
|
50
55
|
async execute() {
|
|
51
56
|
var _a;
|
|
52
|
-
const { argv } = this.
|
|
57
|
+
const { argv } = this.record.params;
|
|
53
58
|
const isDev = (((_a = import_node_process.default.argv[1]) == null ? void 0 : _a.endsWith(".ts")) || import_node_process.default.argv[1].includes("tinypool")) ?? false;
|
|
54
59
|
const appRoot = import_node_process.default.env.APP_PACKAGE_ROOT || "packages/core/app";
|
|
55
60
|
const workerPath = import_path.default.resolve(import_node_process.default.cwd(), appRoot, isDev ? "src/index.ts" : "lib/index.js");
|
|
@@ -57,7 +62,7 @@ class CommandTaskType extends import_task_type.TaskType {
|
|
|
57
62
|
var _a2, _b;
|
|
58
63
|
try {
|
|
59
64
|
(_a2 = this.logger) == null ? void 0 : _a2.info(
|
|
60
|
-
`Creating worker for task ${this.
|
|
65
|
+
`Creating worker for task ${this.record.id} - path: ${workerPath}, argv: ${JSON.stringify(
|
|
61
66
|
argv
|
|
62
67
|
)}, isDev: ${isDev}`
|
|
63
68
|
);
|
|
@@ -65,38 +70,44 @@ class CommandTaskType extends import_task_type.TaskType {
|
|
|
65
70
|
execArgv: isDev ? ["--require", "tsx/cjs"] : [],
|
|
66
71
|
workerData: {
|
|
67
72
|
argv
|
|
73
|
+
},
|
|
74
|
+
env: {
|
|
75
|
+
...import_node_process.default.env,
|
|
76
|
+
WORKER_MODE: "-"
|
|
68
77
|
}
|
|
69
78
|
});
|
|
70
79
|
this.workerThread = worker;
|
|
71
|
-
(_b = this.logger) == null ? void 0 : _b.debug(`Worker created successfully for task ${this.
|
|
80
|
+
(_b = this.logger) == null ? void 0 : _b.debug(`Worker created successfully for task ${this.record.id}`);
|
|
72
81
|
let isCancelling = false;
|
|
73
82
|
this.abortController.signal.addEventListener("abort", () => {
|
|
74
83
|
var _a3;
|
|
75
84
|
isCancelling = true;
|
|
76
|
-
(_a3 = this.logger) == null ? void 0 : _a3.info(`Terminating worker for task ${this.
|
|
85
|
+
(_a3 = this.logger) == null ? void 0 : _a3.info(`Terminating worker for task ${this.record.id} due to cancellation`);
|
|
77
86
|
worker.terminate();
|
|
78
87
|
});
|
|
79
88
|
worker.on("message", (message) => {
|
|
80
89
|
var _a3, _b2;
|
|
81
|
-
(_a3 = this.logger) == null ? void 0 : _a3.
|
|
90
|
+
(_a3 = this.logger) == null ? void 0 : _a3.trace(`Worker message received for task ${this.record.id} - type: ${message.type}`);
|
|
82
91
|
if (message.type === "progress") {
|
|
83
92
|
this.reportProgress(message.payload);
|
|
84
93
|
}
|
|
85
94
|
if (message.type === "success") {
|
|
86
95
|
(_b2 = this.logger) == null ? void 0 : _b2.info(
|
|
87
|
-
`Worker completed successfully for task ${this.
|
|
96
|
+
`Worker completed successfully for task ${this.record.id} with payload: ${JSON.stringify(
|
|
97
|
+
message.payload
|
|
98
|
+
)}`
|
|
88
99
|
);
|
|
89
100
|
resolve(message.payload);
|
|
90
101
|
}
|
|
91
102
|
});
|
|
92
103
|
worker.on("error", (error) => {
|
|
93
104
|
var _a3;
|
|
94
|
-
(_a3 = this.logger) == null ? void 0 : _a3.error(`Worker error for task ${this.
|
|
105
|
+
(_a3 = this.logger) == null ? void 0 : _a3.error(`Worker error for task ${this.record.id}`, error);
|
|
95
106
|
reject(error);
|
|
96
107
|
});
|
|
97
108
|
worker.on("exit", (code) => {
|
|
98
109
|
var _a3;
|
|
99
|
-
(_a3 = this.logger) == null ? void 0 : _a3.info(`Worker exited for task ${this.
|
|
110
|
+
(_a3 = this.logger) == null ? void 0 : _a3.info(`Worker exited for task ${this.record.id} with code ${code}`);
|
|
100
111
|
if (isCancelling) {
|
|
101
112
|
reject(new import_async_task_manager.CancelError());
|
|
102
113
|
} else if (code !== 0) {
|
|
@@ -6,62 +6,25 @@
|
|
|
6
6
|
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
|
|
7
7
|
* For more information, please refer to: https://www.nocobase.com/agreement.
|
|
8
8
|
*/
|
|
9
|
-
/// <reference types="node" />
|
|
10
9
|
import { Logger } from '@nocobase/logger';
|
|
11
|
-
import { ITask, TaskConstructor } from './task';
|
|
12
|
-
import { Application } from '@nocobase/server';
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
export interface CreateTaskOptions {
|
|
16
|
-
|
|
17
|
-
params: TaskOptions;
|
|
18
|
-
tags?: Record<string, string>;
|
|
19
|
-
title?: {
|
|
20
|
-
actionType: string;
|
|
21
|
-
collection: string;
|
|
22
|
-
dataSource: string;
|
|
23
|
-
};
|
|
10
|
+
import { ITask, TaskConstructor, TaskModel } from './task';
|
|
11
|
+
import { Application, QueueMessageOptions } from '@nocobase/server';
|
|
12
|
+
import { TaskId, TaskStatus } from '../../common/types';
|
|
13
|
+
import { Transactionable } from '@nocobase/database';
|
|
14
|
+
export interface CreateTaskOptions extends Transactionable {
|
|
15
|
+
useQueue?: boolean | QueueMessageOptions;
|
|
24
16
|
context?: any;
|
|
25
|
-
useQueue?: boolean;
|
|
26
17
|
}
|
|
27
|
-
export
|
|
28
|
-
|
|
29
|
-
export type ProgressIndicator = 'spinner' | 'progress' | 'success' | 'error';
|
|
30
|
-
export interface PendingStatus {
|
|
31
|
-
type: 'pending';
|
|
32
|
-
indicator?: 'spinner';
|
|
33
|
-
}
|
|
34
|
-
export interface SuccessStatus<T = any> {
|
|
35
|
-
type: 'success';
|
|
36
|
-
indicator?: 'success';
|
|
37
|
-
resultType?: 'file' | 'data';
|
|
38
|
-
payload?: T;
|
|
39
|
-
}
|
|
40
|
-
export interface RunningStatus {
|
|
41
|
-
type: 'running';
|
|
42
|
-
indicator: 'progress';
|
|
43
|
-
}
|
|
44
|
-
export interface FailedStatus {
|
|
45
|
-
type: 'failed';
|
|
46
|
-
indicator?: 'error';
|
|
47
|
-
errors: Array<{
|
|
48
|
-
message: string;
|
|
49
|
-
code?: number;
|
|
50
|
-
}>;
|
|
51
|
-
}
|
|
52
|
-
export interface CancelledStatus {
|
|
53
|
-
type: 'cancelled';
|
|
54
|
-
}
|
|
55
|
-
export interface AsyncTasksManager extends EventEmitter {
|
|
56
|
-
queue: any;
|
|
18
|
+
export interface AsyncTasksManager {
|
|
19
|
+
concurrency: number;
|
|
57
20
|
setLogger(logger: Logger): void;
|
|
58
21
|
setApp(app: Application): void;
|
|
59
22
|
registerTaskType(taskType: TaskConstructor): void;
|
|
60
|
-
createTask
|
|
61
|
-
|
|
62
|
-
cancelTask(taskId: TaskId): Promise<boolean>;
|
|
23
|
+
createTask(data: Omit<TaskModel, 'id'>, options?: CreateTaskOptions): Promise<ITask>;
|
|
24
|
+
cancelTask(taskId: TaskId): Promise<void>;
|
|
63
25
|
getTaskStatus(taskId: TaskId): Promise<TaskStatus>;
|
|
64
26
|
getTask(taskId: TaskId): ITask | undefined;
|
|
27
|
+
runTask(task: ITask): Promise<void>;
|
|
65
28
|
}
|
|
66
29
|
export declare class CancelError extends Error {
|
|
67
30
|
constructor(message?: string);
|
|
@@ -6,41 +6,43 @@
|
|
|
6
6
|
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
|
|
7
7
|
* For more information, please refer to: https://www.nocobase.com/agreement.
|
|
8
8
|
*/
|
|
9
|
-
/// <reference types="node" />
|
|
10
9
|
import { Logger } from '@nocobase/logger';
|
|
11
|
-
import { TaskStatus } from './async-task-manager';
|
|
12
|
-
import { EventEmitter } from 'events';
|
|
13
10
|
import { Application } from '@nocobase/server';
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
import { Model } from '@nocobase/database';
|
|
12
|
+
import { TaskId, TaskStatus } from '../../common/types';
|
|
13
|
+
export declare class TaskModel extends Model {
|
|
14
|
+
id: TaskId;
|
|
15
|
+
origin: string;
|
|
16
|
+
type: string;
|
|
17
|
+
title?: string;
|
|
18
|
+
params: Record<string, any>;
|
|
19
|
+
tags?: Record<string, string>;
|
|
16
20
|
status: TaskStatus;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
context?: any;
|
|
21
|
+
progressTotal?: number;
|
|
22
|
+
progressCurrent?: number;
|
|
23
|
+
createdAt?: Date;
|
|
24
|
+
startedAt?: Date;
|
|
25
|
+
doneAt?: Date;
|
|
26
|
+
createdById?: number;
|
|
27
|
+
}
|
|
28
|
+
export interface ITask {
|
|
29
|
+
record: TaskModel;
|
|
30
|
+
onProgress: (record: TaskModel) => void;
|
|
28
31
|
setLogger(logger: Logger): void;
|
|
29
32
|
setApp(app: Application): void;
|
|
30
|
-
|
|
31
|
-
cancel(
|
|
32
|
-
|
|
33
|
+
isCanceled: boolean;
|
|
34
|
+
cancel(options?: {
|
|
35
|
+
silent?: boolean;
|
|
36
|
+
}): any;
|
|
33
37
|
execute(): Promise<any>;
|
|
34
38
|
reportProgress(progress: {
|
|
35
39
|
total: number;
|
|
36
40
|
current: number;
|
|
37
41
|
}): void;
|
|
38
42
|
run(): Promise<void>;
|
|
39
|
-
toJSON(
|
|
40
|
-
raw?: boolean;
|
|
41
|
-
}): any;
|
|
43
|
+
toJSON<TaskModelAttributes>(): TaskModelAttributes;
|
|
42
44
|
}
|
|
43
45
|
export interface TaskConstructor {
|
|
44
46
|
type: string;
|
|
45
|
-
new (
|
|
47
|
+
new (model: TaskModel): ITask;
|
|
46
48
|
}
|