@kne/fastify-task 3.0.0-alpha.0 → 3.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1052 -255
- package/index.js +14 -4
- package/libs/controllers/main.js +4 -3
- package/libs/helpers/task-service-context.js +556 -0
- package/libs/models/task.js +6 -1
- package/libs/services/append.js +45 -0
- package/libs/services/callback.js +109 -0
- package/libs/services/executor.js +10 -0
- package/libs/services/main.js +7 -674
- package/libs/services/scheduler.js +230 -0
- package/libs/services/statistics.js +614 -38
- package/libs/services/task.js +224 -0
- package/package.json +3 -2
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
const fp = require('fastify-plugin');
|
|
2
|
+
const getTaskServiceContext = require('../helpers/task-service-context');
|
|
3
|
+
|
|
4
|
+
module.exports = fp(async (fastify, options) => {
|
|
5
|
+
const context = getTaskServiceContext(fastify, options);
|
|
6
|
+
const { models, Op } = context;
|
|
7
|
+
|
|
8
|
+
const getRunningRecoveryBefore = () => {
|
|
9
|
+
const recoverAfter = Number.isInteger(options.recoverRunningTaskAfter) && options.recoverRunningTaskAfter >= 0 ? options.recoverRunningTaskAfter : 30 * 60 * 1000;
|
|
10
|
+
return new Date(Date.now() - recoverAfter);
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
const getRunningResetWhere = ({ staleOnly = false, before } = {}) => {
|
|
14
|
+
const where = { status: 'running' };
|
|
15
|
+
if (staleOnly) {
|
|
16
|
+
where[Op.or] = [{ startedAt: { [Op.lte]: before || getRunningRecoveryBefore() } }, { startedAt: null }];
|
|
17
|
+
}
|
|
18
|
+
return where;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const resetAll = async (props = {}) => {
|
|
22
|
+
const [affectedCount] = await models.task.update({ status: 'pending' }, { where: getRunningResetWhere(props) });
|
|
23
|
+
return affectedCount;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
const claimTask = async (task, { allowChild = false } = {}) => {
|
|
27
|
+
const now = new Date();
|
|
28
|
+
const [affectedCount] = await models.task.update(
|
|
29
|
+
{
|
|
30
|
+
status: 'running',
|
|
31
|
+
startedAt: now,
|
|
32
|
+
progress: 0,
|
|
33
|
+
error: null,
|
|
34
|
+
output: null,
|
|
35
|
+
pollCount: 0,
|
|
36
|
+
pollResults: [],
|
|
37
|
+
completedAt: null
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
where: Object.assign(
|
|
41
|
+
{
|
|
42
|
+
id: task.id,
|
|
43
|
+
runnerType: 'system',
|
|
44
|
+
status: 'pending',
|
|
45
|
+
startTime: {
|
|
46
|
+
[Op.lte]: now
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
allowChild ? {} : { parentTaskId: null }
|
|
50
|
+
)
|
|
51
|
+
}
|
|
52
|
+
);
|
|
53
|
+
if (affectedCount !== 1) return null;
|
|
54
|
+
return await models.task.findByPk(task.id);
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const processSystemTask = async (task, { claimed = false } = {}) => {
|
|
58
|
+
if (!claimed) {
|
|
59
|
+
task = await claimTask(task, { allowChild: true });
|
|
60
|
+
if (!task) return false;
|
|
61
|
+
}
|
|
62
|
+
const addLog = props => {
|
|
63
|
+
return fastify[options.name].services.log(Object.assign({}, props, { taskId: task.id }));
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
const createTimeout = ms => new Promise((_, reject) => setTimeout(() => reject(new Error(`任务执行超时(${ms}ms)`)), ms));
|
|
67
|
+
|
|
68
|
+
const executorPromise = context.executor({ type: task.type, scriptName: task.scriptName, task, log: addLog });
|
|
69
|
+
const racePromise = options.taskTimeout > 0 ? Promise.race([executorPromise, createTimeout(options.taskTimeout)]) : executorPromise;
|
|
70
|
+
|
|
71
|
+
return racePromise
|
|
72
|
+
.then(async result => {
|
|
73
|
+
if (result === false) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
await context.finalizeSuccess({ task, output: result });
|
|
77
|
+
})
|
|
78
|
+
.catch(async e => {
|
|
79
|
+
const currentRetryCount = (task.retryCount || 0) + 1;
|
|
80
|
+
const maxRetries = task.maxRetries || 0;
|
|
81
|
+
if (currentRetryCount <= maxRetries) {
|
|
82
|
+
const baseDelay = options.retryBaseDelay || 5000;
|
|
83
|
+
const backoffDelay = baseDelay * Math.pow(2, currentRetryCount - 1);
|
|
84
|
+
const retryScheduled = await context.updateTaskByStatus({
|
|
85
|
+
task,
|
|
86
|
+
allowedStatuses: ['running'],
|
|
87
|
+
updateData: {
|
|
88
|
+
status: 'pending',
|
|
89
|
+
retryCount: currentRetryCount,
|
|
90
|
+
error: (e.stack || '').replaceAll(process.cwd(), '/server'),
|
|
91
|
+
startTime: new Date(Date.now() + backoffDelay),
|
|
92
|
+
completedAt: null
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
if (retryScheduled) {
|
|
96
|
+
fastify.log.info(`任务 ${task.id} 第 ${currentRetryCount} 次重试,${backoffDelay}ms 后执行`);
|
|
97
|
+
}
|
|
98
|
+
} else {
|
|
99
|
+
await context.failTask({ task, error: e, updateData: { retryCount: currentRetryCount } });
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
const triggerChildTasks = async parentTask => {
|
|
105
|
+
const childTasks = await models.task.findAll({
|
|
106
|
+
where: {
|
|
107
|
+
parentTaskId: parentTask.id,
|
|
108
|
+
status: 'pending'
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
if (childTasks.length === 0) return;
|
|
112
|
+
|
|
113
|
+
fastify.log.info(`父任务 ${parentTask.id} 完成,激活 ${childTasks.length} 个子任务`);
|
|
114
|
+
for (const child of childTasks) {
|
|
115
|
+
if (child.runnerType === 'system') {
|
|
116
|
+
await processSystemTask(child);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
const checkTimeout = async () => {
|
|
122
|
+
const now = new Date();
|
|
123
|
+
const timedOutTasks = await models.task.findAll({
|
|
124
|
+
where: {
|
|
125
|
+
status: { [Op.in]: ['running', 'waiting'] },
|
|
126
|
+
timeout: { [Op.gt]: 0 },
|
|
127
|
+
startedAt: { [Op.ne]: null }
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
const tasksToFail = timedOutTasks.filter(task => {
|
|
132
|
+
const elapsed = now.getTime() - new Date(task.startedAt).getTime();
|
|
133
|
+
return elapsed > task.timeout;
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
if (tasksToFail.length === 0) return;
|
|
137
|
+
|
|
138
|
+
fastify.log.info(`检测到 ${tasksToFail.length} 个超时任务`);
|
|
139
|
+
|
|
140
|
+
for (const task of tasksToFail) {
|
|
141
|
+
const elapsed = now.getTime() - new Date(task.startedAt).getTime();
|
|
142
|
+
await context.failTask({ task, error: `任务超时:已执行 ${elapsed}ms,超过设定的 ${task.timeout}ms 超时时间`, updateData: { completedAt: now } });
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
const claimPendingTasks = async limit => {
|
|
147
|
+
if (limit <= 0) return [];
|
|
148
|
+
const now = new Date();
|
|
149
|
+
const pendingTasks = await models.task.findAll({
|
|
150
|
+
where: {
|
|
151
|
+
runnerType: 'system',
|
|
152
|
+
status: 'pending',
|
|
153
|
+
parentTaskId: null,
|
|
154
|
+
startTime: {
|
|
155
|
+
[Op.lte]: now
|
|
156
|
+
}
|
|
157
|
+
},
|
|
158
|
+
order: [
|
|
159
|
+
['priority', 'DESC'],
|
|
160
|
+
['startTime', 'ASC']
|
|
161
|
+
],
|
|
162
|
+
limit: limit
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
const claimedTasks = [];
|
|
166
|
+
for (const task of pendingTasks) {
|
|
167
|
+
const [affectedCount] = await models.task.update(
|
|
168
|
+
{
|
|
169
|
+
status: 'running',
|
|
170
|
+
startedAt: now,
|
|
171
|
+
progress: 0,
|
|
172
|
+
error: null,
|
|
173
|
+
output: null,
|
|
174
|
+
pollCount: 0,
|
|
175
|
+
pollResults: [],
|
|
176
|
+
completedAt: null
|
|
177
|
+
},
|
|
178
|
+
{
|
|
179
|
+
where: {
|
|
180
|
+
id: task.id,
|
|
181
|
+
runnerType: 'system',
|
|
182
|
+
status: 'pending',
|
|
183
|
+
parentTaskId: null,
|
|
184
|
+
startTime: {
|
|
185
|
+
[Op.lte]: now
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
);
|
|
190
|
+
if (affectedCount === 1) {
|
|
191
|
+
const claimedTask = await models.task.findByPk(task.id);
|
|
192
|
+
if (claimedTask) {
|
|
193
|
+
claimedTasks.push(claimedTask);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return claimedTasks;
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
const runner = async () => {
|
|
201
|
+
await checkTimeout();
|
|
202
|
+
|
|
203
|
+
const runningTaskCount = await models.task.count({
|
|
204
|
+
where: {
|
|
205
|
+
runnerType: 'system',
|
|
206
|
+
status: 'running'
|
|
207
|
+
}
|
|
208
|
+
});
|
|
209
|
+
const limit = options.limit - runningTaskCount;
|
|
210
|
+
if (limit <= 0) {
|
|
211
|
+
fastify.log.info(`当前运行中的系统任务数(${runningTaskCount})已达上限(${options.limit}),暂不执行新任务`);
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
const pendingTasks = await claimPendingTasks(limit);
|
|
215
|
+
|
|
216
|
+
if (pendingTasks.length > 0) {
|
|
217
|
+
fastify.log.info(`本轮执行 ${pendingTasks.length} 个待处理的系统任务`);
|
|
218
|
+
await Promise.all(pendingTasks.map(async task => processSystemTask(task, { claimed: true })));
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
Object.assign(fastify[options.name].services, {
|
|
223
|
+
runner,
|
|
224
|
+
resetAll,
|
|
225
|
+
claimPendingTasks,
|
|
226
|
+
claimTask,
|
|
227
|
+
triggerChildTasks,
|
|
228
|
+
processSystemTask
|
|
229
|
+
});
|
|
230
|
+
});
|