echoapi-cron-scheduler-batch 1.0.10 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +329 -350
- package/index_bak1.js +465 -0
- package/package.json +1 -1
package/index.js
CHANGED
|
@@ -1,3 +1,18 @@
|
|
|
1
|
+
const originalLog = console.log;
|
|
2
|
+
console.log = function (...args) {
|
|
3
|
+
// 检查第一个参数是否包含 faker-js 或 node_modules 路径相关的字符串
|
|
4
|
+
const firstArg = String(args[0]);
|
|
5
|
+
if (
|
|
6
|
+
firstArg.includes('node_modules') &&
|
|
7
|
+
(firstArg.includes('@faker-js') || firstArg.includes('Module {'))
|
|
8
|
+
) {
|
|
9
|
+
// 匹配到垃圾日志,直接丢弃,不执行打印
|
|
10
|
+
return;
|
|
11
|
+
}
|
|
12
|
+
// 正常的业务日志,调用原始方法打印
|
|
13
|
+
originalLog.apply(console, args);
|
|
14
|
+
};
|
|
15
|
+
// ==================================
|
|
1
16
|
const cluster = require('cluster');
|
|
2
17
|
const schedule = require('node-schedule');
|
|
3
18
|
const _ = require('lodash');
|
|
@@ -5,144 +20,64 @@ const axios = require('axios');
|
|
|
5
20
|
const path = require('path');
|
|
6
21
|
const os = require('os');
|
|
7
22
|
const JSON5 = require('json5');
|
|
8
|
-
const Database = require('better-sqlite3');
|
|
9
23
|
const fs = require('fs');
|
|
10
|
-
const { v4: uuidv4 } = require('uuid');
|
|
11
|
-
const { execSync } = require('child_process');
|
|
12
24
|
|
|
13
25
|
/**
|
|
14
|
-
* CronScheduler -
|
|
26
|
+
* CronScheduler - 分布式原子文件调度器 (生产全功能版)
|
|
15
27
|
*/
|
|
16
28
|
class CronScheduler {
|
|
17
29
|
constructor(config = {}) {
|
|
18
|
-
|
|
30
|
+
console.log('--------------------------------------------------');
|
|
31
|
+
console.log('🚀 [Init] CronScheduler 构造函数开始初始化...');
|
|
32
|
+
|
|
19
33
|
this.apiUrl = config.apiUrl || process?.env["OPENAPI_DOMAIN"] || "https://ee.apipost.cc";
|
|
20
34
|
|
|
21
|
-
//
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
35
|
+
// 初始化路径:建议指向 NFS 挂载目录
|
|
36
|
+
const sharedRoot = config.sharedPath || path.resolve(process?.env?.TEMP_DIR || os.tmpdir(), 'shared_scheduler');
|
|
37
|
+
this.paths = {
|
|
38
|
+
jobs: path.join(sharedRoot, 'jobs'),
|
|
39
|
+
locks: path.join(sharedRoot, 'locks'),
|
|
40
|
+
results: path.join(sharedRoot, 'results'),
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
if (cluster.isPrimary) {
|
|
44
|
+
process.setMaxListeners(300);
|
|
45
|
+
this._initDirectories();
|
|
46
|
+
console.log(`🏠 [Master] 调度中心就绪 | 节点: ${os.hostname()} | 目录: ${sharedRoot}`);
|
|
25
47
|
}
|
|
26
48
|
|
|
27
|
-
this.dbFile = config.dbPath || path.join(dataDir, 'echoapi-batch-tasks.sqlite');
|
|
28
49
|
this.workerNum = config.workerNum || os.cpus().length;
|
|
29
|
-
|
|
30
|
-
// 结果聚合收集器 (仅在 Master 内存中存在)
|
|
31
50
|
this.resultCollector = new Map();
|
|
32
|
-
|
|
33
|
-
// 2. 数据库连接与安全自检 (仅 Master 进程持有写连接)
|
|
34
|
-
if (cluster.isPrimary) {
|
|
35
|
-
this._connectDB();
|
|
36
|
-
}
|
|
37
51
|
}
|
|
38
52
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
// 预检:如果是 0 字节损坏文件,直接删除
|
|
45
|
-
if (fs.existsSync(this.dbFile) && fs.statSync(this.dbFile).size === 0) {
|
|
46
|
-
fs.unlinkSync(this.dbFile);
|
|
53
|
+
_initDirectories() {
|
|
54
|
+
Object.values(this.paths).forEach(dir => {
|
|
55
|
+
if (!fs.existsSync(dir)) {
|
|
56
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
57
|
+
console.log(`📁 [System] 创建目录: ${dir}`);
|
|
47
58
|
}
|
|
48
|
-
|
|
49
|
-
this.db = new Database(this.dbFile, { timeout: 10000 });
|
|
50
|
-
|
|
51
|
-
// 【核心】执行完整性自检,解决 malformed 和 not a database 问题
|
|
52
|
-
const check = this.db.pragma('integrity_check');
|
|
53
|
-
if (check[0].integrity_check !== 'ok') {
|
|
54
|
-
throw new Error('SQLite integrity check failed');
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
this._initDB();
|
|
58
|
-
console.log(`[Database] Connection verified and initialized.`);
|
|
59
|
-
} catch (e) {
|
|
60
|
-
console.error(`[Database Critical] 数据库损坏(Malformed/NotADb): ${e.message}`);
|
|
61
|
-
if (this.db) this.db.close();
|
|
62
|
-
|
|
63
|
-
// 自动备份损坏文件,腾出位置重建
|
|
64
|
-
const backupPath = `${this.dbFile}.corrupted_${Date.now()}`;
|
|
65
|
-
if (fs.existsSync(this.dbFile)) {
|
|
66
|
-
fs.renameSync(this.dbFile, backupPath);
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
// 重建数据库
|
|
70
|
-
this.db = new Database(this.dbFile);
|
|
71
|
-
this._initDB();
|
|
72
|
-
console.warn(`[Database] 坏库已隔离至 ${backupPath},新库已就绪。`);
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* 初始化表结构与性能参数
|
|
78
|
-
*/
|
|
79
|
-
_initDB() {
|
|
80
|
-
// 性能调优:WAL 模式能极大减少多进程读取时的锁竞争
|
|
81
|
-
this.db.pragma('journal_mode = WAL');
|
|
82
|
-
this.db.pragma('synchronous = NORMAL');
|
|
83
|
-
this.db.pragma('busy_timeout = 5000');
|
|
84
|
-
|
|
85
|
-
// 1. 任务配置表
|
|
86
|
-
this.db.prepare(`
|
|
87
|
-
CREATE TABLE IF NOT EXISTS jobs (
|
|
88
|
-
job_id TEXT PRIMARY KEY,
|
|
89
|
-
name TEXT,
|
|
90
|
-
frequency TEXT,
|
|
91
|
-
is_cancel INTEGER DEFAULT 0,
|
|
92
|
-
api_token TEXT,
|
|
93
|
-
project_id TEXT,
|
|
94
|
-
cases TEXT,
|
|
95
|
-
create_dtime INTEGER
|
|
96
|
-
)
|
|
97
|
-
`).run();
|
|
98
|
-
|
|
99
|
-
// 2. 中间结果暂存表 (Master 写入,聚合后删除)
|
|
100
|
-
this.db.prepare(`
|
|
101
|
-
CREATE TABLE IF NOT EXISTS task_results (
|
|
102
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
103
|
-
execution_id TEXT,
|
|
104
|
-
data TEXT,
|
|
105
|
-
create_at INTEGER
|
|
106
|
-
)
|
|
107
|
-
`).run();
|
|
108
|
-
this.db.prepare(`CREATE INDEX IF NOT EXISTS idx_exec_id ON task_results(execution_id)`).run();
|
|
59
|
+
});
|
|
109
60
|
}
|
|
110
61
|
|
|
62
|
+
// ==========================================
|
|
63
|
+
// 核心运行逻辑
|
|
64
|
+
// ==========================================
|
|
111
65
|
run() {
|
|
112
66
|
if (cluster.isPrimary) {
|
|
67
|
+
console.log(`👑 [Master] 主进程运行中 | PID: ${process.pid}`);
|
|
113
68
|
this._startMaster();
|
|
114
69
|
} else {
|
|
70
|
+
console.log(`👷 [Worker] 子进程运行中 | PID: ${process.pid}`);
|
|
115
71
|
this._startWorker();
|
|
116
72
|
}
|
|
117
73
|
}
|
|
118
74
|
|
|
119
|
-
formatTimeToISO(time) {
|
|
120
|
-
if (!time) return null;
|
|
121
|
-
try {
|
|
122
|
-
const dayjs = require('dayjs');
|
|
123
|
-
const utc = require('dayjs/plugin/utc');
|
|
124
|
-
const timezone = require('dayjs/plugin/timezone');
|
|
125
|
-
dayjs.extend(utc);
|
|
126
|
-
dayjs.extend(timezone);
|
|
127
|
-
return dayjs(time).tz('Asia/Shanghai').format();
|
|
128
|
-
} catch (e) {
|
|
129
|
-
return new Date(time).toISOString();
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
// ==========================================
|
|
134
|
-
// Master 进程逻辑:单点读写
|
|
135
|
-
// ==========================================
|
|
136
|
-
|
|
137
75
|
_startMaster() {
|
|
138
|
-
console.log(
|
|
139
|
-
|
|
140
|
-
for (let i = 0; i < this.workerNum; i++) {
|
|
141
|
-
this._bindWorker(cluster.fork());
|
|
142
|
-
}
|
|
76
|
+
console.log(`🎋 [Master] 正在分配 ${this.workerNum} 个工作进程...`);
|
|
77
|
+
for (let i = 0; i < this.workerNum; i++) this._bindWorker(cluster.fork());
|
|
143
78
|
|
|
144
|
-
cluster.on('exit', (worker
|
|
145
|
-
console.
|
|
79
|
+
cluster.on('exit', (worker) => {
|
|
80
|
+
console.error(`🚨 [Master] Worker ${worker.process.pid} 离线,正在重启...`);
|
|
146
81
|
this._bindWorker(cluster.fork());
|
|
147
82
|
});
|
|
148
83
|
|
|
@@ -152,313 +87,357 @@ class CronScheduler {
|
|
|
152
87
|
_bindWorker(worker) {
|
|
153
88
|
worker.on('message', async (msg) => {
|
|
154
89
|
if (msg.action === 'UNIT_COMPLETED') {
|
|
155
|
-
const { executionId, data } = msg.payload;
|
|
156
|
-
|
|
90
|
+
const { executionId, data, unitIndex } = msg.payload;
|
|
91
|
+
console.log(`📩 [Master] 收到执行反馈: ${executionId} [#${unitIndex}]`);
|
|
92
|
+
|
|
93
|
+
const resPath = path.join(this.paths.results, `${executionId}_${unitIndex}.res`);
|
|
94
|
+
fs.writeFileSync(resPath, JSON.stringify(data));
|
|
157
95
|
|
|
96
|
+
const record = this.resultCollector.get(executionId);
|
|
158
97
|
if (record) {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
// 计数检查
|
|
165
|
-
if (typeof record.received_count !== 'number') record.received_count = 0;
|
|
166
|
-
record.received_count++;
|
|
167
|
-
|
|
168
|
-
console.log(`[Progress] ${executionId}: ${record.received_count}/${record.total}`);
|
|
169
|
-
|
|
170
|
-
if (record.received_count >= record.total) {
|
|
171
|
-
await this._reportFromDB(executionId, record);
|
|
172
|
-
}
|
|
173
|
-
} catch (e) {
|
|
174
|
-
console.error(`[Master DB Write Error]`, e.message);
|
|
98
|
+
record.received_count++;
|
|
99
|
+
if (record.received_count >= record.total) {
|
|
100
|
+
console.log(`📦 [Master] 实例 ${executionId} 单元全部集齐,准备上报...`);
|
|
101
|
+
await this._aggregateAndReport(executionId, record);
|
|
175
102
|
}
|
|
176
103
|
}
|
|
177
104
|
}
|
|
178
105
|
});
|
|
179
106
|
}
|
|
180
107
|
|
|
181
|
-
|
|
108
|
+
// ==========================================
|
|
109
|
+
// 分布式锁与分发控制
|
|
110
|
+
// ==========================================
|
|
111
|
+
_tryAcquireLock(jobId) {
|
|
112
|
+
const lockFile = path.join(this.paths.locks, `${jobId}.lock`);
|
|
182
113
|
try {
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
stats.http_success += (item.http?.success || 0);
|
|
196
|
-
stats.assert_total += (item.assert?.total || 0);
|
|
197
|
-
stats.assert_success += (item.assert?.success || 0);
|
|
198
|
-
stats.total_resp_time += (item.total_response_time || 0);
|
|
199
|
-
|
|
200
|
-
return _.assign(item, {
|
|
201
|
-
source: 'scheduled',
|
|
202
|
-
start_at: this.formatTimeToISO(item?.start_at),
|
|
203
|
-
end_at: this.formatTimeToISO(item?.end_at)
|
|
204
|
-
});
|
|
205
|
-
});
|
|
206
|
-
|
|
207
|
-
const httpRate = stats.http_total > 0 ? ((stats.http_success / stats.http_total) * 100).toFixed(2) : "0.00";
|
|
208
|
-
const assertRate = stats.assert_total > 0 ? ((stats.assert_success / stats.assert_total) * 100).toFixed(2) : "0.00";
|
|
209
|
-
const avgResp = results.length > 0 ? (stats.total_resp_time / results.length).toFixed(0) : 0;
|
|
210
|
-
|
|
211
|
-
const finalPayload = {
|
|
212
|
-
info: {
|
|
213
|
-
job_id: record.job_info.job_id,
|
|
214
|
-
report_name: record.job_info.name,
|
|
215
|
-
project_id: record.job_info.project_id,
|
|
216
|
-
execution_id: executionId,
|
|
217
|
-
start_at: this.formatTimeToISO(record.job_info.start_time),
|
|
218
|
-
end_at: this.formatTimeToISO(Date.now()),
|
|
219
|
-
http_pass_rate: `${httpRate}%`,
|
|
220
|
-
assert_pass_rate: `${assertRate}%`,
|
|
221
|
-
avg_response_time: `${avgResp}ms`,
|
|
222
|
-
total_http: stats.http_total,
|
|
223
|
-
success_http: stats.http_success,
|
|
224
|
-
total_assert: stats.assert_total,
|
|
225
|
-
success_assert: stats.assert_success,
|
|
226
|
-
total_units: record.total,
|
|
227
|
-
actual_units: results.length
|
|
228
|
-
},
|
|
229
|
-
results: results
|
|
230
|
-
};
|
|
231
|
-
|
|
232
|
-
const response = await axios.post(`${this.apiUrl}/open/hnzycfc/scheduled_task/report/add`, finalPayload, {
|
|
233
|
-
headers: { 'Content-Type': 'application/json', 'api-token': record.api_token },
|
|
234
|
-
maxContentLength: Infinity,
|
|
235
|
-
maxBodyLength: Infinity,
|
|
236
|
-
timeout: 60000 // 聚合报告可能很大,增加超时时间
|
|
237
|
-
});
|
|
238
|
-
|
|
239
|
-
if (response?.data?.code == 0) {
|
|
240
|
-
console.log(`[CronScheduler] Execution ${executionId} Success.`);
|
|
241
|
-
// 上报成功后清理中间数据
|
|
242
|
-
this.db.prepare(`DELETE FROM task_results WHERE execution_id = ?`).run(executionId);
|
|
243
|
-
} else {
|
|
244
|
-
console.error(`[Report Error] ${executionId}: ${response?.data?.msg}`);
|
|
245
|
-
}
|
|
246
|
-
} catch (err) {
|
|
247
|
-
console.error(`[Aggregation Error]`, err.stack);
|
|
248
|
-
} finally {
|
|
249
|
-
this.resultCollector.delete(executionId);
|
|
114
|
+
fs.writeFileSync(lockFile, JSON.stringify({ node: os.hostname(), time: Date.now() }), { flag: 'wx' });
|
|
115
|
+
console.log(`🔐 [Lock] 抢锁成功: ${jobId}`);
|
|
116
|
+
return true;
|
|
117
|
+
} catch (e) {
|
|
118
|
+
try {
|
|
119
|
+
const stats = fs.statSync(lockFile);
|
|
120
|
+
if (Date.now() - stats.mtimeMs > 1800000) { // 30分钟死锁保护
|
|
121
|
+
fs.unlinkSync(lockFile);
|
|
122
|
+
console.warn(`🔓 [Lock] 强制清理僵尸锁: ${jobId}`);
|
|
123
|
+
}
|
|
124
|
+
} catch (err) { }
|
|
125
|
+
return false;
|
|
250
126
|
}
|
|
251
127
|
}
|
|
252
128
|
|
|
129
|
+
_releaseLock(jobId) {
|
|
130
|
+
const lockFile = path.join(this.paths.locks, `${jobId}.lock`);
|
|
131
|
+
try { if (fs.existsSync(lockFile)) fs.unlinkSync(lockFile); } catch (e) { }
|
|
132
|
+
}
|
|
133
|
+
|
|
253
134
|
_dispatch(job) {
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
135
|
+
console.log(`🎯 [Master] 准备分发: ${job.job_id} (${job.name || 'Task'})`);
|
|
136
|
+
if (!this._tryAcquireLock(job.job_id)) {
|
|
137
|
+
console.warn(`🔒 [Lock] 抢锁失败,跳过执行: ${job.job_id}`);
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
257
140
|
|
|
141
|
+
try {
|
|
142
|
+
const runners = (typeof job.runners === 'string') ? JSON5.parse(job.runners || '[]') : (job.runners || []);
|
|
258
143
|
const executionId = `${job.job_id}_${Date.now()}`;
|
|
259
|
-
this.resultCollector.set(executionId, {
|
|
260
|
-
total: runners.length,
|
|
261
|
-
received_count: 0,
|
|
262
|
-
api_token: job.api_token,
|
|
263
|
-
job_info: {
|
|
264
|
-
job_id: job.job_id,
|
|
265
|
-
name: job.name,
|
|
266
|
-
project_id: job.project_id,
|
|
267
|
-
start_time: Date.now()
|
|
268
|
-
}
|
|
269
|
-
});
|
|
144
|
+
this.resultCollector.set(executionId, { total: runners.length, received_count: 0, job_info: job });
|
|
270
145
|
|
|
271
|
-
const
|
|
146
|
+
const workers = Object.values(cluster.workers).filter(w => w.isConnected());
|
|
272
147
|
runners.forEach((runner, index) => {
|
|
273
|
-
const worker = _.sample(
|
|
148
|
+
const worker = _.sample(workers);
|
|
274
149
|
if (worker) {
|
|
275
150
|
worker.send({
|
|
276
151
|
action: 'EXECUTE_UNIT',
|
|
277
|
-
payload: { executionId,
|
|
152
|
+
payload: { executionId, test_events: runner.test_events, option: runner.option, unitIndex: index, api_token: job.api_token }
|
|
278
153
|
});
|
|
279
154
|
}
|
|
280
155
|
});
|
|
281
|
-
|
|
156
|
+
console.log(`📤 [Master] 已向 Worker 发送 ${runners.length} 个任务单元`);
|
|
157
|
+
} catch (e) {
|
|
158
|
+
console.error(`💥 [Master] 分发崩溃:`, e.message);
|
|
159
|
+
this._releaseLock(job.job_id);
|
|
160
|
+
}
|
|
282
161
|
}
|
|
283
162
|
|
|
284
163
|
// ==========================================
|
|
285
|
-
//
|
|
164
|
+
// 业务接口 (CRUD)
|
|
286
165
|
// ==========================================
|
|
166
|
+
createJob(payload) {
|
|
167
|
+
console.log(`📝 [API] 创建/更新任务: ${payload.job_id}`);
|
|
168
|
+
const jobPath = path.join(this.paths.jobs, `${payload.job_id}.json`);
|
|
169
|
+
const cleanPayload = _.cloneDeep(payload);
|
|
287
170
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
if (msg.action === 'EXECUTE_UNIT') {
|
|
295
|
-
const { executionId, api_token, test_events, option } = msg.payload;
|
|
296
|
-
const socketPath = path.join(os.tmpdir(), `ea_${uuidv4().slice(0, 8)}.sock`);
|
|
297
|
-
|
|
298
|
-
const server = net.createServer((socket) => {
|
|
299
|
-
socket.on('data', async (stream) => {
|
|
300
|
-
try {
|
|
301
|
-
const info = JSON.parse(stream.toString());
|
|
302
|
-
await this._handleUnitScript(socket, info.action, info.data);
|
|
303
|
-
} catch (e) {
|
|
304
|
-
if (!socket.destroyed) socket.write(JSON.stringify({ status: 'error', message: e.message }) + "\n\n");
|
|
305
|
-
}
|
|
306
|
-
});
|
|
307
|
-
});
|
|
308
|
-
|
|
309
|
-
server.on('error', (err) => console.error(`[Socket Server Error] PID ${process.pid}: ${err.message}`));
|
|
310
|
-
|
|
311
|
-
server.listen(socketPath, () => {
|
|
312
|
-
const finalOptions = _.cloneDeep(option || {});
|
|
313
|
-
const base64Pipe = Buffer.from(socketPath).toString('base64');
|
|
314
|
-
_.set(finalOptions, 'env.ELECTRON_PIPE', base64Pipe);
|
|
315
|
-
|
|
316
|
-
runner(test_events, finalOptions, (res) => {
|
|
317
|
-
if (res?.action === 'complete') {
|
|
318
|
-
if (process.connected) {
|
|
319
|
-
process.send({ action: 'UNIT_COMPLETED', payload: { executionId, data: res.data, api_token } });
|
|
320
|
-
}
|
|
321
|
-
server.close();
|
|
322
|
-
try { if (fs.existsSync(socketPath)) fs.unlinkSync(socketPath); } catch (e) { }
|
|
323
|
-
}
|
|
324
|
-
});
|
|
325
|
-
});
|
|
326
|
-
}
|
|
327
|
-
});
|
|
171
|
+
if (typeof cleanPayload.frequency === 'string') {
|
|
172
|
+
try { cleanPayload.frequency = JSON5.parse(cleanPayload.frequency); } catch (e) { }
|
|
173
|
+
}
|
|
174
|
+
if (typeof cleanPayload.runners === 'string') {
|
|
175
|
+
try { cleanPayload.runners = JSON5.parse(cleanPayload.runners); } catch (e) { }
|
|
176
|
+
}
|
|
328
177
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
178
|
+
const jobData = { ...cleanPayload, is_cancel: 0, create_dtime: Date.now() };
|
|
179
|
+
fs.writeFileSync(jobPath, JSON.stringify(jobData, null, 2), 'utf8');
|
|
180
|
+
this.upsertTimer(jobData.job_id, jobData.frequency, jobData);
|
|
181
|
+
return true;
|
|
332
182
|
}
|
|
333
183
|
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
184
|
+
getAllJobs(project_id) {
|
|
185
|
+
console.log(`🔍 [API] 查询项目任务列表: ${project_id}`);
|
|
186
|
+
if (!fs.existsSync(this.paths.jobs)) return [];
|
|
187
|
+
return fs.readdirSync(this.paths.jobs).map(f => {
|
|
188
|
+
try {
|
|
189
|
+
const raw = fs.readFileSync(path.join(this.paths.jobs, f), 'utf8');
|
|
190
|
+
if (!raw) return null;
|
|
191
|
+
const job = JSON.parse(raw);
|
|
192
|
+
if (job.project_id !== project_id) return null;
|
|
193
|
+
|
|
194
|
+
const cron = this.convertToCron(job.frequency);
|
|
195
|
+
const runners = (typeof job.runners === 'string') ? JSON5.parse(job.runners || '[]') : (job.runners || []);
|
|
196
|
+
|
|
197
|
+
return {
|
|
198
|
+
..._.omit(job, ['runners']),
|
|
199
|
+
status: job.is_cancel === 0 ? 0 : 1,
|
|
200
|
+
next_run_time: this.getNextExecutionTime(cron),
|
|
201
|
+
runners: _.map(runners, v => v?.option?.testing_id).filter(Boolean)
|
|
202
|
+
};
|
|
203
|
+
} catch (e) { return null; }
|
|
204
|
+
}).filter(Boolean);
|
|
205
|
+
}
|
|
337
206
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
break;
|
|
346
|
-
}
|
|
347
|
-
case 'execute': {
|
|
348
|
-
const commandMap = { 'jar': 'java -jar', 'php': 'php -f', 'js': 'node', 'py': 'python3', 'py3': 'python3', 'go': 'go run', 'sh': 'sh' };
|
|
349
|
-
const ext = path.extname(data.file).slice(1).toLowerCase();
|
|
350
|
-
const command = `${commandMap[ext] || ''} ${data.file} ${(data.args || []).join(' ')}`;
|
|
351
|
-
const output = execSync(command, _.assign({ encoding: process.platform === 'win32' ? 'cp936' : 'utf8' }, data.option || {}));
|
|
352
|
-
socket.write(JSON.stringify({ status: 'success', result: String(output) }) + "\n\n");
|
|
353
|
-
break;
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
} catch (err) {
|
|
357
|
-
socket.write(JSON.stringify({ status: 'error', message: err.message }) + "\n\n");
|
|
358
|
-
}
|
|
207
|
+
deleteJob(job_id) {
|
|
208
|
+
console.log(`🗑️ [API] 删除任务: ${job_id}`);
|
|
209
|
+
const filePath = path.join(this.paths.jobs, `${job_id}.json`);
|
|
210
|
+
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
|
|
211
|
+
if (schedule.scheduledJobs[job_id]) schedule.scheduledJobs[job_id].cancel();
|
|
212
|
+
this._releaseLock(job_id);
|
|
213
|
+
return true;
|
|
359
214
|
}
|
|
360
215
|
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
} catch (e) {
|
|
368
|
-
console.error('[Load Jobs Error]', e.message);
|
|
216
|
+
cancelJob(job_id) {
|
|
217
|
+
const filePath = path.join(this.paths.jobs, `${job_id}.json`);
|
|
218
|
+
if (fs.existsSync(filePath)) {
|
|
219
|
+
const job = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
220
|
+
job.is_cancel = 1;
|
|
221
|
+
fs.writeFileSync(filePath, JSON.stringify(job, null, 2), 'utf8');
|
|
369
222
|
}
|
|
223
|
+
if (schedule.scheduledJobs[job_id]) schedule.scheduledJobs[job_id].cancel();
|
|
224
|
+
console.log(`⏸️ [API] 任务已暂停: ${job_id}`);
|
|
225
|
+
return true;
|
|
370
226
|
}
|
|
371
227
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
if (
|
|
375
|
-
const
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
});
|
|
228
|
+
restartJob(job_id) {
|
|
229
|
+
const filePath = path.join(this.paths.jobs, `${job_id}.json`);
|
|
230
|
+
if (fs.existsSync(filePath)) {
|
|
231
|
+
const job = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
232
|
+
job.is_cancel = 0;
|
|
233
|
+
fs.writeFileSync(filePath, JSON.stringify(job, null, 2), 'utf8');
|
|
234
|
+
this.upsertTimer(job_id, job.frequency, job);
|
|
235
|
+
console.log(`▶️ [API] 任务已重启: ${job_id}`);
|
|
236
|
+
return true;
|
|
380
237
|
}
|
|
238
|
+
return false;
|
|
381
239
|
}
|
|
382
240
|
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
const
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
241
|
+
loadJobs() {
|
|
242
|
+
if (!fs.existsSync(this.paths.jobs)) return;
|
|
243
|
+
const files = fs.readdirSync(this.paths.jobs);
|
|
244
|
+
console.log(`🔍 [Load] 正在加载本地任务,共 ${files.length} 个`);
|
|
245
|
+
files.forEach(f => {
|
|
246
|
+
try {
|
|
247
|
+
const job = JSON.parse(fs.readFileSync(path.join(this.paths.jobs, f), 'utf8'));
|
|
248
|
+
if (job.is_cancel < 1) this.upsertTimer(job.job_id, job.frequency, job);
|
|
249
|
+
} catch (e) { }
|
|
250
|
+
});
|
|
393
251
|
}
|
|
394
252
|
|
|
395
|
-
|
|
396
|
-
this.db.prepare(`UPDATE jobs SET is_cancel = 1 WHERE job_id = ?`).run(job_id);
|
|
253
|
+
upsertTimer(job_id, frequency, jobData) {
|
|
397
254
|
if (schedule.scheduledJobs[job_id]) schedule.scheduledJobs[job_id].cancel();
|
|
398
|
-
|
|
255
|
+
const cron = this.convertToCron(frequency);
|
|
256
|
+
|
|
257
|
+
const nodeJob = schedule.scheduleJob(job_id, cron, () => {
|
|
258
|
+
console.log(`🔔 [Timer] >>> 定时点激活: ${job_id}`);
|
|
259
|
+
const jobPath = path.join(this.paths.jobs, `${job_id}.json`);
|
|
260
|
+
if (!fs.existsSync(jobPath)) return;
|
|
261
|
+
const job = JSON.parse(fs.readFileSync(jobPath, 'utf8'));
|
|
262
|
+
if (job.is_cancel < 1) this._dispatch(job);
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
if (nodeJob) {
|
|
266
|
+
console.log(`✅ [Timer] 任务 ${job_id} 挂载成功,下次执行: ${nodeJob.nextInvocation()?.toLocaleString()}`);
|
|
267
|
+
}
|
|
399
268
|
}
|
|
400
269
|
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
270
|
+
// ==========================================
|
|
271
|
+
// 上报与 Worker 执行
|
|
272
|
+
// ==========================================
|
|
273
|
+
async _aggregateAndReport(executionId, record) {
|
|
274
|
+
console.log(`📊 [Report] 任务 ${executionId} 集齐,准备上报...`);
|
|
405
275
|
try {
|
|
406
|
-
//
|
|
407
|
-
this.
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
276
|
+
// 读取文件系统中的分片结果
|
|
277
|
+
const files = fs.readdirSync(this.paths.results).filter(f => f.startsWith(executionId));
|
|
278
|
+
const rawResults = files.map(f => {
|
|
279
|
+
const p = path.join(this.paths.results, f);
|
|
280
|
+
const data = JSON.parse(fs.readFileSync(p, 'utf8'));
|
|
281
|
+
try { fs.unlinkSync(p); } catch (e) { } // 读完立即删除
|
|
282
|
+
return data;
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
// 调用上面的增强统计方法获取 Payload
|
|
286
|
+
const finalPayload = this._calculateStats(rawResults, record, executionId);
|
|
287
|
+
// fs.writeFileSync('finalPayload.json', JSON.stringify(finalPayload))
|
|
288
|
+
const response = await axios.post(`${this.apiUrl}/open/hnzycfc/scheduled_task/report/add`, finalPayload, {
|
|
289
|
+
headers: { 'api-token': record.job_info.api_token },
|
|
290
|
+
maxContentLength: Infinity,
|
|
291
|
+
maxBodyLength: Infinity
|
|
292
|
+
});
|
|
293
|
+
if (response?.data?.code == 0) {
|
|
294
|
+
console.log(`✨ [Report] 上报成功! 任务ID: ${record.job_info.job_id} | 状态: ${response.status}`);
|
|
295
|
+
} else {
|
|
296
|
+
console.error(`❌ [Report] code 上报失败 ${executionId}: ${response?.data?.msg}`);
|
|
415
297
|
}
|
|
416
|
-
|
|
298
|
+
|
|
417
299
|
} catch (e) {
|
|
418
|
-
console.error(
|
|
419
|
-
|
|
300
|
+
console.error(`❌ [Report] 上报失败: ${executionId}`, e.message);
|
|
301
|
+
} finally {
|
|
302
|
+
this._releaseLock(record.job_info.job_id);
|
|
303
|
+
this.resultCollector.delete(executionId);
|
|
420
304
|
}
|
|
421
305
|
}
|
|
422
306
|
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
307
|
+
_startWorker() {
|
|
308
|
+
const { run: runner } = require('runner-runtime');
|
|
309
|
+
process.on('message', async (msg) => {
|
|
310
|
+
if (msg.action === 'EXECUTE_UNIT') {
|
|
311
|
+
console.log(`⚙️ [Worker ${process.pid}] 执行单元: ${msg.payload.executionId}`);
|
|
312
|
+
runner(msg.payload.test_events, msg.payload.option || {}, (res) => {
|
|
313
|
+
if (res?.action === 'complete') {
|
|
314
|
+
process.send({ action: 'UNIT_COMPLETED', payload: { ...msg.payload, data: res.data } });
|
|
315
|
+
}
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
});
|
|
427
319
|
}
|
|
428
320
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
321
|
+
// ==========================================
|
|
322
|
+
// 工具类
|
|
323
|
+
// ==========================================
|
|
324
|
+
_calculateStats(results, record, executionId) {
|
|
325
|
+
let s = {
|
|
326
|
+
http_total: 0, http_success: 0,
|
|
327
|
+
assert_total: 0, assert_success: 0,
|
|
328
|
+
total_resp_time: 0
|
|
329
|
+
};
|
|
330
|
+
|
|
331
|
+
// 1. 遍历并处理每个结果单元
|
|
332
|
+
const processedResults = results.map(item => {
|
|
333
|
+
// 累加统计
|
|
334
|
+
s.http_total += (item.http?.total || 0);
|
|
335
|
+
s.http_success += (item.http?.success || 0);
|
|
336
|
+
s.assert_total += (item.assert?.total || 0);
|
|
337
|
+
s.assert_success += (item.assert?.success || 0);
|
|
338
|
+
s.total_resp_time += (item.total_response_time || 0);
|
|
339
|
+
|
|
340
|
+
// 还原 item 内部字段
|
|
341
|
+
return _.assign(item, {
|
|
342
|
+
source: 'scheduled',
|
|
343
|
+
start_at: this.formatTimeToISO(item?.start_at || Date.now()),
|
|
344
|
+
end_at: this.formatTimeToISO(item?.end_at || Date.now())
|
|
345
|
+
});
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
// 2. 计算通过率等指标
|
|
349
|
+
const httpRate = s.http_total > 0 ? ((s.http_success / s.http_total) * 100).toFixed(2) : "0.00";
|
|
350
|
+
const assertRate = s.assert_total > 0 ? ((s.assert_success / s.assert_total) * 100).toFixed(2) : "0.00";
|
|
351
|
+
const avgResp = processedResults.length > 0 ? (s.total_resp_time / processedResults.length).toFixed(0) : 0;
|
|
352
|
+
|
|
353
|
+
// 3. 封装与之前 DB 版本完全一致的 Payload
|
|
354
|
+
return {
|
|
355
|
+
info: {
|
|
356
|
+
job_id: record.job_info.job_id,
|
|
357
|
+
report_name: record.job_info.name,
|
|
358
|
+
project_id: record.job_info.project_id,
|
|
359
|
+
execution_id: executionId,
|
|
360
|
+
// 使用任务开始时间和当前结束时间
|
|
361
|
+
|
|
362
|
+
start_at: this.formatTimeToISO(record.job_info.start_time || Date.now()),
|
|
363
|
+
end_at: this.formatTimeToISO(Date.now()),
|
|
364
|
+
http_pass_rate: `${httpRate}%`,
|
|
365
|
+
assert_pass_rate: `${assertRate}%`,
|
|
366
|
+
avg_response_time: `${avgResp}ms`,
|
|
367
|
+
total_http: s.http_total,
|
|
368
|
+
success_http: s.http_success,
|
|
369
|
+
total_assert: s.assert_total,
|
|
370
|
+
success_assert: s.assert_success,
|
|
371
|
+
total_units: record.total,
|
|
372
|
+
actual_units: processedResults.length
|
|
373
|
+
},
|
|
374
|
+
results: processedResults
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
getNextExecutionTime(cron) {
|
|
379
|
+
const { CronExpressionParser } = require('cron-parser');
|
|
380
|
+
const interval = CronExpressionParser.parse(cron, { tz: 'Asia/Shanghai', utc: false });
|
|
381
|
+
return new Date(interval.next().toISOString()).getTime()
|
|
437
382
|
}
|
|
438
383
|
|
|
439
384
|
convertToCron(freq) {
|
|
385
|
+
if (!freq) return '0 0 0 * * *';
|
|
440
386
|
const data = typeof freq === 'string' ? JSON5.parse(freq) : freq;
|
|
387
|
+
|
|
388
|
+
// 情况 A:用户直接传 Cron 表达式
|
|
441
389
|
if (data.type === 'cron') {
|
|
442
390
|
const exp = data.cron?.expression || '0 0 * * *';
|
|
443
391
|
const parts = exp.trim().split(/\s+/);
|
|
392
|
+
// 如果是 5 位,补齐第 0 位为 0 秒,确保不重复触发
|
|
444
393
|
return parts.length === 5 ? `0 ${exp}` : exp;
|
|
445
394
|
}
|
|
395
|
+
|
|
396
|
+
// 情况 B:预设周期配置
|
|
446
397
|
if (data.type === 'preset') {
|
|
447
398
|
const { cycle, config } = data.preset;
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
399
|
+
|
|
400
|
+
switch (cycle) {
|
|
401
|
+
case 'minute':
|
|
402
|
+
// 修正:强制 0 秒开始,每 interval 分钟执行
|
|
403
|
+
return `0 */${config.interval} * * * *`;
|
|
404
|
+
|
|
405
|
+
case 'hour':
|
|
406
|
+
// 修正:强制 0 秒 0 分开始,每 interval 小时执行
|
|
407
|
+
return `0 0 */${config.interval} * * *`;
|
|
408
|
+
|
|
409
|
+
case 'day': {
|
|
410
|
+
// 兼容 '10:30' 这种格式
|
|
411
|
+
const [hour, minute] = (config.time || "0:0").split(':');
|
|
412
|
+
return `0 ${parseInt(minute)} ${parseInt(hour)} * * *`;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
case 'week': {
|
|
416
|
+
const [hour, minute] = (config.time || "0:0").split(':');
|
|
417
|
+
// config.weekdays 应该是 [1, 3, 5] 这种数组
|
|
418
|
+
const days = Array.isArray(config.weekdays) ? config.weekdays.join(',') : '*';
|
|
419
|
+
return `0 ${parseInt(minute)} ${parseInt(hour)} * * ${days}`;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
default:
|
|
423
|
+
return '0 0 0 * * *';
|
|
424
|
+
}
|
|
453
425
|
}
|
|
426
|
+
|
|
454
427
|
return '0 0 0 * * *';
|
|
455
428
|
}
|
|
456
429
|
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
430
|
+
|
|
431
|
+
/**
|
|
432
|
+
* Converts timestamp to ISO format
|
|
433
|
+
*/
|
|
434
|
+
formatTimeToISO(time) {
|
|
435
|
+
const dayjs = require('dayjs');
|
|
436
|
+
const utc = require('dayjs/plugin/utc');
|
|
437
|
+
const timezone = require('dayjs/plugin/timezone');
|
|
438
|
+
dayjs.extend(utc);
|
|
439
|
+
dayjs.extend(timezone);
|
|
440
|
+
return dayjs(time).tz('Asia/Shanghai').format();
|
|
462
441
|
}
|
|
463
442
|
}
|
|
464
443
|
|
package/index_bak1.js
ADDED
|
@@ -0,0 +1,465 @@
|
|
|
1
|
+
const cluster = require('cluster');
|
|
2
|
+
const schedule = require('node-schedule');
|
|
3
|
+
const _ = require('lodash');
|
|
4
|
+
const axios = require('axios');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const os = require('os');
|
|
7
|
+
const JSON5 = require('json5');
|
|
8
|
+
const Database = require('better-sqlite3');
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const { v4: uuidv4 } = require('uuid');
|
|
11
|
+
const { execSync } = require('child_process');
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* CronScheduler - 企业级多进程自动化测试调度器 (生产增强版)
|
|
15
|
+
*/
|
|
16
|
+
class CronScheduler {
|
|
17
|
+
constructor(config = {}) {
|
|
18
|
+
// 1. 配置初始化
|
|
19
|
+
this.apiUrl = config.apiUrl || process?.env["OPENAPI_DOMAIN"] || "https://ee.apipost.cc";
|
|
20
|
+
|
|
21
|
+
// 【优化】不要直接放 /tmp,在项目目录下创建持久化 data 目录
|
|
22
|
+
const dataDir = path.resolve(process?.env?.TEMP_DIR || os.tmpdir(), 'scheduler_data');
|
|
23
|
+
if (cluster.isPrimary && !fs.existsSync(dataDir)) {
|
|
24
|
+
fs.mkdirSync(dataDir, { recursive: true });
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
this.dbFile = config.dbPath || path.join(dataDir, 'echoapi-batch-tasks.sqlite');
|
|
28
|
+
this.workerNum = config.workerNum || os.cpus().length;
|
|
29
|
+
|
|
30
|
+
// 结果聚合收集器 (仅在 Master 内存中存在)
|
|
31
|
+
this.resultCollector = new Map();
|
|
32
|
+
|
|
33
|
+
// 2. 数据库连接与安全自检 (仅 Master 进程持有写连接)
|
|
34
|
+
if (cluster.isPrimary) {
|
|
35
|
+
this._connectDB();
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* 内部方法:建立数据库连接并增加损坏自动修复逻辑
|
|
41
|
+
*/
|
|
42
|
+
_connectDB() {
|
|
43
|
+
try {
|
|
44
|
+
// 预检:如果是 0 字节损坏文件,直接删除
|
|
45
|
+
if (fs.existsSync(this.dbFile) && fs.statSync(this.dbFile).size === 0) {
|
|
46
|
+
fs.unlinkSync(this.dbFile);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
this.db = new Database(this.dbFile, { timeout: 10000 });
|
|
50
|
+
|
|
51
|
+
// 【核心】执行完整性自检,解决 malformed 和 not a database 问题
|
|
52
|
+
const check = this.db.pragma('integrity_check');
|
|
53
|
+
if (check[0].integrity_check !== 'ok') {
|
|
54
|
+
throw new Error('SQLite integrity check failed');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
this._initDB();
|
|
58
|
+
console.log(`[Database] Connection verified and initialized.`);
|
|
59
|
+
} catch (e) {
|
|
60
|
+
console.error(`[Database Critical] 数据库损坏(Malformed/NotADb): ${e.message}`);
|
|
61
|
+
if (this.db) this.db.close();
|
|
62
|
+
|
|
63
|
+
// 自动备份损坏文件,腾出位置重建
|
|
64
|
+
const backupPath = `${this.dbFile}.corrupted_${Date.now()}`;
|
|
65
|
+
if (fs.existsSync(this.dbFile)) {
|
|
66
|
+
fs.renameSync(this.dbFile, backupPath);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// 重建数据库
|
|
70
|
+
this.db = new Database(this.dbFile);
|
|
71
|
+
this._initDB();
|
|
72
|
+
console.warn(`[Database] 坏库已隔离至 ${backupPath},新库已就绪。`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* 初始化表结构与性能参数
|
|
78
|
+
*/
|
|
79
|
+
_initDB() {
|
|
80
|
+
// 性能调优:WAL 模式能极大减少多进程读取时的锁竞争
|
|
81
|
+
this.db.pragma('journal_mode = WAL');
|
|
82
|
+
this.db.pragma('synchronous = NORMAL');
|
|
83
|
+
this.db.pragma('busy_timeout = 5000');
|
|
84
|
+
|
|
85
|
+
// 1. 任务配置表
|
|
86
|
+
this.db.prepare(`
|
|
87
|
+
CREATE TABLE IF NOT EXISTS jobs (
|
|
88
|
+
job_id TEXT PRIMARY KEY,
|
|
89
|
+
name TEXT,
|
|
90
|
+
frequency TEXT,
|
|
91
|
+
is_cancel INTEGER DEFAULT 0,
|
|
92
|
+
api_token TEXT,
|
|
93
|
+
project_id TEXT,
|
|
94
|
+
cases TEXT,
|
|
95
|
+
create_dtime INTEGER
|
|
96
|
+
)
|
|
97
|
+
`).run();
|
|
98
|
+
|
|
99
|
+
// 2. 中间结果暂存表 (Master 写入,聚合后删除)
|
|
100
|
+
this.db.prepare(`
|
|
101
|
+
CREATE TABLE IF NOT EXISTS task_results (
|
|
102
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
103
|
+
execution_id TEXT,
|
|
104
|
+
data TEXT,
|
|
105
|
+
create_at INTEGER
|
|
106
|
+
)
|
|
107
|
+
`).run();
|
|
108
|
+
this.db.prepare(`CREATE INDEX IF NOT EXISTS idx_exec_id ON task_results(execution_id)`).run();
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
run() {
|
|
112
|
+
if (cluster.isPrimary) {
|
|
113
|
+
this._startMaster();
|
|
114
|
+
} else {
|
|
115
|
+
this._startWorker();
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
formatTimeToISO(time) {
|
|
120
|
+
if (!time) return null;
|
|
121
|
+
try {
|
|
122
|
+
const dayjs = require('dayjs');
|
|
123
|
+
const utc = require('dayjs/plugin/utc');
|
|
124
|
+
const timezone = require('dayjs/plugin/timezone');
|
|
125
|
+
dayjs.extend(utc);
|
|
126
|
+
dayjs.extend(timezone);
|
|
127
|
+
return dayjs(time).tz('Asia/Shanghai').format();
|
|
128
|
+
} catch (e) {
|
|
129
|
+
return new Date(time).toISOString();
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// ==========================================
|
|
134
|
+
// Master 进程逻辑:单点读写
|
|
135
|
+
// ==========================================
|
|
136
|
+
|
|
137
|
+
_startMaster() {
|
|
138
|
+
console.log(`[CronScheduler] Master ${process.pid} is running.`);
|
|
139
|
+
|
|
140
|
+
for (let i = 0; i < this.workerNum; i++) {
|
|
141
|
+
this._bindWorker(cluster.fork());
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
cluster.on('exit', (worker, code) => {
|
|
145
|
+
console.warn(`[CronScheduler] Worker ${worker.process.pid} died (code:${code}). Forking...`);
|
|
146
|
+
this._bindWorker(cluster.fork());
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
this.loadJobs();
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
_bindWorker(worker) {
|
|
153
|
+
worker.on('message', async (msg) => {
|
|
154
|
+
if (msg.action === 'UNIT_COMPLETED') {
|
|
155
|
+
const { executionId, data } = msg.payload;
|
|
156
|
+
const record = this.resultCollector.get(executionId);
|
|
157
|
+
|
|
158
|
+
if (record) {
|
|
159
|
+
try {
|
|
160
|
+
// 【单进程写入】只有 Master 负责操作数据库,规避并发写导致的损坏
|
|
161
|
+
this.db.prepare(`INSERT INTO task_results (execution_id, data, create_at) VALUES (?, ?, ?)`)
|
|
162
|
+
.run(executionId, JSON.stringify(data), Date.now());
|
|
163
|
+
|
|
164
|
+
// 计数检查
|
|
165
|
+
if (typeof record.received_count !== 'number') record.received_count = 0;
|
|
166
|
+
record.received_count++;
|
|
167
|
+
|
|
168
|
+
console.log(`[Progress] ${executionId}: ${record.received_count}/${record.total}`);
|
|
169
|
+
|
|
170
|
+
if (record.received_count >= record.total) {
|
|
171
|
+
await this._reportFromDB(executionId, record);
|
|
172
|
+
}
|
|
173
|
+
} catch (e) {
|
|
174
|
+
console.error(`[Master DB Write Error]`, e.message);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async _reportFromDB(executionId, record) {
|
|
182
|
+
try {
|
|
183
|
+
console.log(`[CronScheduler] Aggregating results for: ${executionId}`);
|
|
184
|
+
const rows = this.db.prepare(`SELECT data FROM task_results WHERE execution_id = ?`).all(executionId);
|
|
185
|
+
|
|
186
|
+
let stats = {
|
|
187
|
+
http_total: 0, http_success: 0,
|
|
188
|
+
assert_total: 0, assert_success: 0,
|
|
189
|
+
total_resp_time: 0
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
const results = rows.map(row => {
|
|
193
|
+
const item = JSON.parse(row.data);
|
|
194
|
+
stats.http_total += (item.http?.total || 0);
|
|
195
|
+
stats.http_success += (item.http?.success || 0);
|
|
196
|
+
stats.assert_total += (item.assert?.total || 0);
|
|
197
|
+
stats.assert_success += (item.assert?.success || 0);
|
|
198
|
+
stats.total_resp_time += (item.total_response_time || 0);
|
|
199
|
+
|
|
200
|
+
return _.assign(item, {
|
|
201
|
+
source: 'scheduled',
|
|
202
|
+
start_at: this.formatTimeToISO(item?.start_at),
|
|
203
|
+
end_at: this.formatTimeToISO(item?.end_at)
|
|
204
|
+
});
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
const httpRate = stats.http_total > 0 ? ((stats.http_success / stats.http_total) * 100).toFixed(2) : "0.00";
|
|
208
|
+
const assertRate = stats.assert_total > 0 ? ((stats.assert_success / stats.assert_total) * 100).toFixed(2) : "0.00";
|
|
209
|
+
const avgResp = results.length > 0 ? (stats.total_resp_time / results.length).toFixed(0) : 0;
|
|
210
|
+
|
|
211
|
+
const finalPayload = {
|
|
212
|
+
info: {
|
|
213
|
+
job_id: record.job_info.job_id,
|
|
214
|
+
report_name: record.job_info.name,
|
|
215
|
+
project_id: record.job_info.project_id,
|
|
216
|
+
execution_id: executionId,
|
|
217
|
+
start_at: this.formatTimeToISO(record.job_info.start_time),
|
|
218
|
+
end_at: this.formatTimeToISO(Date.now()),
|
|
219
|
+
http_pass_rate: `${httpRate}%`,
|
|
220
|
+
assert_pass_rate: `${assertRate}%`,
|
|
221
|
+
avg_response_time: `${avgResp}ms`,
|
|
222
|
+
total_http: stats.http_total,
|
|
223
|
+
success_http: stats.http_success,
|
|
224
|
+
total_assert: stats.assert_total,
|
|
225
|
+
success_assert: stats.assert_success,
|
|
226
|
+
total_units: record.total,
|
|
227
|
+
actual_units: results.length
|
|
228
|
+
},
|
|
229
|
+
results: results
|
|
230
|
+
};
|
|
231
|
+
|
|
232
|
+
const response = await axios.post(`${this.apiUrl}/open/hnzycfc/scheduled_task/report/add`, finalPayload, {
|
|
233
|
+
headers: { 'Content-Type': 'application/json', 'api-token': record.api_token },
|
|
234
|
+
maxContentLength: Infinity,
|
|
235
|
+
maxBodyLength: Infinity,
|
|
236
|
+
timeout: 60000 // 聚合报告可能很大,增加超时时间
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
if (response?.data?.code == 0) {
|
|
240
|
+
console.log(`[CronScheduler] Execution ${executionId} Success.`);
|
|
241
|
+
// 上报成功后清理中间数据
|
|
242
|
+
this.db.prepare(`DELETE FROM task_results WHERE execution_id = ?`).run(executionId);
|
|
243
|
+
} else {
|
|
244
|
+
console.error(`[Report Error] ${executionId}: ${response?.data?.msg}`);
|
|
245
|
+
}
|
|
246
|
+
} catch (err) {
|
|
247
|
+
console.error(`[Aggregation Error]`, err.stack);
|
|
248
|
+
} finally {
|
|
249
|
+
this.resultCollector.delete(executionId);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
_dispatch(job) {
|
|
254
|
+
try {
|
|
255
|
+
const runners = JSON5.parse(job.cases || '[]');
|
|
256
|
+
if (runners.length === 0) return;
|
|
257
|
+
|
|
258
|
+
const executionId = `${job.job_id}_${Date.now()}`;
|
|
259
|
+
this.resultCollector.set(executionId, {
|
|
260
|
+
total: runners.length,
|
|
261
|
+
received_count: 0,
|
|
262
|
+
api_token: job.api_token,
|
|
263
|
+
job_info: {
|
|
264
|
+
job_id: job.job_id,
|
|
265
|
+
name: job.name,
|
|
266
|
+
project_id: job.project_id,
|
|
267
|
+
start_time: Date.now()
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
const activeWorkers = Object.values(cluster.workers).filter(w => w.isConnected());
|
|
272
|
+
runners.forEach((runner, index) => {
|
|
273
|
+
const worker = _.sample(activeWorkers);
|
|
274
|
+
if (worker) {
|
|
275
|
+
worker.send({
|
|
276
|
+
action: 'EXECUTE_UNIT',
|
|
277
|
+
payload: { executionId, api_token: job.api_token, test_events: runner.test_events, option: runner.option, unit_index: index }
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
});
|
|
281
|
+
} catch (e) { console.error(`[Dispatch Error]:`, e.message); }
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// ==========================================
|
|
285
|
+
// Worker 进程逻辑:仅负责计算,不碰数据库
|
|
286
|
+
// ==========================================
|
|
287
|
+
|
|
288
|
+
_startWorker() {
|
|
289
|
+
console.log(`[Worker ${process.pid}] Ready.`);
|
|
290
|
+
const { run: runner } = require('runner-runtime');
|
|
291
|
+
const net = require('net');
|
|
292
|
+
|
|
293
|
+
process.on('message', async (msg) => {
|
|
294
|
+
if (msg.action === 'EXECUTE_UNIT') {
|
|
295
|
+
const { executionId, api_token, test_events, option } = msg.payload;
|
|
296
|
+
const socketPath = path.join(os.tmpdir(), `ea_${uuidv4().slice(0, 8)}.sock`);
|
|
297
|
+
|
|
298
|
+
const server = net.createServer((socket) => {
|
|
299
|
+
socket.on('data', async (stream) => {
|
|
300
|
+
try {
|
|
301
|
+
const info = JSON.parse(stream.toString());
|
|
302
|
+
await this._handleUnitScript(socket, info.action, info.data);
|
|
303
|
+
} catch (e) {
|
|
304
|
+
if (!socket.destroyed) socket.write(JSON.stringify({ status: 'error', message: e.message }) + "\n\n");
|
|
305
|
+
}
|
|
306
|
+
});
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
server.on('error', (err) => console.error(`[Socket Server Error] PID ${process.pid}: ${err.message}`));
|
|
310
|
+
|
|
311
|
+
server.listen(socketPath, () => {
|
|
312
|
+
const finalOptions = _.cloneDeep(option || {});
|
|
313
|
+
const base64Pipe = Buffer.from(socketPath).toString('base64');
|
|
314
|
+
_.set(finalOptions, 'env.ELECTRON_PIPE', base64Pipe);
|
|
315
|
+
|
|
316
|
+
runner(test_events, finalOptions, (res) => {
|
|
317
|
+
if (res?.action === 'complete') {
|
|
318
|
+
if (process.connected) {
|
|
319
|
+
process.send({ action: 'UNIT_COMPLETED', payload: { executionId, data: res.data, api_token } });
|
|
320
|
+
}
|
|
321
|
+
server.close();
|
|
322
|
+
try { if (fs.existsSync(socketPath)) fs.unlinkSync(socketPath); } catch (e) { }
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
process.on('uncaughtException', (err) => {
|
|
330
|
+
console.error('[Worker Fatal] Exception:', err.message);
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// ==========================================
|
|
335
|
+
// 业务与辅助方法
|
|
336
|
+
// ==========================================
|
|
337
|
+
|
|
338
|
+
async _handleUnitScript(socket, action, data) {
|
|
339
|
+
try {
|
|
340
|
+
switch (action) {
|
|
341
|
+
case 'queryDatabase': {
|
|
342
|
+
const { DatabaseQuery } = require('database-query');
|
|
343
|
+
const result = await DatabaseQuery(data.dbconfig, data.query);
|
|
344
|
+
socket.write(JSON.stringify(result) + "\n\n");
|
|
345
|
+
break;
|
|
346
|
+
}
|
|
347
|
+
case 'execute': {
|
|
348
|
+
const commandMap = { 'jar': 'java -jar', 'php': 'php -f', 'js': 'node', 'py': 'python3', 'py3': 'python3', 'go': 'go run', 'sh': 'sh' };
|
|
349
|
+
const ext = path.extname(data.file).slice(1).toLowerCase();
|
|
350
|
+
const command = `${commandMap[ext] || ''} ${data.file} ${(data.args || []).join(' ')}`;
|
|
351
|
+
const output = execSync(command, _.assign({ encoding: process.platform === 'win32' ? 'cp936' : 'utf8' }, data.option || {}));
|
|
352
|
+
socket.write(JSON.stringify({ status: 'success', result: String(output) }) + "\n\n");
|
|
353
|
+
break;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
} catch (err) {
|
|
357
|
+
socket.write(JSON.stringify({ status: 'error', message: err.message }) + "\n\n");
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
loadJobs() {
|
|
362
|
+
console.log('[CronScheduler] Loading timers from DB...');
|
|
363
|
+
Object.values(schedule.scheduledJobs).forEach(j => j.cancel());
|
|
364
|
+
try {
|
|
365
|
+
const jobs = this.db.prepare(`SELECT * FROM jobs WHERE is_cancel < 1`).all();
|
|
366
|
+
jobs.forEach(job => this.upsertTimer(job.job_id, job.name, job.frequency, job));
|
|
367
|
+
} catch (e) {
|
|
368
|
+
console.error('[Load Jobs Error]', e.message);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
upsertTimer(job_id, name, frequency, jobData) {
|
|
373
|
+
if (schedule.scheduledJobs[job_id]) schedule.scheduledJobs[job_id].cancel();
|
|
374
|
+
if (jobData?.is_cancel < 1) {
|
|
375
|
+
const cronRule = this.convertToCron(frequency);
|
|
376
|
+
schedule.scheduleJob(job_id, cronRule, () => {
|
|
377
|
+
const currentJob = this.db.prepare(`SELECT * FROM jobs WHERE job_id = ?`).get(job_id);
|
|
378
|
+
if (currentJob && currentJob.is_cancel < 1) this._dispatch(currentJob);
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
createJob(payload) {
|
|
384
|
+
const { job_id, name, frequency, api_token, project_id, runners } = payload;
|
|
385
|
+
const stmt = this.db.prepare(`
|
|
386
|
+
INSERT INTO jobs (job_id, name, frequency, api_token, project_id, cases, create_dtime)
|
|
387
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
388
|
+
ON CONFLICT(job_id) DO UPDATE SET name=excluded.name, frequency=excluded.frequency, cases=excluded.cases, api_token=excluded.api_token, is_cancel=0
|
|
389
|
+
`);
|
|
390
|
+
stmt.run(job_id, name, JSON.stringify(frequency), api_token, project_id, JSON.stringify(runners || []), Date.now());
|
|
391
|
+
this.upsertTimer(job_id, name, frequency, { is_cancel: 0 });
|
|
392
|
+
return true;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
cancelJob(job_id) {
|
|
396
|
+
this.db.prepare(`UPDATE jobs SET is_cancel = 1 WHERE job_id = ?`).run(job_id);
|
|
397
|
+
if (schedule.scheduledJobs[job_id]) schedule.scheduledJobs[job_id].cancel();
|
|
398
|
+
return true;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* 重启已暂停的任务
|
|
403
|
+
*/
|
|
404
|
+
restartJob(job_id) {
|
|
405
|
+
try {
|
|
406
|
+
// 1. 更新数据库状态
|
|
407
|
+
this.db.prepare(`UPDATE jobs SET is_cancel = 0 WHERE job_id = ?`).run(job_id);
|
|
408
|
+
|
|
409
|
+
// 2. 获取最新数据并重新注册定时器
|
|
410
|
+
const job = this.db.prepare(`SELECT * FROM jobs WHERE job_id = ?`).get(job_id);
|
|
411
|
+
if (job) {
|
|
412
|
+
this.upsertTimer(job.job_id, job.name, job.frequency, job);
|
|
413
|
+
console.log(`[CronScheduler] Job ${job_id} restarted.`);
|
|
414
|
+
return true;
|
|
415
|
+
}
|
|
416
|
+
return false;
|
|
417
|
+
} catch (e) {
|
|
418
|
+
console.error(`[Restart Job Error] ${job_id}:`, e.message);
|
|
419
|
+
return false;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
deleteJob(job_id) {
|
|
424
|
+
this.db.prepare(`DELETE FROM jobs WHERE job_id = ?`).run(job_id);
|
|
425
|
+
if (schedule.scheduledJobs[job_id]) schedule.scheduledJobs[job_id].cancel();
|
|
426
|
+
return true;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
getAllJobs(project_id) {
|
|
430
|
+
const jobs = this.db.prepare(`SELECT * FROM jobs WHERE project_id = ?`).all(project_id);
|
|
431
|
+
return jobs.map(job => ({
|
|
432
|
+
..._.omit(job, ['cases']),
|
|
433
|
+
status: job.is_cancel === 0 ? 0 : 1,
|
|
434
|
+
next_run_time: this.getNextExecutionTime(this.convertToCron(job.frequency)),
|
|
435
|
+
runners: JSON5.parse(job.cases || '[]').map(v => v?.option?.testing_id)
|
|
436
|
+
}));
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
convertToCron(freq) {
|
|
440
|
+
const data = typeof freq === 'string' ? JSON5.parse(freq) : freq;
|
|
441
|
+
if (data.type === 'cron') {
|
|
442
|
+
const exp = data.cron?.expression || '0 0 * * *';
|
|
443
|
+
const parts = exp.trim().split(/\s+/);
|
|
444
|
+
return parts.length === 5 ? `0 ${exp}` : exp;
|
|
445
|
+
}
|
|
446
|
+
if (data.type === 'preset') {
|
|
447
|
+
const { cycle, config } = data.preset;
|
|
448
|
+
const [h, m] = (config.time || "0:0").split(':');
|
|
449
|
+
if (cycle === 'minute') return `0 */${config.interval} * * * *`;
|
|
450
|
+
if (cycle === 'hour') return `0 0 */${config.interval} * * *`;
|
|
451
|
+
if (cycle === 'day') return `0 ${parseInt(m)} ${parseInt(h)} * * *`;
|
|
452
|
+
if (cycle === 'week') return `0 ${parseInt(m)} ${parseInt(h)} * * ${Array.isArray(config.weekdays) ? config.weekdays.join(',') : '*'}`;
|
|
453
|
+
}
|
|
454
|
+
return '0 0 0 * * *';
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
getNextExecutionTime(cron) {
|
|
458
|
+
try {
|
|
459
|
+
const { CronExpressionParser } = require('cron-parser');
|
|
460
|
+
return CronExpressionParser.parse(cron, { tz: 'Asia/Shanghai' }).next().getTime();
|
|
461
|
+
} catch (e) { return '-'; }
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
module.exports = CronScheduler;
|