echoapi-cron-scheduler-batch 1.0.1

package/index.js

@@ -0,0 +1,581 @@
+const cluster = require('cluster');
+const schedule = require('node-schedule');
+const _ = require('lodash');
+const axios = require('axios');
+const path = require('path');
+const os = require('os');
+const JSON5 = require('json5');
+const Database = require('better-sqlite3');
+const fs = require('fs');
+const { v4: uuidv4 } = require('uuid');
+const { execSync } = require('child_process');
+
+/**
+ * CronScheduler - 企业级多进程自动化测试调度器
+ */
+class CronScheduler {
+    constructor(config = {}) {
+        // 配置初始化
+        this.apiUrl = config.apiUrl || process?.env["OPENAPI_DOMAIN"] || "https://open.apipost.net";
+        this.dbFile = config.dbPath || path.resolve(os.tmpdir(), 'echoapi-batch-tasks.sqlite');
+        this.workerNum = config.workerNum || os.cpus().length;
+
+        // 结果聚合收集器：用于收集同一批次下所有 Runner 的结果
+        this.resultCollector = new Map();
+
+        if (cluster.isPrimary) {
+            this.db = new Database(this.dbFile);
+            this._initDB();
+        }
+    }
+
+    /**
+     * 内部方法：初始化数据库表结构
+     */
+    _initDB() {
+        this.db.pragma('journal_mode = WAL');
+        this.db.prepare(`
+            CREATE TABLE IF NOT EXISTS jobs (
+                job_id TEXT PRIMARY KEY,
+                name TEXT,
+                frequency TEXT,
+                is_cancel INTEGER DEFAULT 0,
+                api_token TEXT,
+                project_id TEXT,
+                user_uid TEXT,        -- 新增：存储用户UID
+                cases TEXT,           -- 存储 runners 数组 JSON
+                create_dtime INTEGER
+            )
+        `).run();
+    }
+
+    /**
+     * 核心启动入口
+     */
+    run() {
+        if (cluster.isPrimary) {
+            this._startMaster();
+        } else {
+            this._startWorker();
+        }
+    }
+    /**
+   * Converts timestamp to ISO format
+   */
+    formatTimeToISO(time) {
+        const dayjs = require('dayjs');
+        const utc = require('dayjs/plugin/utc');
+        const timezone = require('dayjs/plugin/timezone');
+        dayjs.extend(utc);
+        dayjs.extend(timezone);
+        return dayjs(time).tz('Asia/Shanghai').format();
+    }
+
+    // ==========================================
+    // Master 进程逻辑：负责任务分发与结果聚合
+    // ==========================================
+
+    _startMaster() {
+        console.log(`[CronScheduler] Master process ${process.pid} is running.`);
+
+        // 衍生子进程
+        for (let i = 0; i < this.workerNum; i++) {
+            this._bindWorker(cluster.fork());
+        }
+
+        // 进程自动重启逻辑
+        cluster.on('exit', (worker) => {
+            console.warn(`[CronScheduler] Worker ${worker.process.pid} died. Forking a new one...`);
+            this._bindWorker(cluster.fork());
+        });
+
+        // 启动时自动加载数据库中的任务
+        this.loadJobs();
+    }
+
+    _bindWorker(worker) {
+        worker.on('message', async (msg) => {
+            if (msg.action === 'UNIT_COMPLETED') {
+                const { executionId, data } = msg.payload;
+                const record = this.resultCollector.get(executionId);
+
+                if (record) {
+
+                    record.received.push(_.assign(data, {
+                        source: 'scheduled',
+                        start_at: this.formatTimeToISO(data?.start_at),
+                        end_at: this.formatTimeToISO(data?.end_at)
+                    }));
+                    // 检查是否收齐了该 Job 下所有的 runners 结果
+                    if (record.received.length >= record.total) {
+                        await this._reportAggregated(executionId, record);
+                    }
+                }
+            }
+        });
+    }
+
+    /**
+     * 将聚合后的结果一次性推送到 OpenAPI
+     */
+    async _reportAggregated(executionId, record) {
+        try {
+
+            const results = record.received;
+            const endTime = Date.now();
+
+            // --- 核心统计逻辑 ---
+            let stats = {
+                http_total: 0,
+                http_success: 0,
+                assert_total: 0,
+                assert_success: 0,
+                total_response_time: 0,
+                count: 0
+            };
+
+            results.forEach(item => {
+                // 累加 HTTP 统计
+                stats.http_total += (item.http?.total || 0);
+                stats.http_success += (item.http?.success || 0);
+
+                // 累加 断言 统计
+                stats.assert_total += (item.assert?.total || 0);
+                stats.assert_success += (item.assert?.success || 0);
+
+                // 累加 响应时间 用于计算平均值
+                stats.total_response_time += (item.total_response_time || 0);
+                stats.count++;
+            });
+
+            // 计算通过率 (保留两位小数)
+            const httpPassRate = stats.http_total > 0
+                ? ((stats.http_success / stats.http_total) * 100).toFixed(2)
+                : "0.00";
+
+            const assertPassRate = stats.assert_total > 0
+                ? ((stats.assert_success / stats.assert_total) * 100).toFixed(2)
+                : "0.00";
+
+            // 计算平均响应时间
+            const avgResponseTime = stats.count > 0
+                ? (stats.total_response_time / stats.count).toFixed(0)
+                : 0;
+
+            // --- 封装最终 Payload ---
+            const finalPayload = {
+                info: {
+                    job_id: record.job_info.job_id,
+                    name: record.job_info.name,
+                    project_id: record.job_info.project_id,
+                    user_uid: record.job_info.user_uid, // 放入最终报文
+                    execution_id: executionId,
+                    start_at: this.formatTimeToISO(record.job_info.start_time),
+                    end_at: this.formatTimeToISO(endTime),
+
+                    // 业务统计字段
+                    http_pass_rate: `${httpPassRate}%`,
+                    assert_pass_rate: `${assertPassRate}%`,
+                    avg_response_time: `${avgResponseTime}ms`,
+
+                    // 原始计数
+                    total_http: stats.http_total,
+                    success_http: stats.http_success,
+                    total_assert: stats.assert_total,
+                    success_assert: stats.assert_success,
+
+                    total_units: record.total,
+                    actual_units: results.length
+                },
+                results: results // 原始详细列表
+            };
+
+            // 写入本地备份，增加 try-catch 避免权限问题
+            try {
+                // fs.writeFileSync('last_report.json', JSON.stringify(finalPayload));
+            } catch (e) { }
+
+            console.log(`[CronScheduler] Batch job finished. ExecutionID: ${executionId}. Aggregating ${record.total} results...`);
+            // console.log(`[CronScheduler] TOKEN ${record.api_token}`);
+
+            const response = await axios.post(`${this.apiUrl}/open/automated_testing/report/add`, finalPayload, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'api-token': record.api_token
+                }
+            });
+            if (response?.data?.code != 0) {
+                console.log(`Execution Id  ${executionId} reporting error: ${response?.data?.msg}`);
+            } else {
+                console.log(`Execution Id ${executionId} reported success...`);
+            }
+        } catch (err) {
+            console.error(`[CronScheduler] Report aggregation failed:`, err.message);
+        } finally {
+            this.resultCollector.delete(executionId); // 清理内存
+            console.info(`[CronScheduler] Result collector cleared for ${executionId}`);
+        }
+    }
+    /**
+     * 查看当前所有待执行（活跃）的任务及其详细信息
+     */
+    getPendingJobs() {
+        // 1. 从内存中获取 node-schedule 注册的所有 job_id
+        const activeTimerIds = Object.keys(schedule.scheduledJobs);
+
+        if (activeTimerIds.length === 0) {
+            return [];
+        }
+
+        // 2. 从数据库中查询这些活跃任务的详细配置
+        // 使用 IN 语句批量查询，效率更高
+        const placeholders = activeTimerIds.map(() => '?').join(',');
+        const query = `SELECT job_id, name, frequency, project_id, create_dtime FROM jobs WHERE job_id IN (${placeholders}) AND is_cancel < 1`;
+
+        try {
+            const jobs = this.db.prepare(query).all(...activeTimerIds);
+
+            // 3. 结合内存中的下次执行时间进行封装
+            return _.map(jobs, job => {
+                const cronRule = this.convertToCron(JSON5.parse(job.frequency));
+                const nextTime = this.getNextExecutionTime(cronRule);
+
+                return {
+                    job_id: job.job_id,
+                    name: job.name,
+                    project_id: job.project_id,
+                    cron: cronRule,
+                    next_run_at: typeof nextTime === 'number' ? new Date(nextTime).toLocaleString() : nextTime,
+                    timestamp: nextTime // 方便前端排序
+                };
+            }).sort((a, b) => (a.timestamp - b.timestamp)); // 按执行时间先后排序
+        } catch (e) {
+            console.error('[View Jobs Error]', e.message);
+            return [];
+        }
+    }
+
+    /**
+     * 增量更新定时器：解决时间漂移和重复触发的关键
+     */
+    upsertTimer(job_id, name, frequency, jobData) {
+        // 1. 如果已存在，先取消旧的内存定时器
+        if (schedule.scheduledJobs[job_id]) {
+            schedule.scheduledJobs[job_id].cancel();
+        }
+
+        // 2. 如果任务未被取消 (is_cancel < 1)
+        if (jobData && jobData.is_cancel < 1) {
+            try {
+                const freq = typeof frequency === 'string' ? JSON5.parse(frequency) : frequency;
+                const cronRule = this.convertToCron(freq);
+
+                // 重新创建定时器
+                schedule.scheduleJob(job_id, cronRule, () => {
+                    const now = new Date().toLocaleString();
+                    console.log(`[Timer Triggered] ${name} (${job_id}) at ${now}`);
+
+                    // 触发时重新从数据库获取最新数据，确保分发的是最新的 runners
+                    const currentJob = this.db.prepare(`SELECT * FROM jobs WHERE job_id = ?`).get(job_id);
+                    if (currentJob && currentJob.is_cancel < 1) {
+                        this._dispatch(currentJob);
+                    }
+                });
+            } catch (e) {
+                console.error(`[CronScheduler] Upsert timer error (ID: ${job_id}):`, e.message);
+            }
+        }
+    }
+
+    // ==========================================
+    // 公共接口：供 Express 路由层调用
+    // ==========================================
+
+    /**
+     * 创建或更新任务
+     */
+    createJob(payload) {
+        const { job_id, name, frequency, api_token, project_id, user_uid, runners } = payload;
+        const casesJson = JSON.stringify(Array.isArray(runners) ? runners : []);
+        const freqJson = JSON.stringify(frequency);
+
+        const stmt = this.db.prepare(`
+            INSERT INTO jobs (job_id, name, frequency, api_token, project_id, cases, create_dtime)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(job_id) DO UPDATE SET
+                name=excluded.name,
+                frequency=excluded.frequency,
+                cases=excluded.cases,
+                api_token=excluded.api_token,
+                user_uid=excluded.user_uid, -- 更新时也同步 UID
+                is_cancel=0
+        `);
+        stmt.run(job_id, name, freqJson, api_token, project_id, user_uid || '', casesJson, Date.now());
+
+        // 使用增量更新，不再 loadJobs()
+        this.upsertTimer(job_id, name, frequency, { is_cancel: 0 });
+        return true;
+    }
+
+    cancelJob(job_id) {
+        this.db.prepare(`UPDATE jobs SET is_cancel = 1 WHERE job_id = ?`).run(job_id);
+        if (schedule.scheduledJobs[job_id]) {
+            schedule.scheduledJobs[job_id].cancel();
+        }
+        return true;
+    }
+
+    restartJob(job_id) {
+        this.db.prepare(`UPDATE jobs SET is_cancel = 0 WHERE job_id = ?`).run(job_id);
+        const job = this.db.prepare(`SELECT * FROM jobs WHERE job_id = ?`).get(job_id);
+        if (job) {
+            this.upsertTimer(job.job_id, job.name, job.frequency, job);
+        }
+        return true;
+    }
+
+    deleteJob(job_id) {
+        this.db.prepare(`DELETE FROM jobs WHERE job_id = ?`).run(job_id);
+        if (schedule.scheduledJobs[job_id]) {
+            schedule.scheduledJobs[job_id].cancel();
+        }
+        return true;
+    }
+
+    getAllJobs(project_id) {
+        const jobs = this.db.prepare(`SELECT * FROM jobs WHERE project_id = ?`).all(project_id);
+        // 返回列表时剔除巨大的 cases 字段，优化内存
+        return _.map(jobs, job => {
+            const freq = JSON5.parse(job.frequency || '{}');
+            const cases = JSON5.parse(job.cases || '[]');
+            const cronRule = this.convertToCron(freq);
+
+            // 获取下次执行时间
+            // 如果任务已取消，则不显示下次执行时间
+            let nextRunTime = '-';
+            if (job.is_cancel < 1) {
+                nextRunTime = this.getNextExecutionTime(cronRule);
+            }
+
+            return {
+                ..._.omit(job, ['cases']), // 排除大数据字段
+                status: job.is_cancel === 0 ? 0 : 1, // 0:运行中, 1:已暂停
+                next_run_time: nextRunTime,
+                frequency_display: cronRule, // 返回原始 Cron 规则供前端参考
+                runners: _.map(cases, (v) => v?.option?.testing_id)
+            };
+        });
+    }
+
+    loadJobs() {
+        console.log('[CronScheduler] Initializing timers from database...');
+        // 启动时全量清理一次
+        Object.values(schedule.scheduledJobs).forEach(j => j.cancel());
+
+        const jobs = this.db.prepare(`SELECT * FROM jobs WHERE is_cancel < 1`).all();
+        jobs.forEach(job => {
+            this.upsertTimer(job.job_id, job.name, job.frequency, job);
+        });
+    }
+
+    /**
+     * 分发逻辑：将一个批次内的所有 Runner 均衡分配给 Workers
+     */
+    _dispatch(job) {
+        try {
+            const runners = JSON5.parse(job.cases || '[]');
+            if (runners.length === 0) return;
+
+            const executionId = `${job.job_id}_${Date.now()}`;
+            this.resultCollector.set(executionId, {
+                total: runners.length,
+                received: [],
+                api_token: job.api_token,
+                job_info: {
+                    job_id: job.job_id,
+                    name: job.name,
+                    project_id: job.project_id,
+                    user_uid: job.user_uid, // 将 UID 存入内存
+                    start_time: Date.now()
+                    // 如果数据库有创建人字段，也可以存入：creator: job.creator
+                }
+            });
+
+            runners.forEach((runner, index) => {
+                // 负载均衡：选取当前存活的 worker 
+                const workers = Object.values(cluster.workers).filter(w => w.isConnected());
+                const worker = _.sample(workers);
+                if (worker) {
+                    worker.send({
+                        action: 'EXECUTE_UNIT',
+                        payload: {
+                            executionId,
+                            api_token: job.api_token,
+                            test_events: runner.test_events,
+                            option: runner.option,
+                            unit_index: index
+                        }
+                    });
+                }
+            });
+        } catch (e) {
+            console.error(`[CronScheduler] Dispatch error:`, e.message);
+        }
+    }
+
+    // ==========================================
+    // Worker 进程逻辑：负责具体的任务执行
+    // ==========================================
+
+    _startWorker() {
+        const { run: runner } = require('runner-runtime');
+        const net = require('net');
+
+        process.on('message', async (msg) => {
+            if (msg.action === 'EXECUTE_UNIT') {
+                const { executionId, api_token, test_events, option } = msg.payload;
+                const socketPath = path.join(os.tmpdir(), `echoapi_${uuidv4()}.sock`);
+
+                const server = net.createServer((socket) => {
+                    socket.on('data', async (stream) => {
+                        try {
+                            const info = JSON.parse(stream.toString());
+                            const { action, data } = info;
+                            await this._handleUnitScript(socket, action, data);
+                        } catch (e) {
+                            socket.write(JSON.stringify({ status: 'error', message: e.message }) + "\n\n");
+                        }
+                    });
+                });
+
+                server.listen(socketPath, () => {
+                    const finalOptions = _.cloneDeep(option || {});
+                    const base64Pipe = Buffer.from(socketPath).toString('base64');
+                    _.set(finalOptions, 'env.ELECTRON_PIPE', base64Pipe);
+
+                    runner(test_events, finalOptions, (res) => {
+                        if (res?.action === 'complete') {
+                            process.send({
+                                action: 'UNIT_COMPLETED',
+                                payload: { executionId, data: res.data, api_token }
+                            });
+                            server.close();
+                            if (fs.existsSync(socketPath)) {
+                                try { fs.unlinkSync(socketPath); } catch (e) { }
+                            }
+                        }
+                    });
+                });
+            }
+        });
+        process.on('uncaughtException', (err) => {
+            console.error('[Worker Fatal Error] 捕获到沙箱崩溃:', err.message);
+            // 即使崩溃也不要让子进程立即退出，或者让 Master 自动重启它
+        });
+    }
+
+    async _handleUnitScript(socket, action, data) {
+        try {
+            switch (action) {
+                case 'queryDatabase': {
+                    const { dbconfig, query } = data;
+                    const { DatabaseQuery } = require('database-query');
+                    const result = await DatabaseQuery(dbconfig, query);
+                    socket.write(JSON.stringify(result) + "\n\n");
+                    break;
+                }
+                case 'execute': {
+                    const { file, args = [], option: execOption = {} } = data;
+                    const ext = path.extname(file).slice(1).toLowerCase();
+                    let command = "";
+
+                    switch (ext) {
+                        case 'jar': command = `java -jar ${file} ${args.join(' ')}`; break;
+                        case 'php': command = `php -f ${file} ${args.join(' ')}`; break;
+                        case 'js': command = `node ${file} ${args.join(' ')}`; break;
+                        case 'py':
+                        case 'py3': command = `python3 ${file} ${args.join(' ')}`; break;
+                        case 'go': command = `go run ${file} ${args.join(' ')}`; break;
+                        case 'sh': command = `sh ${file} ${args.join(' ')}`; break;
+                        default: command = `${file} ${args.join(' ')}`;
+                    }
+
+                    const isWindows = process.platform === 'win32';
+                    const config = _.assign(isWindows ? { encoding: 'cp936' } : { encoding: 'utf8' }, execOption);
+                    const output = String(execSync(command, config));
+
+                    socket.write(JSON.stringify({ status: 'success', result: output }) + "\n\n");
+                    break;
+                }
+            }
+        } catch (err) {
+            socket.write(JSON.stringify({ status: 'error', message: err.message }) + "\n\n");
+        }
+    }
+
+    // ==========================================
+    // 辅助工具
+    // ==========================================
+
+    /**
+     * 修正后的 Cron 转换逻辑：确保每一项都从第 0 秒开始执行
+     */
+    convertToCron(freq) {
+        if (!freq) return '0 0 0 * * *';
+        const data = typeof freq === 'string' ? JSON5.parse(freq) : freq;
+
+        // 情况 A：用户直接传 Cron 表达式
+        if (data.type === 'cron') {
+            const exp = data.cron?.expression || '0 0 * * *';
+            const parts = exp.trim().split(/\s+/);
+            // 如果是 5 位，补齐第 0 位为 0 秒，确保不重复触发
+            return parts.length === 5 ? `0 ${exp}` : exp;
+        }
+
+        // 情况 B：预设周期配置
+        if (data.type === 'preset') {
+            const { cycle, config } = data.preset;
+
+            switch (cycle) {
+                case 'minute':
+                    // 修正：强制 0 秒开始，每 interval 分钟执行
+                    return `0 */${config.interval} * * * *`;
+
+                case 'hour':
+                    // 修正：强制 0 秒 0 分开始，每 interval 小时执行
+                    return `0 0 */${config.interval} * * *`;
+
+                case 'day': {
+                    // 兼容 '10:30' 这种格式
+                    const [hour, minute] = (config.time || "0:0").split(':');
+                    return `0 ${parseInt(minute)} ${parseInt(hour)} * * *`;
+                }
+
+                case 'week': {
+                    const [hour, minute] = (config.time || "0:0").split(':');
+                    // config.weekdays 应该是 [1, 3, 5] 这种数组
+                    const days = Array.isArray(config.weekdays) ? config.weekdays.join(',') : '*';
+                    return `0 ${parseInt(minute)} ${parseInt(hour)} * * ${days}`;
+                }
+
+                default:
+                    return '0 0 0 * * *';
+            }
+        }
+
+        return '0 0 0 * * *';
+    }
+
+    getNextExecutionTime(cron) {
+
+        try {
+            const { CronExpressionParser } = require('cron-parser');
+            const interval = CronExpressionParser.parse(cron, { tz: 'Asia/Shanghai', utc: false });
+            return new Date(interval.next().toISOString()).getTime()
+        } catch (e) {
+            return '-';
+        }
+    }
+}
+
+module.exports = CronScheduler;

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "echoapi-cron-scheduler-batch",
+  "version": "1.0.1",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "rtc",
+  "license": "MIT",
+  "dependencies": {
+    "axios": "^1.13.4",
+    "better-sqlite3": "^12.6.2",
+    "cron-parser": "^5.5.0",
+    "database-query": "^1.1.19",
+    "dayjs": "^1.11.19",
+    "json5": "^2.2.3",
+    "lodash": "^4.17.23",
+    "node-schedule": "^2.1.1",
+    "runner-runtime": "^1.0.120",
+    "uuid": "^9.0.0"
+  }
+}