echoapi-cron-scheduler-batch 1.0.3 → 1.0.5

package/index.js

@@ -16,7 +16,7 @@ const { execSync } = require('child_process');
 class CronScheduler {
     constructor(config = {}) {
         // 配置初始化
-        this.apiUrl = config.apiUrl || process?.env["OPENAPI_DOMAIN"] || "https://open.apipost.net";
+        this.apiUrl = config.apiUrl || process?.env["OPENAPI_DOMAIN"] || "https://ee.apipost.cc";
         this.dbFile = config.dbPath || path.resolve(os.tmpdir(), 'echoapi-batch-tasks.sqlite');
         this.workerNum = config.workerNum || os.cpus().length;
 
@@ -33,6 +33,8 @@ class CronScheduler {
      * 内部方法：初始化数据库表结构
      */
     _initDB() {
+        // 1. 任务配置表
+
         this.db.pragma('journal_mode = WAL');
         this.db.prepare(`
             CREATE TABLE IF NOT EXISTS jobs (
@@ -46,6 +48,17 @@ class CronScheduler {
                 create_dtime INTEGER
             )
         `).run();
+
+        // 2. 新增：中间结果结果暂存表 (用于卸载 Master 内存压力)
+        this.db.prepare(`
+            CREATE TABLE IF NOT EXISTS task_results (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                execution_id TEXT,
+                data TEXT,
+                create_at INTEGER
+            )
+        `).run();
+        this.db.prepare(`CREATE INDEX IF NOT EXISTS idx_exec_id ON task_results(execution_id)`).run();
     }
 
     /**
@@ -100,14 +113,29 @@ class CronScheduler {
 
                 if (record) {
 
-                    record.received.push(_.assign(data, {
-                        source: 'scheduled',
-                        start_at: this.formatTimeToISO(data?.start_at),
-                        end_at: this.formatTimeToISO(data?.end_at)
-                    }));
-                    // 检查是否收齐了该 Job 下所有的 runners 结果
-                    if (record.received.length >= record.total) {
-                        await this._reportAggregated(executionId, record);
+                    // record.received.push(_.assign(data, {
+                    //     source: 'scheduled',
+                    //     start_at: this.formatTimeToISO(data?.start_at),
+                    //     end_at: this.formatTimeToISO(data?.end_at)
+                    // }));
+                    // // 检查是否收齐了该 Job 下所有的 runners 结果
+                    // if (record.received.length >= record.total) {
+                    //     await this._reportAggregated(executionId, record);
+                    // }
+                    try {
+                        // 【关键优化】不再存入内存 Map，而是存入 SQLite 临时表
+                        this.db.prepare(`INSERT INTO task_results (execution_id, data, create_at) VALUES (?, ?, ?)`)
+                            .run(executionId, JSON.stringify(data), Date.now());
+
+                        record.received_count++;
+                        console.log(record.total, record.received_count, 'record.received_count');
+
+                        // 检查是否收齐
+                        if (record.received_count >= record.total) {
+                            await this._reportFromDB(executionId, record);
+                        }
+                    } catch (e) {
+                        console.error(`[Master Result Collect Error]`, e.message);
                     }
                 }
             }
@@ -117,103 +145,103 @@ class CronScheduler {
     /**
      * 将聚合后的结果一次性推送到 OpenAPI
      */
-    async _reportAggregated(executionId, record) {
-        try {
-
-            const results = record.received;
-            const endTime = Date.now();
-
-            // --- 核心统计逻辑 ---
-            let stats = {
-                http_total: 0,
-                http_success: 0,
-                assert_total: 0,
-                assert_success: 0,
-                total_response_time: 0,
-                count: 0
-            };
-
-            results.forEach(item => {
-                // 累加 HTTP 统计
-                stats.http_total += (item.http?.total || 0);
-                stats.http_success += (item.http?.success || 0);
-
-                // 累加 断言 统计
-                stats.assert_total += (item.assert?.total || 0);
-                stats.assert_success += (item.assert?.success || 0);
-
-                // 累加 响应时间 用于计算平均值
-                stats.total_response_time += (item.total_response_time || 0);
-                stats.count++;
-            });
-
-            // 计算通过率 (保留两位小数)
-            const httpPassRate = stats.http_total > 0
-                ? ((stats.http_success / stats.http_total) * 100).toFixed(2)
-                : "0.00";
-
-            const assertPassRate = stats.assert_total > 0
-                ? ((stats.assert_success / stats.assert_total) * 100).toFixed(2)
-                : "0.00";
-
-            // 计算平均响应时间
-            const avgResponseTime = stats.count > 0
-                ? (stats.total_response_time / stats.count).toFixed(0)
-                : 0;
-
-            // --- 封装最终 Payload ---
-            const finalPayload = {
-                info: {
-                    job_id: record.job_info.job_id,
-                    report_name: record.job_info.name,
-                    project_id: record.job_info.project_id,
-                    execution_id: executionId,
-                    start_at: this.formatTimeToISO(record.job_info.start_time),
-                    end_at: this.formatTimeToISO(endTime),
-
-                    // 业务统计字段
-                    http_pass_rate: `${httpPassRate}%`,
-                    assert_pass_rate: `${assertPassRate}%`,
-                    avg_response_time: `${avgResponseTime}ms`,
-
-                    // 原始计数
-                    total_http: stats.http_total,
-                    success_http: stats.http_success,
-                    total_assert: stats.assert_total,
-                    success_assert: stats.assert_success,
-
-                    total_units: record.total,
-                    actual_units: results.length
-                },
-                results: results // 原始详细列表
-            };
-
-            // 写入本地备份，增加 try-catch 避免权限问题
-            try {
-                // fs.writeFileSync('last_report.json', JSON.stringify(finalPayload));
-            } catch (e) { }
-
-            console.log(`[CronScheduler] Batch job finished. ExecutionID: ${executionId}. Aggregating ${record.total} results...`);
-            // console.log(`[CronScheduler] TOKEN ${record.api_token}`);
-
-            const response = await axios.post(`${this.apiUrl}/open/hnzycfc/scheduled_task/report/add`, finalPayload, {
-                headers: {
-                    'Content-Type': 'application/json',
-                    'api-token': record.api_token
-                }
-            });
-            if (response?.data?.code != 0) {
-                console.log(`Execution Id  ${executionId} reporting error: ${response?.data?.msg}`);
-            } else {
-                console.log(`Execution Id ${executionId} reported success...`);
-            }
-        } catch (err) {
-            console.error(`[CronScheduler] Report aggregation failed:`, err.message);
-        } finally {
-            this.resultCollector.delete(executionId); // 清理内存
-            console.info(`[CronScheduler] Result collector cleared for ${executionId}`);
-        }
-    }
+    // async _reportAggregated(executionId, record) {
+    //     try {
+
+    //         const results = record.received;
+    //         const endTime = Date.now();
+
+    //         // --- 核心统计逻辑 ---
+    //         let stats = {
+    //             http_total: 0,
+    //             http_success: 0,
+    //             assert_total: 0,
+    //             assert_success: 0,
+    //             total_response_time: 0,
+    //             count: 0
+    //         };
+
+    //         results.forEach(item => {
+    //             // 累加 HTTP 统计
+    //             stats.http_total += (item.http?.total || 0);
+    //             stats.http_success += (item.http?.success || 0);
+
+    //             // 累加 断言 统计
+    //             stats.assert_total += (item.assert?.total || 0);
+    //             stats.assert_success += (item.assert?.success || 0);
+
+    //             // 累加 响应时间 用于计算平均值
+    //             stats.total_response_time += (item.total_response_time || 0);
+    //             stats.count++;
+    //         });
+
+    //         // 计算通过率 (保留两位小数)
+    //         const httpPassRate = stats.http_total > 0
+    //             ? ((stats.http_success / stats.http_total) * 100).toFixed(2)
+    //             : "0.00";
+
+    //         const assertPassRate = stats.assert_total > 0
+    //             ? ((stats.assert_success / stats.assert_total) * 100).toFixed(2)
+    //             : "0.00";
+
+    //         // 计算平均响应时间
+    //         const avgResponseTime = stats.count > 0
+    //             ? (stats.total_response_time / stats.count).toFixed(0)
+    //             : 0;
+
+    //         // --- 封装最终 Payload ---
+    //         const finalPayload = {
+    //             info: {
+    //                 job_id: record.job_info.job_id,
+    //                 report_name: record.job_info.name,
+    //                 project_id: record.job_info.project_id,
+    //                 execution_id: executionId,
+    //                 start_at: this.formatTimeToISO(record.job_info.start_time),
+    //                 end_at: this.formatTimeToISO(endTime),
+
+    //                 // 业务统计字段
+    //                 http_pass_rate: `${httpPassRate}%`,
+    //                 assert_pass_rate: `${assertPassRate}%`,
+    //                 avg_response_time: `${avgResponseTime}ms`,
+
+    //                 // 原始计数
+    //                 total_http: stats.http_total,
+    //                 success_http: stats.http_success,
+    //                 total_assert: stats.assert_total,
+    //                 success_assert: stats.assert_success,
+
+    //                 total_units: record.total,
+    //                 actual_units: results.length
+    //             },
+    //             results: results // 原始详细列表
+    //         };
+
+    //         // 写入本地备份，增加 try-catch 避免权限问题
+    //         try {
+    //             // fs.writeFileSync('last_report.json', JSON.stringify(finalPayload));
+    //         } catch (e) { }
+
+    //         console.log(`[CronScheduler] Batch job finished. ExecutionID: ${executionId}. Aggregating ${record.total} results...`);
+    //         // console.log(`[CronScheduler] TOKEN ${record.api_token}`);
+
+    //         const response = await axios.post(`${this.apiUrl}/open/hnzycfc/scheduled_task/report/add`, finalPayload, {
+    //             headers: {
+    //                 'Content-Type': 'application/json',
+    //                 'api-token': record.api_token
+    //             }
+    //         });
+    //         if (response?.data?.code != 0) {
+    //             console.log(`Execution Id  ${executionId} reporting error: ${response?.data?.msg}`);
+    //         } else {
+    //             console.log(`Execution Id ${executionId} reported success...`);
+    //         }
+    //     } catch (err) {
+    //         console.error(`[CronScheduler] Report aggregation failed:`, err.message);
+    //     } finally {
+    //         this.resultCollector.delete(executionId); // 清理内存
+    //         console.info(`[CronScheduler] Result collector cleared for ${executionId}`);
+    //     }
+    // }
     /**
      * 查看当前所有待执行（活跃）的任务及其详细信息
      */
@@ -375,6 +403,86 @@ class CronScheduler {
         });
     }
 
+    /**
+     * 从数据库提取数据并聚合报告
+     */
+    async _reportFromDB(executionId, record) {
+        try {
+            console.log(`[CronScheduler] Batch job finished. Aggregating results from DB for: ${executionId}`);
+
+            // 1. 从 DB 读取该执行批次的所有结果
+            const rows = this.db.prepare(`SELECT data FROM task_results WHERE execution_id = ?`).all(executionId);
+
+            let stats = {
+                http_total: 0, http_success: 0,
+                assert_total: 0, assert_success: 0,
+                total_resp_time: 0
+            };
+
+            // 2. 转换数据并计算统计信息
+            const results = rows.map(row => {
+                const item = JSON.parse(row.data);
+
+                // 累加统计
+                stats.http_total += (item.http?.total || 0);
+                stats.http_success += (item.http?.success || 0);
+                stats.assert_total += (item.assert?.total || 0);
+                stats.assert_success += (item.assert?.success || 0);
+                stats.total_resp_time += (item.total_response_time || 0);
+
+                return _.assign(item, {
+                    source: 'scheduled',
+                    start_at: this.formatTimeToISO(item?.start_at),
+                    end_at: this.formatTimeToISO(item?.end_at)
+                });
+            });
+
+            // 3. 计算通过率等指标
+            const httpRate = stats.http_total > 0 ? ((stats.http_success / stats.http_total) * 100).toFixed(2) : "0.00";
+            const assertRate = stats.assert_total > 0 ? ((stats.assert_success / stats.assert_total) * 100).toFixed(2) : "0.00";
+            const avgResp = results.length > 0 ? (stats.total_resp_time / results.length).toFixed(0) : 0;
+
+            // 4. 封装 Payload
+            const finalPayload = {
+                info: {
+                    job_id: record.job_info.job_id,
+                    report_name: record.job_info.name,
+                    project_id: record.job_info.project_id,
+                    user_uid: record.job_info.user_uid,
+                    execution_id: executionId,
+                    start_at: this.formatTimeToISO(record.job_info.start_time),
+                    end_at: this.formatTimeToISO(Date.now()),
+                    http_pass_rate: `${httpRate}%`,
+                    assert_pass_rate: `${assertRate}%`,
+                    avg_response_time: `${avgResp}ms`,
+                    total_units: record.total,
+                    actual_units: results.length
+                },
+                results: results
+            };
+            // fs.writeFileSync('finalPayload.json', JSON.stringify(finalPayload))
+            // 5. 上报 API
+            const response = await axios.post(`${this.apiUrl}/open/hnzycfc/scheduled_task/report/add`, finalPayload, {
+                headers: { 'Content-Type': 'application/json', 'api-token': record.api_token },
+                maxContentLength: Infinity,
+                maxBodyLength: Infinity
+            });
+            // console.log(response?.data, 'response?.data');
+
+            if (response?.data?.code == 0) {
+                console.log(`[CronScheduler] Execution ${executionId} reported success.`);
+                // 成功后删除数据库里的临时结果
+                this.db.prepare(`DELETE FROM task_results WHERE execution_id = ?`).run(executionId);
+            } else {
+                console.error(`[Report Error] ${executionId}: ${response?.data?.msg}`);
+            }
+
+        } catch (err) {
+            console.error(`[Aggregation Critical Error]`, err.stack);
+        } finally {
+            this.resultCollector.delete(executionId);
+        }
+    }
     /**
      * 分发逻辑：将一个批次内的所有 Runner 均衡分配给 Workers
      */
@@ -386,7 +494,7 @@ class CronScheduler {
             const executionId = `${job.job_id}_${Date.now()}`;
             this.resultCollector.set(executionId, {
                 total: runners.length,
-                received: [],
+                received_count: 0, // 仅计数
                 api_token: job.api_token,
                 job_info: {
                     job_id: job.job_id,
@@ -400,8 +508,10 @@ class CronScheduler {
             runners.forEach((runner, index) => {
                 // 负载均衡：选取当前存活的 worker 
                 const workers = Object.values(cluster.workers).filter(w => w.isConnected());
+                console.log(`[Dispatch] Active workers count: ${workers.length}`);
                 const worker = _.sample(workers);
                 if (worker) {
+                    console.log(`[Dispatch] Sending Unit ${index} to Worker ${worker.process.pid}`);
                     worker.send({
                         action: 'EXECUTE_UNIT',
                         payload: {
@@ -412,6 +522,8 @@ class CronScheduler {
                             unit_index: index
                         }
                     });
+                } else {
+                    console.error(`[Dispatch Error] No active workers available to handle job ${job.job_id}`);
                 }
             });
         } catch (e) {
@@ -424,50 +536,64 @@ class CronScheduler {
     // ==========================================
 
     _startWorker() {
-        const { run: runner } = require('runner-runtime');
-        const net = require('net');
-
-        process.on('message', async (msg) => {
-            if (msg.action === 'EXECUTE_UNIT') {
-                const { executionId, api_token, test_events, option } = msg.payload;
-                const socketPath = path.join(os.tmpdir(), `echoapi_${uuidv4()}.sock`);
-
-                const server = net.createServer((socket) => {
-                    socket.on('data', async (stream) => {
-                        try {
-                            const info = JSON.parse(stream.toString());
-                            const { action, data } = info;
-                            await this._handleUnitScript(socket, action, data);
-                        } catch (e) {
-                            socket.write(JSON.stringify({ status: 'error', message: e.message }) + "\n\n");
-                        }
+        try {
+            console.log(`[Worker ${process.pid}] Starting initialization...`);
+            const { run: runner } = require('runner-runtime');
+            const net = require('net');
+
+            process.on('message', async (msg) => {
+                if (msg.action === 'EXECUTE_UNIT') {
+                    const { executionId, api_token, test_events, option } = msg.payload;
+                    const socketPath = path.join(os.tmpdir(), `echoapi_${uuidv4()}.sock`);
+
+                    const server = net.createServer((socket) => {
+                        socket.on('data', async (stream) => {
+                            try {
+                                const info = JSON.parse(stream.toString());
+                                const { action, data } = info;
+                                await this._handleUnitScript(socket, action, data);
+                            } catch (e) {
+                                socket.write(JSON.stringify({ status: 'error', message: e.message }) + "\n\n");
+                            }
+                        });
+                    });
+
+                    // --- 这里是关键！注册错误监听 ---
+                    server.on('error', (err) => {
+                        // 在服务器上看到这个日志，就能定位是权限问题还是路径问题
+                        console.error(`[Worker Socket Server Error] PID: ${process.pid}`);
+                        console.error(`[Error Details] Code: ${err.code} | Message: ${err.message}`);
+                        console.error(`[Attempted Path] ${socketPath}`);
                     });
-                });
 
-                server.listen(socketPath, () => {
-                    const finalOptions = _.cloneDeep(option || {});
-                    const base64Pipe = Buffer.from(socketPath).toString('base64');
-                    _.set(finalOptions, 'env.ELECTRON_PIPE', base64Pipe);
-
-                    runner(test_events, finalOptions, (res) => {
-                        if (res?.action === 'complete') {
-                            process.send({
-                                action: 'UNIT_COMPLETED',
-                                payload: { executionId, data: res.data, api_token }
-                            });
-                            server.close();
-                            if (fs.existsSync(socketPath)) {
-                                try { fs.unlinkSync(socketPath); } catch (e) { }
+                    server.listen(socketPath, () => {
+                        const finalOptions = _.cloneDeep(option || {});
+                        const base64Pipe = Buffer.from(socketPath).toString('base64');
+                        _.set(finalOptions, 'env.ELECTRON_PIPE', base64Pipe);
+
+                        runner(test_events, finalOptions, (res) => {
+                            if (res?.action === 'complete') {
+                                process.send({
+                                    action: 'UNIT_COMPLETED',
+                                    payload: { executionId, data: res.data, api_token }
+                                });
+                                server.close();
+                                if (fs.existsSync(socketPath)) {
+                                    try { fs.unlinkSync(socketPath); } catch (e) { }
+                                }
                             }
-                        }
+                        });
                     });
-                });
-            }
-        });
-        process.on('uncaughtException', (err) => {
-            console.error('[Worker Fatal Error] 捕获到沙箱崩溃:', err.message);
-            // 即使崩溃也不要让子进程立即退出，或者让 Master 自动重启它
-        });
+                }
+            });
+            process.on('uncaughtException', (err) => {
+                console.error('[Worker Fatal Error] 捕获到沙箱崩溃:', err.message);
+                // 即使崩溃也不要让子进程立即退出，或者让 Master 自动重启它
+            });
+        } catch (error) {
+            console.error(`[Worker ${process.pid}] Critical Boot Error:`, err.stack);
+            process.exit(1);
+        }
     }
 
     async _handleUnitScript(socket, action, data) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "echoapi-cron-scheduler-batch",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "",
   "main": "index.js",
   "scripts": {