079project 8.0.0 → 9.1.0

package/test_automatic.cjs

@@ -0,0 +1,441 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const http = require('http');
+const { spawn } = require('child_process');
+
+const DEFAULT_HOST = '127.0.0.1';
+const DEFAULT_PORT = 5080;
+const DEFAULT_MAIN = path.join(__dirname, 'main.cjs');
+const DEFAULT_Q = path.join(__dirname, 'test_automatic', 'question.csv');
+const DEFAULT_A = path.join(__dirname, 'test_automatic', 'answer.csv');
+
+const parseArgs = (argv) => {
+    const out = { _: [] };
+    for (const item of argv) {
+        if (!item.startsWith('--')) {
+            out._.push(item);
+            continue;
+        }
+        const eq = item.indexOf('=');
+        if (eq === -1) {
+            out[item.slice(2)] = true;
+        } else {
+            out[item.slice(2, eq)] = item.slice(eq + 1);
+        }
+    }
+    return out;
+};
+
+const coerceScalar = (value) => {
+    if (value === true) return true;
+    if (value === false) return false;
+    if (value == null) return value;
+    const s = String(value).trim();
+    if (!s) return s;
+    const low = s.toLowerCase();
+    if (low === 'true' || low === 'on' || low === 'yes') return true;
+    if (low === 'false' || low === 'off' || low === 'no') return false;
+    if (/^-?\d+(?:\.\d+)?$/.test(s)) return Number(s);
+    return s;
+};
+
+// 极简 CSV 解析：支持单列或多列；支持引号；默认第一行为 header。
+const parseCsvFile = (filePath) => {
+    const content = fs.readFileSync(filePath, 'utf8');
+    const lines = content
+        .split(/\r?\n/)
+        .map((l) => l.trimEnd())
+        .filter((l) => l.trim().length > 0);
+    if (lines.length === 0) return [];
+
+    const parseLine = (line) => {
+        const out = [];
+        let cur = '';
+        let inQuotes = false;
+        for (let i = 0; i < line.length; i++) {
+            const ch = line[i];
+            if (inQuotes) {
+                if (ch === '"') {
+                    if (line[i + 1] === '"') {
+                        cur += '"';
+                        i += 1;
+                    } else {
+                        inQuotes = false;
+                    }
+                } else {
+                    cur += ch;
+                }
+            } else {
+                if (ch === ',') {
+                    out.push(cur);
+                    cur = '';
+                } else if (ch === '"') {
+                    inQuotes = true;
+                } else {
+                    cur += ch;
+                }
+            }
+        }
+        out.push(cur);
+        return out.map((x) => String(x ?? '').trim());
+    };
+
+    const header = parseLine(lines[0]);
+    const rows = [];
+    for (const line of lines.slice(1)) {
+        const cols = parseLine(line);
+        if (cols.every((c) => !c)) continue;
+        const row = {};
+        for (let i = 0; i < header.length; i++) {
+            row[header[i] || `col${i + 1}`] = cols[i] ?? '';
+        }
+        // 多余列拼到最后一列，防止列数不齐
+        if (cols.length > header.length) {
+            const extra = cols.slice(header.length).join(',');
+            const lastKey = header[header.length - 1] || `col${header.length}`;
+            row[lastKey] = String(row[lastKey] || '') + (row[lastKey] ? ',' : '') + extra;
+        }
+        rows.push(row);
+    }
+    return rows;
+};
+
+const normalizeText = (text) => {
+    return String(text || '')
+        .toLowerCase()
+        .replace(/[\r\n\t]+/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+};
+
+const tokenize = (text) => {
+    const s = normalizeText(text);
+    if (!s) return [];
+    return s
+        .split(/[^a-z0-9_\-\u4e00-\u9fff]+/)
+        .filter(Boolean);
+};
+
+const jaccard = (a, b) => {
+    const A = new Set(Array.isArray(a) ? a : []);
+    const B = new Set(Array.isArray(b) ? b : []);
+    if (A.size === 0 && B.size === 0) return 1;
+    if (A.size === 0 || B.size === 0) return 0;
+    let inter = 0;
+    for (const x of A) if (B.has(x)) inter++;
+    const uni = A.size + B.size - inter;
+    return uni <= 0 ? 0 : inter / uni;
+};
+
+const levenshtein = (a, b, maxLen = 2000) => {
+    const A = String(a || '');
+    const B = String(b || '');
+    if (A === B) return 0;
+    if (!A.length) return Math.min(B.length, maxLen);
+    if (!B.length) return Math.min(A.length, maxLen);
+    // 防止极端长文本卡死
+    const a2 = A.slice(0, maxLen);
+    const b2 = B.slice(0, maxLen);
+
+    const n = a2.length;
+    const m = b2.length;
+    const prev = new Array(m + 1);
+    const cur = new Array(m + 1);
+    for (let j = 0; j <= m; j++) prev[j] = j;
+    for (let i = 1; i <= n; i++) {
+        cur[0] = i;
+        const ca = a2.charCodeAt(i - 1);
+        for (let j = 1; j <= m; j++) {
+            const cb = b2.charCodeAt(j - 1);
+            const cost = ca === cb ? 0 : 1;
+            cur[j] = Math.min(
+                prev[j] + 1,
+                cur[j - 1] + 1,
+                prev[j - 1] + cost
+            );
+        }
+        for (let j = 0; j <= m; j++) prev[j] = cur[j];
+    }
+    return prev[m];
+};
+
+const similarityScore = (response, answer) => {
+    const r = normalizeText(response);
+    const a = normalizeText(answer);
+    const tj = jaccard(tokenize(r), tokenize(a));
+    const maxLen = Math.max(r.length, a.length, 1);
+    const dist = levenshtein(r, a, 2000);
+    const cs = Math.max(0, 1 - dist / maxLen);
+    // 创造力：token 重合更代表“意思相近”，字符距离更代表“表达相近”
+    return 0.65 * tj + 0.35 * cs;
+};
+
+const httpJson = (method, url, body, { timeoutMs = 20_000 } = {}) => {
+    return new Promise((resolve, reject) => {
+        const u = new URL(url);
+        const payload = body == null ? '' : JSON.stringify(body);
+        const req = http.request(
+            {
+                method,
+                hostname: u.hostname,
+                port: u.port,
+                path: u.pathname + (u.search || ''),
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Content-Length': Buffer.byteLength(payload)
+                },
+                timeout: timeoutMs
+            },
+            (res) => {
+                let data = '';
+                res.setEncoding('utf8');
+                res.on('data', (chunk) => (data += chunk));
+                res.on('end', () => {
+                    try {
+                        const parsed = data ? JSON.parse(data) : {};
+                        resolve({ status: res.statusCode, json: parsed, raw: data });
+                    } catch (e) {
+                        resolve({ status: res.statusCode, json: null, raw: data });
+                    }
+                });
+            }
+        );
+        req.on('error', reject);
+        req.on('timeout', () => {
+            req.destroy(new Error('request-timeout'));
+        });
+        if (payload) req.write(payload);
+        req.end();
+    });
+};
+
+const waitForReady = async (baseUrl, { timeoutMs = 60_000, intervalMs = 800 } = {}) => {
+    const started = Date.now();
+    while (Date.now() - started < timeoutMs) {
+        try {
+            const r = await httpJson('GET', `${baseUrl}/api/system/status`, null, { timeoutMs: 5_000 });
+            if (r.status && r.status >= 200 && r.status < 500) {
+                return true;
+            }
+        } catch (_e) {
+            // ignore
+        }
+        await new Promise((r) => setTimeout(r, intervalMs));
+    }
+    return false;
+};
+
+const main = async () => {
+    const args = parseArgs(process.argv.slice(2));
+
+    const host = String(args.host || args['gateway-host'] || DEFAULT_HOST);
+    const port = Number(args.port || args['gateway-port'] || args['portGateway'] || DEFAULT_PORT) || DEFAULT_PORT;
+    const baseUrl = `http://${host}:${port}`;
+
+    const mainFile = path.resolve(String(args.main || DEFAULT_MAIN));
+    const qFile = path.resolve(String(args.questions || DEFAULT_Q));
+    const aFile = path.resolve(String(args.answers || DEFAULT_A));
+
+    const passThreshold = Number(args.pass ?? 0.65);
+
+    // 透传给 main.cjs 的参数：除本脚本消费的以外，全部原样传递
+    const reserved = new Set([
+        'main', 'host', 'gateway-host', 'port', 'gateway-port', 'portGateway',
+        'questions', 'answers', 'pass', 'model-json'
+    ]);
+
+    const forwardArgs = [];
+    const modelPatch = {};
+
+    for (const raw of process.argv.slice(2)) {
+        if (!raw.startsWith('--')) {
+            continue;
+        }
+        const key = raw.slice(2).split('=')[0];
+        if (reserved.has(key)) {
+            continue;
+        }
+        // 支持 --model.xxx=yyy 作为运行时 params patch（通过 /api/model/params）
+        if (key.startsWith('model.')) {
+            const k2 = key.slice('model.'.length);
+            const v = raw.includes('=') ? raw.slice(raw.indexOf('=') + 1) : true;
+            modelPatch[k2] = coerceScalar(v);
+            continue;
+        }
+        forwardArgs.push(raw);
+    }
+
+    if (args['model-json']) {
+        try {
+            const obj = JSON.parse(String(args['model-json']));
+            if (obj && typeof obj === 'object') {
+                Object.assign(modelPatch, obj);
+            }
+        } catch (e) {
+            console.warn('[test_automatic] model-json parse failed:', e.message);
+        }
+    }
+
+    if (!fs.existsSync(mainFile)) {
+        console.error('[test_automatic] main.cjs not found:', mainFile);
+        process.exit(2);
+    }
+    if (!fs.existsSync(qFile)) {
+        console.error('[test_automatic] question.csv not found:', qFile);
+        process.exit(2);
+    }
+    if (!fs.existsSync(aFile)) {
+        console.error('[test_automatic] answer.csv not found:', aFile);
+        process.exit(2);
+    }
+
+    console.log(`[test_automatic] starting main: ${mainFile}`);
+    console.log(`[test_automatic] baseUrl: ${baseUrl}`);
+
+    const child = spawn(process.execPath, [mainFile, ...forwardArgs], {
+        stdio: 'inherit',
+        cwd: path.dirname(mainFile),
+        env: { ...process.env }
+    });
+
+    const shutdown = async () => {
+        try {
+            if (!child.killed) {
+                child.kill('SIGINT');
+            }
+        } catch (_e) {
+            // ignore
+        }
+        const deadline = Date.now() + 6_000;
+        while (Date.now() < deadline) {
+            if (child.exitCode != null) return;
+            await new Promise((r) => setTimeout(r, 200));
+        }
+        try {
+            if (!child.killed) {
+                child.kill('SIGKILL');
+            }
+        } catch (_e) {
+            // ignore
+        }
+    };
+
+    const ready = await waitForReady(baseUrl, { timeoutMs: 60_000, intervalMs: 800 });
+    if (!ready) {
+        console.error('[test_automatic] main not ready within timeout');
+        await shutdown();
+        process.exit(3);
+    }
+
+    // 应用运行时参数（不重启 main）
+    if (Object.keys(modelPatch).length) {
+        console.log('[test_automatic] applying /api/model/params patch:', modelPatch);
+        try {
+            await httpJson('POST', `${baseUrl}/api/model/params`, modelPatch, { timeoutMs: 15_000 });
+        } catch (e) {
+            console.warn('[test_automatic] apply model params failed:', e.message);
+        }
+    }
+
+    const qRows = parseCsvFile(qFile);
+    const aRows = parseCsvFile(aFile);
+
+    const pickCol = (row, preferredKeys) => {
+        if (!row || typeof row !== 'object') return '';
+        for (const k of preferredKeys) {
+            if (row[k] != null && String(row[k]).trim()) return String(row[k]);
+        }
+        const keys = Object.keys(row);
+        if (keys.length) return String(row[keys[0]] ?? '');
+        return '';
+    };
+
+    const questions = qRows.map((r) => pickCol(r, ['question', 'q', 'text', 'content']));
+    const answers = aRows.map((r) => pickCol(r, ['answer', 'a', 'text', 'content']));
+
+    const N = Math.min(questions.length, answers.length);
+    if (N === 0) {
+        console.error('[test_automatic] empty questions/answers');
+        await shutdown();
+        process.exit(4);
+    }
+    if (questions.length !== answers.length) {
+        console.warn(`[test_automatic] warning: questions(${questions.length}) != answers(${answers.length}), using N=${N}`);
+    }
+
+    const results = [];
+    let sum = 0;
+    let passed = 0;
+
+    for (let i = 0; i < N; i++) {
+        const q = String(questions[i] || '').trim();
+        const a = String(answers[i] || '').trim();
+        if (!q) {
+            results.push({ i: i + 1, ok: false, reason: 'empty-question' });
+            continue;
+        }
+
+        let reply = '';
+        let status = 0;
+        const started = Date.now();
+        try {
+            const r = await httpJson('POST', `${baseUrl}/api/chat`, { text: q }, { timeoutMs: 30_000 });
+            status = r.status || 0;
+            reply = r?.json?.result?.reply ?? '';
+            if (!reply && r?.json?.result?.result?.reply) {
+                // 兼容某些返回结构
+                reply = r.json.result.result.reply;
+            }
+        } catch (e) {
+            results.push({ i: i + 1, ok: false, question: q, answer: a, error: e.message });
+            continue;
+        }
+        const latency = Date.now() - started;
+        const score = similarityScore(reply, a);
+        const ok = score >= passThreshold;
+        sum += score;
+        if (ok) passed += 1;
+
+        results.push({
+            i: i + 1,
+            ok,
+            score: Number(score.toFixed(4)),
+            status,
+            latency,
+            question: q,
+            expected: a,
+            reply
+        });
+
+        console.log(`[${i + 1}/${N}] score=${score.toFixed(3)} ok=${ok} latency=${latency}ms`);
+    }
+
+    const avg = sum / N;
+    console.log('');
+    console.log('[test_automatic] done');
+    console.log(`[test_automatic] avgScore=${avg.toFixed(4)} pass=${passed}/${N} (threshold=${passThreshold})`);
+
+    // 打印低分样例（最多 5 个）
+    const worst = results
+        .filter((r) => r && r.score != null)
+        .sort((a, b) => a.score - b.score)
+        .slice(0, 5);
+    for (const w of worst) {
+        console.log('---');
+        console.log(`#${w.i} score=${w.score}`);
+        console.log('Q:', w.question);
+        console.log('A:', w.expected);
+        console.log('R:', w.reply);
+    }
+
+    await shutdown();
+    // 以平均分作为退出码信号：>=阈值则 0，否则 5
+    process.exit(avg >= passThreshold ? 0 : 5);
+};
+
+main().catch((err) => {
+    console.error('[test_automatic] fatal:', err);
+    process.exit(1);
+});