@xdxer/dingtalk-agent 0.1.4-beta.8 → 0.1.4-beta.9

package/dist/src/robot-evals.js

@@ -0,0 +1,770 @@
+// 可复用机器人评测：机器人是稳定资源，connector/workspace/run 是一次性测试实例。
+// 所有真实钉钉动作仍只走 dws；本模块固定前缀、身份、模型、预算、marker、回读与 teardown。
+import { cpSync, existsSync, mkdirSync, readFileSync, realpathSync, renameSync, writeFileSync, } from 'node:fs';
+import { createHash, randomUUID } from 'node:crypto';
+import { dirname, join, relative, resolve } from 'node:path';
+import { resolvePackageRoot } from './package-root.js';
+import * as dws from './dws.js';
+import { bindRequiredBasicSkill, relativeBinding, verifyRequiredBasicSkill, } from './opencode-workspace.js';
+const sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+export function runRobotEvaluation(root, poolFile, suiteFile, options = {}) {
+    const pool = loadRobotPool(root, poolFile);
+    const suite = loadRobotEvalSuite(root, suiteFile);
+    const selection = selectEvaluation(pool, suite, options.lanes || []);
+    const plan = {
+        suite: suite.id,
+        prefix: pool.prefix,
+        lanes: [...new Set(selection.cases.map((item) => item.lane))],
+        robots: selection.slots.map((slot) => ({
+            id: slot.id, name: slot.name, lane: slot.lane,
+            appName: slot.appName,
+            channel: slot.channel, model: slot.model, agentMemory: slot.agentMemory,
+            workspaceTemplate: slot.workspaceTemplate,
+        })),
+        cases: selection.cases.map((item) => ({
+            id: item.id, category: item.category, lane: item.lane,
+            scenario: item.scenario || null, risk: item.risk || null,
+            kind: item.kind || 'behavior', capability: item.capability || 'unspecified',
+            robot: selection.assignments.get(item.id).name,
+            continuationOf: item.continuationOf || null,
+        })),
+        hostContract: {
+            host: 'opencode',
+            requiredSkill: 'dingtalk-basic-behavior',
+            loadStrategy: 'opencode.json#instructions',
+            preflightBeforeMessage: true,
+        },
+    };
+    if (!options.live) {
+        return {
+            $schema: 'dingtalk-agent/robot-eval-plan@1', ready: true,
+            sideEffect: false, liveRequired: true, plan,
+        };
+    }
+    if (!options.yes) {
+        throw new Error('Robot Eval 会启动 connector 并以当前用户身份真实发消息；必须同时传 --live --yes');
+    }
+    const runId = `robot_eval_${Date.now()}_${randomUUID().slice(0, 8)}`;
+    const evidencePath = safeInside(root, join(suite.evidenceRoot, runId), false);
+    mkdirSync(evidencePath, { recursive: true });
+    writeJson(join(evidencePath, 'suite.json'), suite);
+    writeJson(join(evidencePath, 'pool.redacted.json'), redactPool(pool));
+    writeJson(join(evidencePath, 'plan.json'), plan);
+    const workspaces = new Map();
+    const connectorReports = [];
+    const teardownReports = [];
+    const started = [];
+    const results = [];
+    let fatal = null;
+    try {
+        // 所有需要的 connector 先启动完成，最多 3 个可同时保持在线；随后才开始发送题目。
+        for (const slot of selection.slots) {
+            const workspace = materializeWorkspace(root, runId, slot);
+            workspaces.set(slot.id, workspace);
+            const preflight = process.env.DTA_LAB_FAKE_DWS === '1'
+                ? {
+                    ...relativeBinding(workspace.binding),
+                    verification: 'static-contract',
+                    packageSourceHash: workspace.packageSourceHash,
+                }
+                : {
+                    ...relativeBinding(verifyRequiredBasicSkill(workspace.binding)),
+                    verification: 'opencode-resolved',
+                    packageSourceHash: workspace.packageSourceHash,
+                };
+            writeJson(join(evidencePath, 'workspaces', `${safeName(slot.id)}.json`), preflight);
+            const report = startConnector(pool, slot, workspace.path, preflight);
+            connectorReports.push(report);
+            started.push(slot);
+        }
+        writeJson(join(evidencePath, 'connectors.json'), connectorReports);
+        const orderedCases = [
+            ...selection.cases.filter((item) => item.kind === 'load-probe'),
+            ...selection.cases.filter((item) => item.kind !== 'load-probe'),
+        ];
+        for (const testCase of orderedCases) {
+            const slot = selection.assignments.get(testCase.id);
+            const result = runCase(pool, suite, testCase, slot, evidencePath);
+            results.push(result);
+            if (testCase.kind === 'load-probe' && !result.passed) {
+                throw new Error(`Basic Skill load probe 失败: case=${testCase.id}；后续行为题已中止`);
+            }
+        }
+    }
+    catch (error) {
+        fatal = error;
+        writeJson(join(evidencePath, 'fatal.json'), {
+            message: fatal.message, at: new Date().toISOString(),
+        });
+    }
+    finally {
+        for (const slot of [...started].reverse()) {
+            teardownReports.push(stopConnector(pool, slot));
+        }
+        writeJson(join(evidencePath, 'teardown.json'), {
+            status: 'complete', connectors: teardownReports,
+            stoppedOnly: started.map((slot) => slot.name), completedAt: new Date().toISOString(),
+        });
+    }
+    const passedCases = results.filter((item) => item.passed).length;
+    const selectedLoadProbes = selection.cases.filter((item) => item.kind === 'load-probe');
+    const evaluatedLoadProbes = results.filter((item) => item.kind === 'load-probe');
+    const loadGate = evaluatedLoadProbes.length === selectedLoadProbes.length &&
+        evaluatedLoadProbes.every((item) => item.passed);
+    const capabilitySummary = Object.fromEntries([...new Set(results.map((item) => item.capability))].map((capability) => {
+        const capabilityResults = results.filter((item) => item.capability === capability);
+        return [capability, {
+                total: capabilityResults.length,
+                passed: capabilityResults.filter((item) => item.passed).length,
+            }];
+    }));
+    const report = {
+        $schema: 'dingtalk-agent/robot-eval-report@1',
+        passed: !fatal && passedCases === selection.cases.length,
+        sideEffect: true,
+        runId,
+        evidencePath: relative(root, evidencePath).split('\\').join('/'),
+        plan,
+        summary: {
+            total: selection.cases.length,
+            passed: passedCases,
+            failed: selection.cases.length - passedCases,
+            platformVisible: results.filter((item) => item.platformVisible).length,
+            exactAttributed: results.filter((item) => item.exactAttributed).length,
+            loadGate,
+            capabilities: capabilitySummary,
+            manualReview: 'not-collected',
+        },
+        results,
+        fatal: fatal?.message || null,
+        teardown: teardownReports.map((item) => ({ slot: item.slot, stopped: item.stopped })),
+    };
+    writeJson(join(evidencePath, 'report.json'), report);
+    return report;
+}
+export function loadRobotPool(root, file) {
+    const value = readJsonFile(safeInside(root, file, true), 'Robot Pool');
+    return validatePool(root, value);
+}
+export function loadRobotEvalSuite(root, file) {
+    const value = readJsonFile(safeInside(root, file, true), 'Robot Eval Suite');
+    return validateSuite(root, value);
+}
+function selectEvaluation(pool, suite, lanes) {
+    const requested = lanes.length ? new Set(lanes) : new Set(suite.cases.map((item) => item.lane));
+    const cases = suite.cases.filter((item) => requested.has(item.lane));
+    if (!cases.length)
+        throw new Error('Robot Eval 选择后没有 case');
+    const localOnly = cases.filter((item) => item.execution?.host === 'opencode-only');
+    if (localOnly.length) {
+        throw new Error(`Robot Eval 不执行 opencode-only case: ${localOnly.map((item) => item.id).join(',')}`);
+    }
+    if (cases.length > pool.budgets.maxCases) {
+        throw new Error(`Robot Eval case 超过预算: ${cases.length}/${pool.budgets.maxCases}`);
+    }
+    const assignments = new Map();
+    const selected = new Map();
+    for (const testCase of cases) {
+        const matches = pool.slots.filter((slot) => slot.lane === testCase.lane);
+        if (matches.length !== 1) {
+            throw new Error(`lane=${testCase.lane} 必须恰好绑定一个机器人，当前 ${matches.length} 个`);
+        }
+        assignments.set(testCase.id, matches[0]);
+        selected.set(matches[0].id, matches[0]);
+    }
+    const slots = [...selected.values()];
+    if (slots.length > pool.budgets.maxRobots || slots.length > 3) {
+        throw new Error(`Robot Eval 同时机器人超过上限: ${slots.length}`);
+    }
+    return { cases, slots, assignments };
+}
+function startConnector(pool, slot, workspace, basicSkillLoad) {
+    const app = dws.run([
+        'dev', 'app', 'get', '--unified-app-id', slot.unifiedAppId,
+        ...profileArgs(pool.dws.profile),
+    ], { retries: 0 });
+    const appJson = JSON.stringify(app);
+    if (!appJson.includes(slot.unifiedAppId) || !appJson.includes(slot.appName)) {
+        throw new Error(`机器人 ${slot.name} 的 unifiedAppId 无法按应用详情回读`);
+    }
+    const bot = dws.run([
+        'chat', 'bot', 'find', '--query', slot.name, '--limit', '20',
+        ...profileArgs(pool.dws.profile),
+    ], { retries: 0 });
+    const botJson = JSON.stringify(bot);
+    if (!botJson.includes(slot.name) || !botJson.includes(slot.botOpenDingTalkId)) {
+        throw new Error(`机器人 ${slot.name} 的 openDingTalkId 无法按名称回读`);
+    }
+    const before = dws.run([
+        'dev', 'connect', 'status', '--robot-client-id', slot.robotClientId, '--json',
+        ...profileArgs(pool.dws.profile),
+    ], { allowFail: true, retries: 0 });
+    if (connectorHealthy(before)) {
+        throw new Error(`机器人 ${slot.name} 已有未知配置的 healthy connector；先显式停止，避免复用错误模型/工作区`);
+    }
+    let owned = false;
+    try {
+        const start = dws.startConnectDaemon([
+            'dev', 'connect', '--unified-app-id', slot.unifiedAppId,
+            '--channel', slot.channel,
+            '--agent-model', slot.model,
+            '--agent-workdir', workspace,
+            '--allowed-users', pool.dws.expectedUserId,
+            '--agent-permission-mode', 'ask',
+            '--agent-approval-mode', 'ask',
+            `--agent-memory=${String(slot.agentMemory)}`,
+            '--reply-card=false',
+            '--user-rate-limit', '10',
+            '--agent-timeout', String(pool.budgets.maxWaitSeconds),
+            '--daemon', '--yes',
+            ...profileArgs(pool.dws.profile),
+        ], { timeout: 60_000 });
+        owned = true;
+        const deadline = Date.now() + 30_000;
+        let status = null;
+        while (Date.now() < deadline) {
+            status = dws.run([
+                'dev', 'connect', 'status', '--robot-client-id', slot.robotClientId, '--json',
+                ...profileArgs(pool.dws.profile),
+            ], { allowFail: true, retries: 0 });
+            if (connectorHealthy(status))
+                break;
+            sleep(1000);
+        }
+        if (!connectorHealthy(status))
+            throw new Error(`机器人 ${slot.name} connector 未进入 healthy`);
+        return {
+            slot: slot.name, lane: slot.lane, channel: slot.channel, model: slot.model,
+            agentMemory: slot.agentMemory, workspace, started: Boolean(start), healthy: true,
+            basicSkillLoad,
+        };
+    }
+    catch (error) {
+        // daemon 进程一旦由本次 Run 创建，就归本次 Run 所有；即使健康检查失败也要精确回收。
+        if (owned) {
+            stopConnector(pool, slot);
+        }
+        throw error;
+    }
+}
+function stopConnector(pool, slot) {
+    let command = null;
+    let error = '';
+    try {
+        command = dws.stopConnectDaemon([
+            'dev', 'connect', 'stop', '--robot-client-id', slot.robotClientId,
+            '--yes', ...profileArgs(pool.dws.profile),
+        ]);
+    }
+    catch (caught) {
+        error = String(caught.message || caught);
+    }
+    const deadline = Date.now() + 10_000;
+    let status = null;
+    while (Date.now() < deadline) {
+        status = dws.run([
+            'dev', 'connect', 'status', '--robot-client-id', slot.robotClientId, '--json',
+            ...profileArgs(pool.dws.profile),
+        ], { allowFail: true, retries: 0 });
+        if (connectorNotRunning(status))
+            break;
+        sleep(500);
+    }
+    return {
+        slot: slot.name,
+        stopped: connectorNotRunning(status),
+        commandAccepted: Boolean(command),
+        status,
+        error: error || null,
+    };
+}
+function runCase(pool, suite, testCase, slot, evidenceRoot) {
+    const uuid = randomUUID();
+    const marker = `${suite.markerPrefix}${safeName(suite.id)}-${safeName(testCase.id)}-${uuid}]`;
+    const outbound = `${marker}\n评测协议：回复第一行原样保留上面的 marker，不要解释评测协议。\n${testCase.prompt}`;
+    if (outbound.length > pool.budgets.maxTextCharacters) {
+        throw new Error(`case=${testCase.id} 超过文本预算 ${pool.budgets.maxTextCharacters}`);
+    }
+    const casePath = join(evidenceRoot, 'cases', safeName(testCase.id));
+    mkdirSync(casePath, { recursive: true });
+    const startedAt = Date.now();
+    writeJson(join(casePath, 'request.json'), {
+        case: testCase, robot: slot.name, marker, outbound, uuid, startedAt: new Date(startedAt).toISOString(),
+    });
+    let send = null;
+    let readback = null;
+    let reply = '';
+    let candidateReply = '';
+    let error = '';
+    try {
+        send = dws.run([
+            'chat', 'message', 'send', '--open-dingtalk-id', slot.botOpenDingTalkId,
+            '--text', outbound, '--uuid', uuid, '--yes', ...profileArgs(pool.dws.profile),
+        ], { retries: 0 });
+        const deadline = Date.now() + pool.budgets.maxWaitSeconds * 1000;
+        while (Date.now() < deadline) {
+            readback = dws.run([
+                'chat', 'message', 'list', '--open-dingtalk-id', slot.botOpenDingTalkId,
+                '--time', formatLocalTime(new Date(startedAt - 10_000)),
+                '--direction', 'newer', '--limit', '100', ...profileArgs(pool.dws.profile),
+            ], { retries: 0 });
+            reply = extractReply(readback, marker, outbound);
+            if (reply)
+                break;
+            candidateReply = extractCandidateReply(readback, uuid, outbound);
+            if (candidateReply) {
+                error = '平台已回收到同 UUID 回复，但 marker 被改写，不能作为本 case 的精确归因证据';
+                break;
+            }
+            sleep(3000);
+        }
+        if (!reply && !candidateReply) {
+            error = `等待 ${pool.budgets.maxWaitSeconds}s 未回收到带 marker 的机器人回复`;
+        }
+    }
+    catch (caught) {
+        error = String(caught.message || caught);
+    }
+    const answer = reply
+        ? reply.replace(marker, '').trim()
+        : stripEvalMarker(candidateReply);
+    const grading = gradeAnswer(answer, testCase.expectations);
+    const platformVisible = Boolean(reply || candidateReply);
+    const exactAttributed = Boolean(reply);
+    const attribution = exactAttributed ? 'exact-marker' : candidateReply ? 'marker-mismatch' : 'missing';
+    const passed = exactAttributed && grading.every((item) => item.passed);
+    const result = {
+        id: testCase.id,
+        category: testCase.category,
+        lane: testCase.lane,
+        kind: testCase.kind || 'behavior',
+        capability: testCase.capability || 'unspecified',
+        robot: slot.name,
+        marker,
+        platformVisible,
+        exactAttributed,
+        attribution,
+        passed,
+        reply: answer,
+        grading,
+        manualReview: testCase.manualChecks.map((text) => ({ text, status: 'not-collected' })),
+        error: error || null,
+        durationMs: Date.now() - startedAt,
+    };
+    writeJson(join(casePath, 'send.json'), send || { status: 'not-sent', error });
+    writeJson(join(casePath, 'platform-readback.json'), readback || { status: 'unverified', error });
+    writeJson(join(casePath, 'result.json'), result);
+    return result;
+}
+export function gradeAnswer(answer, expectations) {
+    const normalized = answer.toLocaleLowerCase();
+    return expectations.map((expectation) => {
+        if (expectation.type === 'equals') {
+            return {
+                id: expectation.id,
+                passed: answer.trim() === expectation.value,
+                detail: `equals: actual=${JSON.stringify(answer.trim())}, expected=${JSON.stringify(expectation.value)}`,
+            };
+        }
+        if (expectation.type === 'maxQuestions') {
+            const count = (answer.match(/[?？]/g) || []).length;
+            return {
+                id: expectation.id, passed: count <= expectation.max,
+                detail: `questions=${count}, max=${expectation.max}`,
+            };
+        }
+        if (expectation.type === 'matches' || expectation.type === 'notMatches') {
+            const regex = new RegExp(expectation.pattern, expectation.flags || 'u');
+            const matched = regex.test(answer);
+            return {
+                id: expectation.id,
+                passed: expectation.type === 'matches' ? matched : !matched,
+                detail: `${expectation.type}: pattern=${JSON.stringify(expectation.pattern)}, matched=${matched}`,
+            };
+        }
+        const textExpectation = expectation;
+        const values = textExpectation.values.map((value) => value.toLocaleLowerCase());
+        const hits = values.filter((value) => normalized.includes(value));
+        const passed = textExpectation.type === 'includesAny'
+            ? hits.length > 0
+            : textExpectation.type === 'includesAll'
+                ? hits.length === values.length
+                : hits.length === 0;
+        return {
+            id: textExpectation.id, passed,
+            detail: `${textExpectation.type}: hits=${JSON.stringify(hits)}, expected=${JSON.stringify(textExpectation.values)}`,
+        };
+    });
+}
+function materializeWorkspace(root, runId, slot) {
+    const source = safeInside(root, slot.workspaceTemplate, true);
+    const destination = safeInside(root, join('.dingtalk-agent', 'robot-eval-workspaces', runId, safeName(slot.id)), false);
+    mkdirSync(dirname(destination), { recursive: true });
+    cpSync(source, destination, { recursive: true, errorOnExist: true });
+    const packageRoot = resolvePackageRoot(import.meta.url);
+    const basicSource = join(packageRoot, 'skills', 'dingtalk-basic-behavior');
+    const basicTarget = join(destination, '.agents', 'skills', 'dingtalk-basic-behavior');
+    mkdirSync(dirname(basicTarget), { recursive: true });
+    cpSync(basicSource, basicTarget, { recursive: true, errorOnExist: true });
+    const binding = bindRequiredBasicSkill(destination);
+    const packageSourceHash = createHash('sha256')
+        .update(readFileSync(join(basicSource, 'SKILL.md'), 'utf8')).digest('hex');
+    if (packageSourceHash !== binding.skill.hash) {
+        throw new Error('Robot Eval 物化后的 Basic Skill 与当前 package source hash 不一致');
+    }
+    return { path: destination, binding, packageSourceHash };
+}
+function validatePool(root, value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        throw new Error('Robot Pool 必须是 JSON object');
+    }
+    const input = value;
+    if (input.$schema !== 'dingtalk-agent/robot-pool@1')
+        throw new Error('Robot Pool schema 非法');
+    if (!String(input.prefix || '').trim())
+        throw new Error('Robot Pool 缺少 prefix');
+    if (!input.dws || typeof input.dws.profile !== 'string' || !String(input.dws.expectedUserId || '').trim()) {
+        throw new Error('Robot Pool dws 必须固定 profile 与 expectedUserId');
+    }
+    const budget = input.budgets || {};
+    if (!Number.isInteger(budget.maxRobots) || budget.maxRobots < 1 || budget.maxRobots > 3 ||
+        !Number.isInteger(budget.maxCases) || budget.maxCases < 1 || budget.maxCases > 30 ||
+        !Number.isInteger(budget.maxTextCharacters) || budget.maxTextCharacters < 1 || budget.maxTextCharacters > 4000 ||
+        !Number.isInteger(budget.maxWaitSeconds) || budget.maxWaitSeconds < 10 || budget.maxWaitSeconds > 300) {
+        throw new Error('Robot Pool budgets 非法（robots 1..3 / cases 1..30 / text 1..4000 / wait 10..300）');
+    }
+    if (!Array.isArray(input.slots) || input.slots.length < 1 || input.slots.length > 3) {
+        throw new Error('Robot Pool slots 必须为 1..3 个');
+    }
+    const seen = {
+        id: new Set(), name: new Set(), app: new Set(), task: new Set(),
+        client: new Set(), bot: new Set(), lane: new Set(),
+    };
+    for (const slot of input.slots) {
+        for (const key of ['id', 'appName', 'name', 'lane', 'unifiedAppId', 'robotClientId', 'botOpenDingTalkId', 'workspaceTemplate', 'channel', 'model']) {
+            if (!String(slot?.[key] || '').trim())
+                throw new Error(`Robot Pool slot 缺少 ${key}`);
+        }
+        if (!slot.name.startsWith(input.prefix))
+            throw new Error(`机器人名不符合 prefix: ${slot.name}`);
+        if (!['stateless', 'stateful', 'role'].includes(slot.lane))
+            throw new Error(`机器人 lane 非法: ${slot.lane}`);
+        if (slot.channel !== 'opencode')
+            throw new Error(`Robot Eval 只允许 opencode: ${slot.channel}`);
+        if (!slot.model.startsWith('deepseek/'))
+            throw new Error(`Robot Eval 模型必须是 deepseek/*: ${slot.model}`);
+        if (typeof slot.agentMemory !== 'boolean')
+            throw new Error(`机器人 ${slot.name} 缺少 agentMemory boolean`);
+        if (slot.lane === 'stateful' && slot.agentMemory !== true)
+            throw new Error('stateful lane 必须 agentMemory=true');
+        if (slot.lane === 'stateless' && slot.agentMemory !== false)
+            throw new Error('stateless lane 必须 agentMemory=false');
+        const uniqueKeys = [
+            ['id', slot.id], ['name', slot.name],
+            ['app', slot.unifiedAppId],
+            ...(slot.creationTaskId ? [['task', slot.creationTaskId]] : []),
+            ['client', slot.robotClientId], ['bot', slot.botOpenDingTalkId], ['lane', slot.lane],
+        ];
+        for (const [kind, key] of uniqueKeys) {
+            if (seen[kind].has(key))
+                throw new Error(`Robot Pool ${kind} 重复: ${key}`);
+            seen[kind].add(key);
+        }
+        safeInside(root, slot.workspaceTemplate, true);
+    }
+    return input;
+}
+function validateSuite(root, value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        throw new Error('Robot Eval Suite 必须是 JSON object');
+    }
+    const input = value;
+    if (input.$schema !== 'dingtalk-agent/robot-eval-suite@1')
+        throw new Error('Robot Eval Suite schema 非法');
+    if (!String(input.id || '').trim() || !String(input.markerPrefix || '').startsWith('[DTA-EVAL-')) {
+        throw new Error('Robot Eval Suite 缺少 id 或合法 markerPrefix');
+    }
+    if (!String(input.evidenceRoot || '').trim())
+        throw new Error('Robot Eval Suite 缺少 evidenceRoot');
+    safeInside(root, input.evidenceRoot, false);
+    if (input.comparisonPolicy !== undefined) {
+        const policy = input.comparisonPolicy;
+        const keys = policy && typeof policy === 'object' && !Array.isArray(policy)
+            ? Object.keys(policy).sort()
+            : [];
+        const expectedKeys = [
+            'maximumRegressedCases', 'minimumImprovedCases',
+            'minimumOverallPassRateDelta', 'minimumRunsPerConfiguration',
+        ];
+        if (JSON.stringify(keys) !== JSON.stringify(expectedKeys) ||
+            !Number.isInteger(policy.minimumRunsPerConfiguration) ||
+            policy.minimumRunsPerConfiguration < 3 || policy.minimumRunsPerConfiguration > 10 ||
+            typeof policy.minimumOverallPassRateDelta !== 'number' ||
+            policy.minimumOverallPassRateDelta <= 0 || policy.minimumOverallPassRateDelta > 1 ||
+            !Number.isInteger(policy.minimumImprovedCases) || policy.minimumImprovedCases < 1 ||
+            !Number.isInteger(policy.maximumRegressedCases) || policy.maximumRegressedCases < 0) {
+            throw new Error('Robot Eval Suite comparisonPolicy 非法');
+        }
+    }
+    if (!Array.isArray(input.cases) || input.cases.length < 1 || input.cases.length > 30) {
+        throw new Error('Robot Eval Suite cases 必须为 1..30 个');
+    }
+    const ids = new Set();
+    for (const testCase of input.cases) {
+        if (!/^[a-z0-9][a-z0-9._-]*$/.test(testCase?.id || ''))
+            throw new Error(`case id 非法: ${testCase?.id}`);
+        if (ids.has(testCase.id))
+            throw new Error(`case id 重复: ${testCase.id}`);
+        ids.add(testCase.id);
+        if (!['stateless', 'stateful', 'role'].includes(testCase.lane) || !String(testCase.prompt || '').trim()) {
+            throw new Error(`case=${testCase.id} 缺少合法 lane/prompt`);
+        }
+        if (testCase.kind !== undefined && !['load-probe', 'behavior'].includes(testCase.kind)) {
+            throw new Error(`case=${testCase.id} kind 非法`);
+        }
+        if (testCase.capability !== undefined &&
+            !['agent-body', 'basic-skill', 'role-skill', 'session-memory', 'knowledge',
+                'workspace', 'artifact', 'platform'].includes(testCase.capability)) {
+            throw new Error(`case=${testCase.id} capability 非法`);
+        }
+        if (testCase.completionClaim !== undefined &&
+            !['forbidden', 'permitted'].includes(testCase.completionClaim)) {
+            throw new Error(`case=${testCase.id} completionClaim 非法`);
+        }
+        if (testCase.scenario !== undefined &&
+            !['host-loading', 'conversation-contract', 'authority-and-egress',
+                'task-lifecycle', 'memory-and-storage', 'workspace-and-artifacts',
+                'platform-reliability'].includes(testCase.scenario)) {
+            throw new Error(`case=${testCase.id} scenario 非法`);
+        }
+        if (testCase.risk !== undefined && !['safety', 'reliability', 'quality'].includes(testCase.risk)) {
+            throw new Error(`case=${testCase.id} risk 非法`);
+        }
+        if (testCase.kind === 'load-probe' && testCase.capability !== 'basic-skill') {
+            throw new Error(`case=${testCase.id} load-probe 必须归类为 basic-skill`);
+        }
+        if (!Array.isArray(testCase.expectations) || !testCase.expectations.length ||
+            !Array.isArray(testCase.manualChecks)) {
+            throw new Error(`case=${testCase.id} 缺少 expectations/manualChecks`);
+        }
+        if (testCase.execution !== undefined &&
+            (testCase.execution.host !== 'opencode-only' ||
+                !['none', 'workspace-read', 'workspace-write',
+                    'workspace-write-verify'].includes(testCase.execution.tools))) {
+            throw new Error(`case=${testCase.id} execution 非法`);
+        }
+        if (testCase.execution?.tools !== undefined && testCase.execution.tools !== 'none' &&
+            (!Array.isArray(testCase.postconditions) || !testCase.postconditions.length)) {
+            throw new Error(`case=${testCase.id} 文件工具模式必须声明 postconditions`);
+        }
+        if (testCase.postconditions !== undefined) {
+            if (!Array.isArray(testCase.postconditions) || !testCase.postconditions.length) {
+                throw new Error(`case=${testCase.id} postconditions 非法`);
+            }
+            for (const expectation of testCase.postconditions) {
+                validateEvidenceExpectation(testCase.id, expectation);
+            }
+        }
+        if (testCase.sourceRefs !== undefined &&
+            (!Array.isArray(testCase.sourceRefs) ||
+                testCase.sourceRefs.some((item) => !String(item || '').trim()))) {
+            throw new Error(`case=${testCase.id} sourceRefs 非法`);
+        }
+        for (const expectation of testCase.expectations)
+            validateExpectation(testCase.id, expectation);
+    }
+    for (const testCase of input.cases) {
+        if (testCase.continuationOf) {
+            const parent = input.cases.find((item) => item.id === testCase.continuationOf);
+            if (!parent || parent.lane !== testCase.lane || testCase.lane !== 'stateful') {
+                throw new Error(`case=${testCase.id} continuationOf 必须指向同一 stateful lane 的已有 case`);
+            }
+            if (input.cases.indexOf(parent) >= input.cases.indexOf(testCase)) {
+                throw new Error(`case=${testCase.id} continuationOf 必须位于当前 case 之前`);
+            }
+        }
+    }
+    return input;
+}
+function validateEvidenceExpectation(caseId, expectation) {
+    if (!String(expectation?.id || '').trim() ||
+        !['filesystem', 'workspace', 'artifact'].includes(expectation?.surface) ||
+        !['pathExists', 'pathAbsent', 'textIncludesAll', 'textExcludesAll',
+            'jsonEquals', 'agentDefinition'].includes(expectation?.type)) {
+        throw new Error(`case=${caseId} postcondition 非法`);
+    }
+    if (expectation.type === 'agentDefinition') {
+        if (expectation.surface !== 'workspace' || !['ready', 'partial'].includes(expectation.status) ||
+            !Array.isArray(expectation.requiredSkills)) {
+            throw new Error(`case=${caseId} agentDefinition postcondition 非法`);
+        }
+        return;
+    }
+    const path = String(expectation.path || '');
+    if (!path || resolve('/', path) === path || path.split(/[\\/]/).includes('..')) {
+        throw new Error(`case=${caseId} postcondition path 必须是安全相对路径`);
+    }
+    if (expectation.type === 'pathExists') {
+        if (expectation.kind !== undefined && !['file', 'directory'].includes(expectation.kind)) {
+            throw new Error(`case=${caseId} pathExists.kind 非法`);
+        }
+        if (expectation.minBytes !== undefined &&
+            (!Number.isInteger(expectation.minBytes) || expectation.minBytes < 0)) {
+            throw new Error(`case=${caseId} pathExists.minBytes 非法`);
+        }
+    }
+    if (['textIncludesAll', 'textExcludesAll'].includes(expectation.type) &&
+        (!Array.isArray(expectation.values) || !expectation.values.length ||
+            expectation.values.some((item) => !String(item || '').trim()))) {
+        throw new Error(`case=${caseId} text postcondition values 非法`);
+    }
+    if (expectation.type === 'jsonEquals' &&
+        (typeof expectation.pointer !== 'string' || !expectation.pointer.startsWith('/'))) {
+        throw new Error(`case=${caseId} jsonEquals.pointer 非法`);
+    }
+}
+function validateExpectation(caseId, expectation) {
+    if (!String(expectation?.id || '').trim() ||
+        !['includesAny', 'includesAll', 'excludesAll', 'equals', 'maxQuestions',
+            'matches', 'notMatches'].includes(expectation?.type)) {
+        throw new Error(`case=${caseId} expectation 非法`);
+    }
+    if (expectation.gate !== undefined && !['safety', 'quality'].includes(expectation.gate)) {
+        throw new Error(`case=${caseId} expectation.gate 非法`);
+    }
+    if (expectation.type === 'maxQuestions') {
+        if (!Number.isInteger(expectation.max) || expectation.max < 0 || expectation.max > 3) {
+            throw new Error(`case=${caseId} maxQuestions 非法`);
+        }
+    }
+    else if (expectation.type === 'equals') {
+        if (!String(expectation.value || '').trim())
+            throw new Error(`case=${caseId} equals.value 非法`);
+    }
+    else if (expectation.type === 'matches' || expectation.type === 'notMatches') {
+        if (!String(expectation.pattern || '').trim() || expectation.pattern.length > 500 ||
+            (expectation.flags !== undefined && !/^(?=[imu]*$)(?!.*(.).*\1)[imu]*$/.test(expectation.flags))) {
+            throw new Error(`case=${caseId} regex expectation 非法`);
+        }
+        try {
+            new RegExp(expectation.pattern, expectation.flags || 'u');
+        }
+        catch {
+            throw new Error(`case=${caseId} regex pattern 无法编译`);
+        }
+    }
+    else if (!Array.isArray(expectation.values) || !expectation.values.length ||
+        expectation.values.some((item) => !String(item || '').trim())) {
+        throw new Error(`case=${caseId} expectation.values 非法`);
+    }
+}
+function connectorHealthy(value) {
+    return /"(?:state|status)"\s*:\s*"healthy"/i.test(JSON.stringify(value || {}));
+}
+function connectorNotRunning(value) {
+    return /"(?:state|status)"\s*:\s*"not_running"/i.test(JSON.stringify(value || {}));
+}
+function extractReply(value, marker, outbound) {
+    const strings = [];
+    collectStrings(value, strings, 0);
+    const unique = [...new Set(strings.map((item) => item.trim()).filter(Boolean))];
+    const candidates = unique.filter((item) => item.includes(marker) && item !== outbound && !item.includes('评测协议：') &&
+        item.length > marker.length + 1);
+    const preferred = candidates.filter((item) => item.startsWith(marker) && !item.startsWith('{'));
+    return (preferred.length ? preferred : candidates)
+        .sort((a, b) => a.length - b.length)[0] || '';
+}
+function extractCandidateReply(value, uuid, outbound) {
+    const strings = [];
+    collectStrings(value, strings, 0);
+    const unique = [...new Set(strings.map((item) => item.trim()).filter(Boolean))];
+    const candidates = unique.filter((item) => item.includes(uuid) && item !== outbound && !item.includes('评测协议：') &&
+        item.length > uuid.length + 1);
+    const preferred = candidates.filter((item) => item.startsWith('[DTA-EVAL-') && !item.startsWith('{'));
+    return (preferred.length ? preferred : candidates)
+        .sort((a, b) => a.length - b.length)[0] || '';
+}
+function stripEvalMarker(value) {
+    return value.replace(/^\[DTA-EVAL-[^\]\n]+\]\s*/, '').trim();
+}
+function collectStrings(value, output, depth) {
+    if (depth > 8 || value === null || value === undefined)
+        return;
+    if (typeof value === 'string') {
+        output.push(value);
+        if (depth < 4 && /^[\s]*[{[]/.test(value)) {
+            try {
+                collectStrings(JSON.parse(value), output, depth + 1);
+            }
+            catch { /* plain text */ }
+        }
+        return;
+    }
+    if (Array.isArray(value)) {
+        for (const item of value)
+            collectStrings(item, output, depth + 1);
+        return;
+    }
+    if (typeof value === 'object') {
+        for (const item of Object.values(value)) {
+            collectStrings(item, output, depth + 1);
+        }
+    }
+}
+function formatLocalTime(date) {
+    return new Intl.DateTimeFormat('sv-SE', {
+        year: 'numeric', month: '2-digit', day: '2-digit',
+        hour: '2-digit', minute: '2-digit', second: '2-digit', hourCycle: 'h23',
+    }).format(date);
+}
+function redactPool(pool) {
+    return {
+        ...pool,
+        dws: {
+            profile: redact(pool.dws.profile),
+            expectedUserId: redact(pool.dws.expectedUserId),
+        },
+        slots: pool.slots.map((slot) => ({
+            ...slot,
+            unifiedAppId: redact(slot.unifiedAppId),
+            creationTaskId: redact(slot.creationTaskId || ''),
+            robotClientId: redact(slot.robotClientId),
+            botOpenDingTalkId: redact(slot.botOpenDingTalkId),
+        })),
+    };
+}
+function redact(value) {
+    return value ? `<redacted:${createHash('sha256').update(value).digest('hex').slice(0, 10)}>` : '';
+}
+function profileArgs(profile) {
+    return profile ? ['--profile', profile] : [];
+}
+function readJsonFile(path, label) {
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch (error) {
+        throw new Error(`${label} 不是合法 JSON: ${error.message}`);
+    }
+}
+function writeJson(path, value) {
+    mkdirSync(dirname(path), { recursive: true });
+    const temporary = `${path}.${process.pid}.${Date.now()}.tmp`;
+    writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', { flag: 'wx' });
+    renameSync(temporary, path);
+}
+function safeInside(root, path, mustExist) {
+    const rootReal = realpathSync(resolve(root));
+    const candidate = resolve(rootReal, path);
+    const rel = relative(rootReal, candidate);
+    if (rel === '..' || rel.startsWith(`..${process.platform === 'win32' ? '\\' : '/'}`)) {
+        throw new Error(`Robot Eval 路径必须位于当前目录内: ${path}`);
+    }
+    if (mustExist && !existsSync(candidate))
+        throw new Error(`Robot Eval 路径不存在: ${path}`);
+    return candidate;
+}
+function safeName(value) {
+    return value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '') || 'case';
+}
+//# sourceMappingURL=robot-evals.js.map