@xdxer/dingtalk-agent 0.1.5-beta.14 → 0.1.5-beta.15

package/dist/src/opencode-evals.js

@@ -6,7 +6,7 @@ import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, re
 import { tmpdir } from 'node:os';
 import { basename, dirname, join, parse, relative, resolve } from 'node:path';
 import { resolvePackageRoot } from './package-root.js';
-import { bindRequiredBasicSkill, removeRequiredBasicSkillInstruction, verifyRequiredBasicSkill, } from './opencode-workspace.js';
+import { bindRequiredBasicSkill, DEFAULT_OPENCODE_MODEL, removeRequiredBasicSkillInstruction, REQUIRED_BASIC_SKILL_INSTRUCTION, verifyRequiredBasicSkill, } from './opencode-workspace.js';
 import { gradeAnswer, loadRobotEvalSuite, } from './robot-evals.js';
 import { evaluateWorkspaceEvidence, isWorkspacePathInside, relativeWorkspacePath, snapshotWorkspaceEvidence, } from './eval-evidence.js';
 import { bundledSkillPath } from './skill-manager.js';
@@ -47,16 +47,18 @@ export function runOpenCodeEvaluation(root, workspacePath, suitePath, options =
         ? new Set(options.lanes)
         : new Set(suite.cases.map((item) => item.lane));
     const requestedCases = new Set(options.caseIds || []);
-    const knownBehaviorCases = new Set(suite.cases
-        .filter((item) => item.kind !== 'load-probe').map((item) => item.id));
-    const unknownCases = [...requestedCases].filter((id) => !knownBehaviorCases.has(id));
+    const knownCases = new Set(suite.cases.map((item) => item.id));
+    const selectedLoadCases = suite.cases.filter((item) => item.kind === 'load-probe' && requestedCases.has(item.id));
+    const unknownCases = [...requestedCases].filter((id) => !knownCases.has(id));
     if (unknownCases.length)
         throw new Error(`OpenCode Eval case 不存在: ${unknownCases.join(',')}`);
-    const cases = options.loadOnly ? [] : suite.cases.filter((item) => lanes.has(item.lane) && item.kind !== 'load-probe' &&
+    const loadOnly = Boolean(options.loadOnly ||
+        (requestedCases.size > 0 && selectedLoadCases.length === requestedCases.size));
+    const cases = loadOnly ? [] : suite.cases.filter((item) => lanes.has(item.lane) && item.kind !== 'load-probe' &&
         (!requestedCases.size || requestedCases.has(item.id)));
-    if (!cases.length && !options.loadOnly)
+    if (!cases.length && !loadOnly)
         throw new Error('OpenCode Eval 选择后没有行为 case');
-    const model = options.model || 'deepseek/deepseek-chat';
+    const model = options.model || DEFAULT_OPENCODE_MODEL;
     if (!model.startsWith('deepseek/'))
         throw new Error('OpenCode Eval 模型必须固定为 deepseek/*');
     const packageRoot = resolvePackageRoot(import.meta.url);
@@ -218,6 +220,14 @@ export function runOpenCodeEvaluation(root, workspacePath, suitePath, options =
                         baselineDidNotGuess: configuration.expectsLoad
                             ? null : probeExecution.answer.trim() === baselineSentinel &&
                             probeExecution.answer.trim() !== exactProbe,
+                        // baseline 失败其实有两种，混成一个布尔会把它们说成同一件事（issue #67 §7）：
+                        // 「没猜中随机 challenge」是安全属性，「精确吐出 sentinel」是协议遵守。
+                        // 模型输出 <bash> 之类伪调用时前者成立、后者不成立——报告要分面看得见。
+                        // 门禁不因此放宽：baseline 通过仍然要求 sentinel 精确匹配（见 loadProbeCorePassed）。
+                        challengeNotGuessed: configuration.expectsLoad
+                            ? null : !probeExecution.answer.includes(probe),
+                        sentinelExact: configuration.expectsLoad
+                            ? null : probeExecution.answer.trim() === baselineSentinel,
                         noTools: probeExecution.toolCalls === 0,
                         directoryMatched: probeExecution.directoryMatched,
                         sessionDirectory: probeExecution.sessionDirectory,
@@ -373,7 +383,7 @@ export function runOpenCodeEvaluation(root, workspacePath, suitePath, options =
     const comparisonConfiguration = configurations[1].name;
     const primaryResults = results.filter((item) => item.configuration === primaryConfiguration);
     const comparisonResults = results.filter((item) => item.configuration === comparisonConfiguration);
-    const behaviorGate = !options.loadOnly && loadGate &&
+    const behaviorGate = !loadOnly && loadGate &&
         primaryResults.length === cases.length * runs && primaryResults.every((item) => item.passed);
     const expectedResults = cases.length * runs * configurations.length;
     const hardGate = loadGate && results.length === expectedResults &&
@@ -425,14 +435,14 @@ export function runOpenCodeEvaluation(root, workspacePath, suitePath, options =
             hardGate,
             behaviorGate,
             falseCompletionGate,
-            behaviorScoresValid: !options.loadOnly && loadGate && hardGate,
+            behaviorScoresValid: !loadOnly && loadGate && hardGate,
             primaryConfiguration,
             comparisonConfiguration,
-            currentPassRate: loadGate && !options.loadOnly ? primaryPassRate : null,
-            comparisonPassRate: loadGate && !options.loadOnly ? comparisonPassRate : null,
-            withSkillPassRate: loadGate && !options.loadOnly ? primaryPassRate : null,
-            baselinePassRate: loadGate && !options.loadOnly ? comparisonPassRate : null,
-            discriminatingCases: loadGate && hardGate && !options.loadOnly
+            currentPassRate: loadGate && !loadOnly ? primaryPassRate : null,
+            comparisonPassRate: loadGate && !loadOnly ? comparisonPassRate : null,
+            withSkillPassRate: loadGate && !loadOnly ? primaryPassRate : null,
+            baselinePassRate: loadGate && !loadOnly ? comparisonPassRate : null,
+            discriminatingCases: loadGate && hardGate && !loadOnly
                 ? discriminating.filter((item) => item.delta > 0).length : 0,
             effectivenessProven: effectivenessAssessment.passed,
             effectivenessAssessment,
@@ -1470,11 +1480,13 @@ function assertSafeEvalProjectConfig(workspace) {
     for (const instruction of instructions) {
         const raw = instruction.trim();
         const candidate = resolve(workspace, raw);
+        const managedBasicExposure = raw === REQUIRED_BASIC_SKILL_INSTRUCTION && !existsSync(candidate);
         if (!raw || /^https?:\/\//i.test(raw) || raw.startsWith('~/') ||
             raw.startsWith('/') || /^[a-zA-Z]:[\\/]/.test(raw) ||
             raw.split(/[\\/]/).includes('..') || instructionUsesGlob(raw) ||
-            !isWorkspacePathInside(workspace, candidate) || !existsSync(candidate) ||
-            !lstatSync(candidate).isFile()) {
+            !isWorkspacePathInside(workspace, candidate) ||
+            (!managedBasicExposure &&
+                (!existsSync(candidate) || !lstatSync(candidate).isFile()))) {
             throw new Error(`OpenCode Eval instruction 必须是 sandbox 内已存在的本地相对文件: ${instruction}`);
         }
     }