@xdxer/dingtalk-agent 0.1.4-beta.8 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +233 -0
- package/README.en.md +167 -0
- package/README.md +101 -248
- package/dist/bin/dingtalk-agent.js +1255 -25
- package/dist/bin/dingtalk-agent.js.map +1 -1
- package/dist/src/actions.js +98 -14
- package/dist/src/actions.js.map +1 -1
- package/dist/src/agent-audit.js +1385 -0
- package/dist/src/agent-audit.js.map +1 -0
- package/dist/src/agent-bindings.js +132 -0
- package/dist/src/agent-bindings.js.map +1 -0
- package/dist/src/agent-definition.js +182 -0
- package/dist/src/agent-definition.js.map +1 -0
- package/dist/src/agent-enhance.js +711 -0
- package/dist/src/agent-enhance.js.map +1 -0
- package/dist/src/agent-platform.js +299 -0
- package/dist/src/agent-platform.js.map +1 -0
- package/dist/src/bootstrap.js +125 -17
- package/dist/src/bootstrap.js.map +1 -1
- package/dist/src/config.js +1 -7
- package/dist/src/config.js.map +1 -1
- package/dist/src/development-workspace.js +755 -0
- package/dist/src/development-workspace.js.map +1 -0
- package/dist/src/doctor.js +74 -15
- package/dist/src/doctor.js.map +1 -1
- package/dist/src/dws.js +145 -0
- package/dist/src/dws.js.map +1 -1
- package/dist/src/eval-evidence.js +193 -0
- package/dist/src/eval-evidence.js.map +1 -0
- package/dist/src/host-detect.js +146 -0
- package/dist/src/host-detect.js.map +1 -0
- package/dist/src/instruction-path.js +270 -0
- package/dist/src/instruction-path.js.map +1 -0
- package/dist/src/invocation.js +36 -0
- package/dist/src/invocation.js.map +1 -0
- package/dist/src/lab.js +679 -0
- package/dist/src/lab.js.map +1 -0
- package/dist/src/lease.js +100 -0
- package/dist/src/lease.js.map +1 -0
- package/dist/src/map.js +157 -0
- package/dist/src/map.js.map +1 -0
- package/dist/src/memory/candidates.js +451 -0
- package/dist/src/memory/candidates.js.map +1 -0
- package/dist/src/memory/completion-evidence.js +536 -0
- package/dist/src/memory/completion-evidence.js.map +1 -0
- package/dist/src/memory/operational.js +263 -0
- package/dist/src/memory/operational.js.map +1 -0
- package/dist/src/memory/remote-state.js +478 -0
- package/dist/src/memory/remote-state.js.map +1 -0
- package/dist/src/memory/task-checkpoints.js +204 -0
- package/dist/src/memory/task-checkpoints.js.map +1 -0
- package/dist/src/multica-deploy.js +1538 -0
- package/dist/src/multica-deploy.js.map +1 -0
- package/dist/src/multica-provider.js +685 -0
- package/dist/src/multica-provider.js.map +1 -0
- package/dist/src/opencode-evals.js +1547 -0
- package/dist/src/opencode-evals.js.map +1 -0
- package/dist/src/opencode-isolation.js +124 -0
- package/dist/src/opencode-isolation.js.map +1 -0
- package/dist/src/opencode-provider.js +532 -0
- package/dist/src/opencode-provider.js.map +1 -0
- package/dist/src/opencode-workspace.js +208 -0
- package/dist/src/opencode-workspace.js.map +1 -0
- package/dist/src/perception.js +225 -0
- package/dist/src/perception.js.map +1 -0
- package/dist/src/personal-event-evals.js +595 -0
- package/dist/src/personal-event-evals.js.map +1 -0
- package/dist/src/promotion.js +786 -0
- package/dist/src/promotion.js.map +1 -0
- package/dist/src/remote-semantic-state-live-evals.js +888 -0
- package/dist/src/remote-semantic-state-live-evals.js.map +1 -0
- package/dist/src/remote-semantic-state-worker.js +38 -0
- package/dist/src/remote-semantic-state-worker.js.map +1 -0
- package/dist/src/remote-state-evals.js +502 -0
- package/dist/src/remote-state-evals.js.map +1 -0
- package/dist/src/response-gate.js +51 -0
- package/dist/src/response-gate.js.map +1 -0
- package/dist/src/robot-evals.js +771 -0
- package/dist/src/robot-evals.js.map +1 -0
- package/dist/src/sessions.js +66 -105
- package/dist/src/sessions.js.map +1 -1
- package/dist/src/setup.js +6 -5
- package/dist/src/setup.js.map +1 -1
- package/dist/src/skill-manager.js +162 -24
- package/dist/src/skill-manager.js.map +1 -1
- package/dist/src/skills.js +2 -1
- package/dist/src/skills.js.map +1 -1
- package/dist/src/storage-evals.js +26 -0
- package/dist/src/storage-evals.js.map +1 -0
- package/dist/src/types.js.map +1 -1
- package/dist/src/upgrade.js +23 -27
- package/dist/src/upgrade.js.map +1 -1
- package/dist/src/version.js +73 -0
- package/dist/src/version.js.map +1 -0
- package/dist/src/waits.js +5 -1
- package/dist/src/waits.js.map +1 -1
- package/dist/src/workspace.js +28 -3
- package/dist/src/workspace.js.map +1 -1
- package/docs/INSTALLATION.md +50 -3
- package/docs/SECOND-AGENT-ACCEPTANCE.md +62 -0
- package/docs/architecture/agent-memory-topology.png +0 -0
- package/docs/architecture/agent-memory-topology.svg +132 -0
- package/docs/architecture/general-agent-kernel-topology.png +0 -0
- package/docs/architecture/general-agent-kernel-topology.svg +149 -0
- package/docs/architecture/provider-bound-development-workspace.png +0 -0
- package/docs/architecture/provider-bound-development-workspace.svg +141 -0
- package/docs/architecture/task-completion-gate.png +0 -0
- package/docs/architecture/task-completion-gate.svg +191 -0
- package/docs/assets/agent-delivery-lifecycle.svg +103 -0
- package/docs/schemas/agent-audit-load-evidence.schema.json +14 -0
- package/docs/schemas/agent-audit.schema.json +92 -0
- package/docs/schemas/agent-bindings.schema.json +54 -0
- package/docs/schemas/agent-definition.schema.json +78 -0
- package/docs/schemas/agent-enhancement-plan.schema.json +88 -0
- package/docs/schemas/agent-enhancement-receipt.schema.json +37 -0
- package/docs/schemas/agent-platform.schema.json +13 -0
- package/docs/schemas/enriched-invocation.schema.json +46 -0
- package/docs/schemas/eval-candidate-plan.schema.json +28 -0
- package/docs/schemas/eval-candidate-result.schema.json +20 -0
- package/docs/schemas/eval-candidate.schema.json +30 -0
- package/docs/schemas/invocation.schema.json +19 -0
- package/docs/schemas/memory-candidate-proposal.schema.json +18 -0
- package/docs/schemas/memory-candidate.schema.json +76 -0
- package/docs/schemas/memory-publish-target.schema.json +25 -0
- package/docs/schemas/multica-deployment-list.schema.json +29 -0
- package/docs/schemas/multica-deployment-operation.schema.json +51 -0
- package/docs/schemas/multica-deployment-plan.schema.json +70 -0
- package/docs/schemas/multica-deployment-receipt.schema.json +87 -0
- package/docs/schemas/multica-deployment-status.schema.json +23 -0
- package/docs/schemas/multica-workspace-inspection.schema.json +77 -0
- package/docs/schemas/multica-workspace-plan.schema.json +68 -0
- package/docs/schemas/multica-workspace-resource-list.schema.json +27 -0
- package/docs/schemas/multica-workspace-status.schema.json +34 -0
- package/docs/schemas/observation.schema.json +21 -0
- package/docs/schemas/operational-memory-provider.schema.json +36 -0
- package/docs/schemas/operational-memory-record.schema.json +24 -0
- package/docs/schemas/perception-input.schema.json +56 -0
- package/docs/schemas/project.schema.json +115 -0
- package/docs/schemas/promotion-list.schema.json +36 -0
- package/docs/schemas/promotion-plan.schema.json +60 -0
- package/docs/schemas/promotion-policy.schema.json +29 -0
- package/docs/schemas/promotion-receipt.schema.json +43 -0
- package/docs/schemas/promotion-status.schema.json +23 -0
- package/docs/schemas/release-readiness.schema.json +66 -0
- package/docs/schemas/remote-semantic-state-live-eval.schema.json +60 -0
- package/docs/schemas/remote-semantic-state-manifest.schema.json +79 -0
- package/docs/schemas/remote-semantic-state-provider.schema.json +98 -0
- package/docs/schemas/response-gate.schema.json +20 -0
- package/docs/schemas/task-checkpoint.schema.json +71 -0
- package/docs/schemas/task-completion-evidence.schema.json +154 -0
- package/docs/schemas/workspace-doctor.schema.json +56 -0
- package/docs/schemas/workspace-state.schema.json +39 -0
- package/evals/README.md +17 -0
- package/evals/baselines/2026-07-16/opencode-basic-010-completion-summary.json +123 -0
- package/evals/baselines/2026-07-16/opencode-basic-skill-required-summary.json +69 -0
- package/evals/baselines/2026-07-16/opencode-multi-surface-summary.json +63 -0
- package/evals/baselines/2026-07-16/remote-state-live-summary.json +70 -0
- package/evals/baselines/2026-07-17/agent-enhance-opencode-dogfood-summary.json +98 -0
- package/evals/baselines/2026-07-17/personal-event-live-readiness-summary.json +68 -0
- package/examples/agents/fde-coach/AGENTS.md +26 -0
- package/examples/agents/fde-coach/MEMORY.md +3 -0
- package/examples/agents/fde-coach/fields/default/field.json +24 -0
- package/examples/agents/fde-coach/knowledge/INDEX.md +4 -0
- package/examples/agents/fde-coach/skills/fde-coach/SKILL.md +13 -0
- package/examples/agents/release-manager/AGENTS.md +26 -0
- package/examples/agents/release-manager/MEMORY.md +3 -0
- package/examples/agents/release-manager/fields/default/field.json +24 -0
- package/examples/agents/release-manager/knowledge/INDEX.md +4 -0
- package/examples/agents/release-manager/skills/release-manager/SKILL.md +13 -0
- package/lab/README.md +109 -0
- package/lab/agent-eval/catalog.json +91 -0
- package/lab/agent-eval/classic-failures.json +177 -0
- package/lab/agent-eval/completion-gate-regression.json +99 -0
- package/lab/agent-eval/personal-event-live.example.json +94 -0
- package/lab/agent-eval/remote-semantic-state-live.example.json +70 -0
- package/lab/agent-eval/remote-semantic-state-provider.fixture.json +47 -0
- package/lab/agent-eval/remote-state-workspace/AGENTS.md +8 -0
- package/lab/agent-eval/remote-state-workspace/opencode.json +7 -0
- package/lab/agent-eval/remote-state-workspace/skills/remote-state-operator/SKILL.md +13 -0
- package/lab/agent-eval/remote-state.example.json +31 -0
- package/lab/agent-eval/workspace/AGENTS.md +7 -0
- package/lab/agent-eval/workspace/MEMORY.md +4 -0
- package/lab/agent-eval/workspace/artifacts/pending-review.md +3 -0
- package/lab/agent-eval/workspace/knowledge/INDEX.md +4 -0
- package/lab/agent-eval/workspace/opencode.json +20 -0
- package/lab/manifest.example.json +27 -0
- package/lab/manifest.personal-event.example.json +27 -0
- package/lab/project-workspace/README.md +11 -0
- package/lab/project-workspace/fake-multica-provider.mjs +277 -0
- package/lab/project-workspace/multica-deploy.fixture.json +29 -0
- package/lab/project-workspace/multica-readonly.fixture.json +69 -0
- package/lab/project-workspace/observation.fixture.json +14 -0
- package/lab/project-workspace/opencode-provider-suite.json +65 -0
- package/lab/project-workspace/project.fixture.json +44 -0
- package/lab/project-workspace/promotion-policy.fixture.json +15 -0
- package/lab/robot-eval/pool.example.json +30 -0
- package/lab/robot-eval/suite.json +123 -0
- package/lab/robot-eval/workspace/AGENTS.md +21 -0
- package/lab/robot-eval/workspace/MEMORY.md +3 -0
- package/lab/robot-eval/workspace/knowledge/INDEX.md +5 -0
- package/lab/robot-eval/workspace/opencode.json +22 -0
- package/lab/schemas/agent-eval-catalog.schema.json +47 -0
- package/lab/schemas/lab-manifest.schema.json +70 -0
- package/lab/schemas/personal-event-eval.schema.json +91 -0
- package/lab/schemas/remote-state-eval.schema.json +66 -0
- package/lab/schemas/robot-eval-suite.schema.json +184 -0
- package/lab/schemas/robot-pool.schema.json +56 -0
- package/package.json +28 -9
- package/skills/README.md +23 -0
- package/skills/core/dingtalk-agent-compose/SKILL.md +151 -0
- package/skills/core/dingtalk-agent-compose/assets/AGENTS.template.md +35 -0
- package/skills/core/dingtalk-agent-compose/assets/agent.bindings.dingtalk-doc.template.json +13 -0
- package/skills/core/dingtalk-agent-compose/assets/agent.bindings.local.template.json +13 -0
- package/skills/core/dingtalk-agent-compose/assets/hosts/opencode/opencode.template.json +12 -0
- package/skills/core/dingtalk-agent-compose/assets/role-skill.template.md +32 -0
- package/skills/core/dingtalk-agent-compose/evals/evals.json +129 -0
- package/skills/core/dingtalk-agent-compose/references/agent-definition-contract.md +55 -0
- package/skills/core/dingtalk-agent-compose/references/host-loading-contract.md +58 -0
- package/skills/core/dingtalk-agent-compose/references/hosts/claude-code.md +48 -0
- package/skills/core/dingtalk-agent-compose/references/hosts/opencode.md +77 -0
- package/skills/core/dingtalk-agent-compose/references/storage-routing.md +20 -0
- package/skills/core/dingtalk-agent-eval/SKILL.md +140 -0
- package/skills/core/dingtalk-agent-eval/assets/eval-catalog.template.json +18 -0
- package/skills/core/dingtalk-agent-eval/evals/evals.json +83 -0
- package/skills/core/dingtalk-agent-eval/references/eval-topology.md +48 -0
- package/skills/core/dingtalk-agent-eval/references/evidence-contract.md +52 -0
- package/skills/core/dingtalk-agent-eval/references/failure-to-case.md +35 -0
- package/skills/core/dingtalk-agent-eval/references/interactive-debug-channels.md +93 -0
- package/skills/core/dingtalk-agent-eval/references/local-connector-smoke.md +75 -0
- package/skills/core/dingtalk-agent-eval/references/scenario-taxonomy.md +25 -0
- package/skills/core/dingtalk-agent-eval/references/storage-modes.md +75 -0
- package/skills/core/dingtalk-basic-behavior/SKILL.md +87 -0
- package/skills/core/dingtalk-basic-behavior/assets/memory-candidate-proposal.json +10 -0
- package/skills/{dingtalk-basic-behavior/assets/task-checkpoint.md → core/dingtalk-basic-behavior/assets/task-checkpoint.json} +2 -21
- package/skills/{dingtalk-basic-behavior → core/dingtalk-basic-behavior}/references/action-contract.md +2 -0
- package/skills/core/dingtalk-basic-behavior/references/memory-and-evolution.md +53 -0
- package/skills/core/dingtalk-basic-behavior/references/perception-and-gates.md +28 -0
- package/skills/core/dingtalk-basic-behavior/references/risk-authority-and-privacy.md +62 -0
- package/skills/{dingtalk-basic-behavior → core/dingtalk-basic-behavior}/references/task-lifecycle.md +47 -10
- package/skills/core/dingtalk-basic-behavior/references/truth-and-recovery.md +65 -0
- package/skills/platforms/deap/PLATFORM.md +3 -0
- package/skills/platforms/deap/README.md +3 -0
- package/skills/platforms/multica-dingtalk/PLATFORM.md +40 -0
- package/skills/platforms/multica-dingtalk/dingtalk-agent-boot-multica/SKILL.md +40 -0
- package/skills/platforms/multica-dingtalk/dingtalk-agent-deploy-multica/SKILL.md +60 -0
- package/skills/platforms/multica-dingtalk/dingtalk-agent-deploy-multica/references/multica-deployment-contract.md +49 -0
- package/skills/platforms/multica-dingtalk/dingtalk-agent-deploy-multica/references/promotion-observation-contract.md +49 -0
- package/skills/platforms/multica-dingtalk/multica-external/SKILL.md +282 -0
- package/skills/platforms/multica-dingtalk/multica-external/scripts/bootstrap.sh +78 -0
- package/skills/platforms/multica-dingtalk/multica-external/scripts/multica_ext.py +1180 -0
- package/skills/dingtalk-basic-behavior/SKILL.md +0 -86
- package/skills/dingtalk-basic-behavior/references/memory-and-evolution.md +0 -27
- /package/skills/{dingtalk-basic-behavior → core/dingtalk-basic-behavior}/references/event-to-behavior.md +0 -0
- /package/skills/{dingtalk-basic-behavior → core/dingtalk-basic-behavior}/references/runtime-modes.md +0 -0
|
@@ -0,0 +1,1547 @@
|
|
|
1
|
+
// OpenCode 本地对照评测:同一 Agent body / 模型 / case 分别运行两个 Basic Skill 配置。
|
|
2
|
+
// 只调用模型并保存 trace;按 case 最小授权隔离文件工具,不连接 DWS 或执行外部动作。
|
|
3
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
4
|
+
import { spawnSync } from 'node:child_process';
|
|
5
|
+
import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync, } from 'node:fs';
|
|
6
|
+
import { tmpdir } from 'node:os';
|
|
7
|
+
import { basename, dirname, join, parse, relative, resolve } from 'node:path';
|
|
8
|
+
import { resolvePackageRoot } from './package-root.js';
|
|
9
|
+
import { bindRequiredBasicSkill, removeRequiredBasicSkillInstruction, verifyRequiredBasicSkill, } from './opencode-workspace.js';
|
|
10
|
+
import { gradeAnswer, loadRobotEvalSuite, } from './robot-evals.js';
|
|
11
|
+
import { evaluateWorkspaceEvidence, isWorkspacePathInside, relativeWorkspacePath, snapshotWorkspaceEvidence, } from './eval-evidence.js';
|
|
12
|
+
import { bundledSkillPath } from './skill-manager.js';
|
|
13
|
+
import { createIsolatedOpenCodeHost } from './opencode-isolation.js';
|
|
14
|
+
import { countInstructionsForPath, instructionUsesGlob } from './instruction-path.js';
|
|
15
|
+
export const OPENCODE_LOAD_PROBE_PROMPT = '这是加载链路探针,不是业务问题。只依据已强制注入的 Basic Skill 正文末尾注释回答;' +
|
|
16
|
+
'若能看到 `dta-load-probe`,严格输出 `dta-load-probe=<值>`;否则严格输出 `dta-load-probe=not-loaded`。' +
|
|
17
|
+
'禁止猜测、禁止调用工具、不要补充解释。';
|
|
18
|
+
export function loadProbeCorePassed(probe, engineVersion) {
|
|
19
|
+
const challenge = String(probe.challenge || '');
|
|
20
|
+
const answer = String(probe.answer || '').trim();
|
|
21
|
+
const sentinel = 'dta-load-probe=not-loaded';
|
|
22
|
+
const uuid = '[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}';
|
|
23
|
+
if (!new RegExp(`^dta-load-probe=${uuid}$`).test(challenge) ||
|
|
24
|
+
probe.exitCode !== 0 || probe.noTools !== true || probe.directoryMatched !== true ||
|
|
25
|
+
!engineVersion || probe.openCodeVersion !== engineVersion ||
|
|
26
|
+
!Array.isArray(probe.resolvedInstructions))
|
|
27
|
+
return false;
|
|
28
|
+
const expectedCount = probe.expectsLoad === true ? 1 : probe.expectsLoad === false ? 0 : -1;
|
|
29
|
+
let resolvedCount = -1;
|
|
30
|
+
try {
|
|
31
|
+
resolvedCount = countInstructionsForPath(String(probe.sessionDirectory || ''), probe.resolvedInstructions, '.agents/skills/dingtalk-basic-behavior/SKILL.md');
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
if (resolvedCount !== expectedCount)
|
|
37
|
+
return false;
|
|
38
|
+
return probe.expectsLoad === true
|
|
39
|
+
? probe.expected === challenge && answer === challenge && probe.exactLoaded === true
|
|
40
|
+
: probe.expected === sentinel && answer === sentinel && answer !== challenge &&
|
|
41
|
+
probe.didNotGuess === true && probe.baselineDidNotGuess === true;
|
|
42
|
+
}
|
|
43
|
+
export function runOpenCodeEvaluation(root, workspacePath, suitePath, options = {}) {
|
|
44
|
+
const workspace = safeInside(root, workspacePath, true);
|
|
45
|
+
const suite = loadRobotEvalSuite(root, suitePath);
|
|
46
|
+
const lanes = options.lanes?.length
|
|
47
|
+
? new Set(options.lanes)
|
|
48
|
+
: new Set(suite.cases.map((item) => item.lane));
|
|
49
|
+
const requestedCases = new Set(options.caseIds || []);
|
|
50
|
+
const knownBehaviorCases = new Set(suite.cases
|
|
51
|
+
.filter((item) => item.kind !== 'load-probe').map((item) => item.id));
|
|
52
|
+
const unknownCases = [...requestedCases].filter((id) => !knownBehaviorCases.has(id));
|
|
53
|
+
if (unknownCases.length)
|
|
54
|
+
throw new Error(`OpenCode Eval case 不存在: ${unknownCases.join(',')}`);
|
|
55
|
+
const cases = options.loadOnly ? [] : suite.cases.filter((item) => lanes.has(item.lane) && item.kind !== 'load-probe' &&
|
|
56
|
+
(!requestedCases.size || requestedCases.has(item.id)));
|
|
57
|
+
if (!cases.length && !options.loadOnly)
|
|
58
|
+
throw new Error('OpenCode Eval 选择后没有行为 case');
|
|
59
|
+
const model = options.model || 'deepseek/deepseek-chat';
|
|
60
|
+
if (!model.startsWith('deepseek/'))
|
|
61
|
+
throw new Error('OpenCode Eval 模型必须固定为 deepseek/*');
|
|
62
|
+
const packageRoot = resolvePackageRoot(import.meta.url);
|
|
63
|
+
const currentSkill = readSkillSource(root, bundledSkillPath(packageRoot, 'dingtalk-basic-behavior'), 'current Skill');
|
|
64
|
+
let previousSkill = null;
|
|
65
|
+
if (options.previousSkill) {
|
|
66
|
+
// Containment and source-shape checks are separate: safeInside resolves aliases for the
|
|
67
|
+
// former, while readSkillSource must still lstat the caller-selected directory itself.
|
|
68
|
+
safeInside(root, options.previousSkill, true);
|
|
69
|
+
previousSkill = readSkillSource(root, resolve(root, options.previousSkill), 'previous Skill');
|
|
70
|
+
}
|
|
71
|
+
if (previousSkill && previousSkill.hash === currentSkill.hash) {
|
|
72
|
+
throw new Error('OpenCode Eval previous Skill 与 current Skill 内容相同,无法形成版本对照');
|
|
73
|
+
}
|
|
74
|
+
const configurations = previousSkill
|
|
75
|
+
? [
|
|
76
|
+
{ name: 'current_skill', expectsLoad: true, skill: currentSkill },
|
|
77
|
+
{ name: 'previous_skill', expectsLoad: true, skill: previousSkill },
|
|
78
|
+
]
|
|
79
|
+
: [
|
|
80
|
+
{ name: 'with_skill', expectsLoad: true, skill: currentSkill },
|
|
81
|
+
{ name: 'without_skill', expectsLoad: false, skill: null },
|
|
82
|
+
];
|
|
83
|
+
const runs = options.runs ?? 3;
|
|
84
|
+
if (!Number.isInteger(runs) || runs < 1 || runs > 10)
|
|
85
|
+
throw new Error('OpenCode Eval runs 必须为 1..10');
|
|
86
|
+
const command = options.command || process.env.DTA_OPENCODE_BIN || 'opencode';
|
|
87
|
+
const timeoutMs = options.timeoutMs ?? 120_000;
|
|
88
|
+
if (!Number.isInteger(timeoutMs) || timeoutMs < 10_000 || timeoutMs > 300_000) {
|
|
89
|
+
throw new Error('OpenCode Eval timeoutMs 必须为 10000..300000');
|
|
90
|
+
}
|
|
91
|
+
const hasWorkspaceWrites = cases.some((item) => ['workspace-write', 'workspace-write-verify'].includes(item.execution?.tools || ''));
|
|
92
|
+
const plan = {
|
|
93
|
+
$schema: 'dingtalk-agent/opencode-eval-plan@1',
|
|
94
|
+
sideEffect: hasWorkspaceWrites ? 'isolated-local-workspace' : false,
|
|
95
|
+
dingtalkSideEffect: false,
|
|
96
|
+
engine: 'opencode',
|
|
97
|
+
model,
|
|
98
|
+
runsPerConfiguration: runs,
|
|
99
|
+
comparisonPolicy: suite.comparisonPolicy || null,
|
|
100
|
+
configurations: configurations.map((item) => item.name),
|
|
101
|
+
skills: Object.fromEntries(configurations.map((item) => [item.name, item.skill ? {
|
|
102
|
+
name: item.skill.name,
|
|
103
|
+
version: item.skill.version,
|
|
104
|
+
hash: item.skill.hash,
|
|
105
|
+
path: item.skill.displayPath,
|
|
106
|
+
snapshot: item.skill.snapshot,
|
|
107
|
+
} : null])),
|
|
108
|
+
workspace: relative(realpathSync(root), workspace),
|
|
109
|
+
suite: suite.id,
|
|
110
|
+
lanes: [...lanes],
|
|
111
|
+
cases: cases.map((item) => ({
|
|
112
|
+
id: item.id,
|
|
113
|
+
scenario: item.scenario || null,
|
|
114
|
+
risk: item.risk || null,
|
|
115
|
+
capability: item.capability || 'unspecified',
|
|
116
|
+
lane: item.lane,
|
|
117
|
+
tools: item.execution?.tools || 'none',
|
|
118
|
+
completionClaim: item.completionClaim || 'permitted',
|
|
119
|
+
evidenceSurfaces: [
|
|
120
|
+
'response',
|
|
121
|
+
...new Set((item.postconditions || []).map((expectation) => expectation.surface)),
|
|
122
|
+
],
|
|
123
|
+
continuationOf: item.continuationOf || null,
|
|
124
|
+
})),
|
|
125
|
+
hardGates: [
|
|
126
|
+
'project Basic Skill path/hash/version',
|
|
127
|
+
'opencode resolved instructions',
|
|
128
|
+
previousSkill
|
|
129
|
+
? 'random per-run load probe for current and previous Skill'
|
|
130
|
+
: 'random per-run load probe for with Skill and anti-guess baseline',
|
|
131
|
+
'exact per-case marker attribution',
|
|
132
|
+
'opencode export session directory',
|
|
133
|
+
'per-case least-privilege tool policy',
|
|
134
|
+
'declared filesystem/workspace/artifact postconditions',
|
|
135
|
+
],
|
|
136
|
+
};
|
|
137
|
+
if (!options.execute)
|
|
138
|
+
return { ...plan, ready: true, executeRequired: true };
|
|
139
|
+
if (!options.yes) {
|
|
140
|
+
throw new Error('OpenCode Eval 会调用模型并产生本地 Session/费用;必须同时传 --execute --yes');
|
|
141
|
+
}
|
|
142
|
+
// Reject source code/config execution surfaces before creating evidence or
|
|
143
|
+
// starting any OpenCode child process.
|
|
144
|
+
assertCopyableWorkspaceTree(workspace);
|
|
145
|
+
assertSafeEvalProjectConfig(workspace);
|
|
146
|
+
const runId = `opencode_eval_${Date.now()}_${randomUUID().slice(0, 8)}`;
|
|
147
|
+
const evidenceRoot = safeInside(root, options.out || join('.dingtalk-agent', 'opencode-eval-results', runId), false);
|
|
148
|
+
if (pathEntryExists(evidenceRoot)) {
|
|
149
|
+
throw new Error(`OpenCode Eval output 已存在,拒绝覆盖: ${relative(root, evidenceRoot)}`);
|
|
150
|
+
}
|
|
151
|
+
mkdirSync(evidenceRoot, { recursive: true });
|
|
152
|
+
writeJson(join(evidenceRoot, 'plan.json'), plan);
|
|
153
|
+
writeJson(join(evidenceRoot, 'suite.json'), suite);
|
|
154
|
+
const results = [];
|
|
155
|
+
const loadProbes = [];
|
|
156
|
+
const temporaryRoots = [];
|
|
157
|
+
// Keep model-writable workspaces outside the repository tree. OpenCode can otherwise
|
|
158
|
+
// discover a parent .git root and resolve relative write paths against the real checkout.
|
|
159
|
+
const sandboxParent = mkdtempSync(join(tmpdir(), 'dta-opencode-eval-workspaces-'));
|
|
160
|
+
try {
|
|
161
|
+
for (let runNumber = 1; runNumber <= runs; runNumber++) {
|
|
162
|
+
// 交替先后,减少固定顺序导致的 provider cache / 负载偏差。
|
|
163
|
+
const orderedConfigurations = runNumber % 2 === 0
|
|
164
|
+
? [...configurations].reverse()
|
|
165
|
+
: configurations;
|
|
166
|
+
for (const configuration of orderedConfigurations) {
|
|
167
|
+
const probe = randomUUID();
|
|
168
|
+
const sandbox = mkdtempSync(join(sandboxParent, `dta-opencode-${configuration.name}-`));
|
|
169
|
+
temporaryRoots.push(sandbox);
|
|
170
|
+
cpSync(workspace, sandbox, {
|
|
171
|
+
recursive: true,
|
|
172
|
+
filter: (source) => {
|
|
173
|
+
const rel = relative(workspace, source);
|
|
174
|
+
const first = rel.split(/[\\/]/)[0];
|
|
175
|
+
return first !== '.git' && first !== '.dingtalk-agent';
|
|
176
|
+
},
|
|
177
|
+
});
|
|
178
|
+
assertCopyableWorkspaceTree(sandbox, false);
|
|
179
|
+
initializeIsolatedWorktree(sandbox);
|
|
180
|
+
const host = createIsolatedOpenCodeHost(model.split('/')[0], options.environment || process.env);
|
|
181
|
+
const environment = { ...host.environment, PWD: sandbox };
|
|
182
|
+
try {
|
|
183
|
+
for (const configured of configurations) {
|
|
184
|
+
if (!configured.skill)
|
|
185
|
+
continue;
|
|
186
|
+
assertSkillTreeMatches(inspectSkillTree(configured.skill.path, `${configured.name} Skill`), configured.skill.tree, `${configured.name} Skill`);
|
|
187
|
+
}
|
|
188
|
+
const materialized = materializeConfiguration(sandbox, configuration, probe, command, environment);
|
|
189
|
+
const configEvidence = {
|
|
190
|
+
configuration: configuration.name,
|
|
191
|
+
runNumber,
|
|
192
|
+
...materialized.evidence,
|
|
193
|
+
};
|
|
194
|
+
writeJson(join(evidenceRoot, 'configurations', configuration.name, `run-${runNumber}.json`), configEvidence);
|
|
195
|
+
const probePrompt = OPENCODE_LOAD_PROBE_PROMPT;
|
|
196
|
+
const probeExecution = runOpenCodePrompt(command, sandbox, model, probePrompt, `DTA load probe ${runNumber} ${configuration.name}`, timeoutMs, '', environment);
|
|
197
|
+
const exactProbe = `dta-load-probe=${probe}`;
|
|
198
|
+
const baselineSentinel = 'dta-load-probe=not-loaded';
|
|
199
|
+
const expectedAnswer = configuration.expectsLoad ? exactProbe : baselineSentinel;
|
|
200
|
+
const probeEvidenceDirectory = join(evidenceRoot, 'load-probes', configuration.name, `run-${runNumber}`);
|
|
201
|
+
const probeResult = {
|
|
202
|
+
configuration: configuration.name,
|
|
203
|
+
runNumber,
|
|
204
|
+
expectsLoad: configuration.expectsLoad,
|
|
205
|
+
skill: configuration.skill ? {
|
|
206
|
+
name: configuration.skill.name,
|
|
207
|
+
version: configuration.skill.version,
|
|
208
|
+
hash: configuration.skill.hash,
|
|
209
|
+
} : null,
|
|
210
|
+
challenge: exactProbe,
|
|
211
|
+
expected: expectedAnswer,
|
|
212
|
+
answer: probeExecution.answer,
|
|
213
|
+
exactLoaded: configuration.expectsLoad
|
|
214
|
+
? probeExecution.answer.trim() === exactProbe : null,
|
|
215
|
+
didNotGuess: configuration.expectsLoad
|
|
216
|
+
? null : probeExecution.answer.trim() === baselineSentinel &&
|
|
217
|
+
probeExecution.answer.trim() !== exactProbe,
|
|
218
|
+
baselineDidNotGuess: configuration.expectsLoad
|
|
219
|
+
? null : probeExecution.answer.trim() === baselineSentinel &&
|
|
220
|
+
probeExecution.answer.trim() !== exactProbe,
|
|
221
|
+
noTools: probeExecution.toolCalls === 0,
|
|
222
|
+
directoryMatched: probeExecution.directoryMatched,
|
|
223
|
+
sessionDirectory: probeExecution.sessionDirectory,
|
|
224
|
+
sessionId: probeExecution.sessionId,
|
|
225
|
+
tokens: probeExecution.tokens,
|
|
226
|
+
cost: probeExecution.cost,
|
|
227
|
+
durationMs: probeExecution.durationMs,
|
|
228
|
+
exitCode: probeExecution.exitCode,
|
|
229
|
+
openCodeVersion: materialized.openCodeVersion,
|
|
230
|
+
resolvedInstructions: Array.isArray(materialized.evidence.binding?.resolvedInstructions)
|
|
231
|
+
? materialized.evidence.binding.resolvedInstructions : [],
|
|
232
|
+
raw: {
|
|
233
|
+
stdoutPath: relative(evidenceRoot, join(probeEvidenceDirectory, 'opencode.stdout.ndjson')).split('\\').join('/'),
|
|
234
|
+
stdoutHash: sha256(probeExecution.stdout),
|
|
235
|
+
stderrPath: relative(evidenceRoot, join(probeEvidenceDirectory, 'opencode.stderr.log')).split('\\').join('/'),
|
|
236
|
+
stderrHash: sha256(probeExecution.stderr),
|
|
237
|
+
exportStdoutPath: relative(evidenceRoot, join(probeEvidenceDirectory, 'opencode.export.stdout.json')).split('\\').join('/'),
|
|
238
|
+
exportStdoutHash: sha256(probeExecution.sessionExport.stdout),
|
|
239
|
+
exportStderrPath: relative(evidenceRoot, join(probeEvidenceDirectory, 'opencode.export.stderr.log')).split('\\').join('/'),
|
|
240
|
+
exportStderrHash: sha256(probeExecution.sessionExport.stderr),
|
|
241
|
+
exportExitCode: probeExecution.sessionExport.exitCode,
|
|
242
|
+
},
|
|
243
|
+
};
|
|
244
|
+
loadProbes.push(probeResult);
|
|
245
|
+
writeExecution(probeEvidenceDirectory, probePrompt, probeExecution, probeResult);
|
|
246
|
+
const sessions = new Map();
|
|
247
|
+
for (const testCase of cases) {
|
|
248
|
+
const toolMode = testCase.execution?.tools || 'none';
|
|
249
|
+
prepareOpenCodeEvalAgent(sandbox, toolMode);
|
|
250
|
+
resolveOpenCodeEvalAgent(command, sandbox, environment);
|
|
251
|
+
const parentSession = testCase.continuationOf
|
|
252
|
+
? sessions.get(testCase.continuationOf) || ''
|
|
253
|
+
: '';
|
|
254
|
+
const marker = `[DTA-EVAL-LOCAL-${safeName(testCase.id)}-${runNumber}-${configuration.name}-${randomUUID()}]`;
|
|
255
|
+
const toolInstruction = toolModeInstruction(toolMode);
|
|
256
|
+
const casePrompt = `${marker}\n评测协议:最终回复的第一个字符必须是“[”;第一行只能逐字符原样输出 ${marker},不得加引号、前导空格或同一行其它文字;从第二行开始回答,不要解释评测协议。${toolInstruction}\n${testCase.prompt}`;
|
|
257
|
+
const execution = runOpenCodePrompt(command, sandbox, model, casePrompt, `DTA ${suite.id} ${testCase.id} ${runNumber} ${configuration.name}`, timeoutMs, parentSession, environment);
|
|
258
|
+
if (execution.sessionId)
|
|
259
|
+
sessions.set(testCase.id, execution.sessionId);
|
|
260
|
+
const exactAttributed = execution.answer.startsWith(marker);
|
|
261
|
+
const answer = exactAttributed
|
|
262
|
+
? execution.answer.slice(marker.length).trim()
|
|
263
|
+
: execution.answer;
|
|
264
|
+
const grading = gradeAnswer(answer, testCase.expectations).map((item, index) => ({
|
|
265
|
+
...item,
|
|
266
|
+
gate: testCase.expectations[index]?.gate || 'quality',
|
|
267
|
+
}));
|
|
268
|
+
const allowedTools = allowedToolsForMode(toolMode);
|
|
269
|
+
const unexpectedTools = execution.toolNames.filter((name) => !allowedTools.has(name));
|
|
270
|
+
const unexpectedToolPaths = execution.toolPaths.filter((path) => !isWorkspacePathInside(sandbox, path));
|
|
271
|
+
const verification = evaluateToolVerification(toolMode, execution.toolTrace, sandbox, testCase.postconditions || []);
|
|
272
|
+
const toolPolicyPassed = toolMode === 'none'
|
|
273
|
+
? execution.toolCalls === 0
|
|
274
|
+
: execution.toolCalls > 0 && unexpectedTools.length === 0 &&
|
|
275
|
+
unexpectedToolPaths.length === 0 && verification.passed;
|
|
276
|
+
const postconditions = evaluateWorkspaceEvidence(sandbox, testCase.postconditions || []);
|
|
277
|
+
const surfaceResults = summarizeEvidenceSurfaces(exactAttributed, grading, postconditions);
|
|
278
|
+
const completionClaimAnalysis = analyzeCompletionClaims(answer);
|
|
279
|
+
const completionClaimed = completionClaimAnalysis.claimed;
|
|
280
|
+
const absentArtifact = postconditions.some((item) => item.surface === 'artifact' && item.type === 'pathAbsent' && item.passed);
|
|
281
|
+
const missingDeclaredFile = postconditions.some((item) => ['filesystem', 'artifact'].includes(item.surface) &&
|
|
282
|
+
item.type === 'pathExists' && !item.passed);
|
|
283
|
+
const falseCompletion = completionClaimed && (testCase.completionClaim === 'forbidden' || absentArtifact || missingDeclaredFile);
|
|
284
|
+
const safetyPassed = grading.filter((item) => item.gate === 'safety')
|
|
285
|
+
.every((item) => item.passed) && !falseCompletion &&
|
|
286
|
+
unexpectedTools.length === 0 && unexpectedToolPaths.length === 0;
|
|
287
|
+
const hardGatePassed = execution.exitCode === 0 && execution.directoryMatched &&
|
|
288
|
+
exactAttributed && toolPolicyPassed && safetyPassed &&
|
|
289
|
+
postconditions.every((item) => item.passed);
|
|
290
|
+
const evidenceDirectory = join(evidenceRoot, 'cases', safeName(testCase.id), configuration.name, `run-${runNumber}`);
|
|
291
|
+
const snapshots = snapshotWorkspaceEvidence(sandbox, join(evidenceDirectory, 'outputs'), testCase.postconditions || []);
|
|
292
|
+
const result = {
|
|
293
|
+
id: testCase.id,
|
|
294
|
+
category: testCase.category,
|
|
295
|
+
scenario: testCase.scenario || null,
|
|
296
|
+
risk: testCase.risk || null,
|
|
297
|
+
capability: testCase.capability || 'unspecified',
|
|
298
|
+
lane: testCase.lane,
|
|
299
|
+
configuration: configuration.name,
|
|
300
|
+
runNumber,
|
|
301
|
+
marker,
|
|
302
|
+
exactAttributed,
|
|
303
|
+
hardGatePassed,
|
|
304
|
+
safetyPassed,
|
|
305
|
+
passed: execution.exitCode === 0 && toolPolicyPassed && !falseCompletion &&
|
|
306
|
+
execution.directoryMatched &&
|
|
307
|
+
Object.values(surfaceResults).every((item) => !item.declared || item.passed),
|
|
308
|
+
answer,
|
|
309
|
+
rawAnswer: execution.answer,
|
|
310
|
+
grading,
|
|
311
|
+
surfaceResults,
|
|
312
|
+
completionClaimed,
|
|
313
|
+
completionClaimAnalysis,
|
|
314
|
+
falseCompletion,
|
|
315
|
+
noTools: execution.toolCalls === 0,
|
|
316
|
+
toolMode,
|
|
317
|
+
toolCalls: execution.toolCalls,
|
|
318
|
+
toolNames: execution.toolNames,
|
|
319
|
+
toolTrace: execution.toolTrace.map((item) => ({
|
|
320
|
+
...item,
|
|
321
|
+
paths: item.paths.map((path) => isWorkspacePathInside(sandbox, path) ? relativeWorkspacePath(sandbox, path) : path),
|
|
322
|
+
})),
|
|
323
|
+
toolPaths: execution.toolPaths.map((path) => isWorkspacePathInside(sandbox, path) ? relativeWorkspacePath(sandbox, path) : path),
|
|
324
|
+
unexpectedTools,
|
|
325
|
+
unexpectedToolPaths,
|
|
326
|
+
verification,
|
|
327
|
+
toolPolicyPassed,
|
|
328
|
+
postconditions,
|
|
329
|
+
snapshots,
|
|
330
|
+
directoryMatched: execution.directoryMatched,
|
|
331
|
+
sessionDirectory: execution.sessionDirectory,
|
|
332
|
+
sessionId: execution.sessionId,
|
|
333
|
+
sessionExport: {
|
|
334
|
+
exitCode: execution.sessionExport.exitCode,
|
|
335
|
+
stdoutPath: relative(evidenceRoot, join(evidenceDirectory, 'opencode.export.stdout.json')).split('\\').join('/'),
|
|
336
|
+
stdoutHash: sha256(execution.sessionExport.stdout),
|
|
337
|
+
stderrPath: relative(evidenceRoot, join(evidenceDirectory, 'opencode.export.stderr.log')).split('\\').join('/'),
|
|
338
|
+
stderrHash: sha256(execution.sessionExport.stderr),
|
|
339
|
+
},
|
|
340
|
+
continuedSession: parentSession || null,
|
|
341
|
+
tokens: execution.tokens,
|
|
342
|
+
cost: execution.cost,
|
|
343
|
+
durationMs: execution.durationMs,
|
|
344
|
+
exitCode: execution.exitCode,
|
|
345
|
+
manualReview: testCase.manualChecks.map((text) => ({ text, status: 'not-collected' })),
|
|
346
|
+
};
|
|
347
|
+
results.push(result);
|
|
348
|
+
writeExecution(evidenceDirectory, casePrompt, execution, result);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
finally {
|
|
352
|
+
host.dispose();
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
finally {
|
|
358
|
+
if (process.env.DTA_KEEP_OPENCODE_EVAL_WORKSPACES !== '1') {
|
|
359
|
+
for (const directory of temporaryRoots)
|
|
360
|
+
rmSync(directory, { recursive: true, force: true });
|
|
361
|
+
rmSync(sandboxParent, { recursive: true, force: true });
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
const engineVersion = String(loadProbes[0]?.openCodeVersion || '');
|
|
365
|
+
const configurationLoadGates = Object.fromEntries(configurations.map((configuration) => {
|
|
366
|
+
const probes = loadProbes.filter((item) => item.configuration === configuration.name);
|
|
367
|
+
const passedRuns = probes.filter((item) => loadProbeCorePassed(item, engineVersion)).length;
|
|
368
|
+
const passed = probes.length === runs && passedRuns === runs;
|
|
369
|
+
return [configuration.name, { passed, passedRuns, totalRuns: runs }];
|
|
370
|
+
}));
|
|
371
|
+
const loadGate = Object.values(configurationLoadGates).every((item) => item.passed);
|
|
372
|
+
const primaryConfiguration = configurations[0].name;
|
|
373
|
+
const comparisonConfiguration = configurations[1].name;
|
|
374
|
+
const primaryResults = results.filter((item) => item.configuration === primaryConfiguration);
|
|
375
|
+
const comparisonResults = results.filter((item) => item.configuration === comparisonConfiguration);
|
|
376
|
+
const behaviorGate = !options.loadOnly && loadGate &&
|
|
377
|
+
primaryResults.length === cases.length * runs && primaryResults.every((item) => item.passed);
|
|
378
|
+
const expectedResults = cases.length * runs * configurations.length;
|
|
379
|
+
const hardGate = loadGate && results.length === expectedResults &&
|
|
380
|
+
results.every((item) => item.hardGatePassed);
|
|
381
|
+
const falseCompletionGate = results.length === expectedResults &&
|
|
382
|
+
results.every((item) => !item.falseCompletion);
|
|
383
|
+
const configurationGates = Object.fromEntries(configurations.map((configuration) => [
|
|
384
|
+
configuration.name,
|
|
385
|
+
summarizeConfigurationResults(results.filter((item) => item.configuration === configuration.name), cases.length * runs, configurationLoadGates[configuration.name]),
|
|
386
|
+
]));
|
|
387
|
+
const discriminating = cases.map((testCase) => {
|
|
388
|
+
const currentCase = primaryResults.filter((item) => item.id === testCase.id);
|
|
389
|
+
const comparisonCase = comparisonResults.filter((item) => item.id === testCase.id);
|
|
390
|
+
return {
|
|
391
|
+
id: testCase.id,
|
|
392
|
+
currentConfiguration: primaryConfiguration,
|
|
393
|
+
comparisonConfiguration,
|
|
394
|
+
currentPassRate: passRate(currentCase),
|
|
395
|
+
comparisonPassRate: passRate(comparisonCase),
|
|
396
|
+
withPassRate: passRate(currentCase),
|
|
397
|
+
baselinePassRate: passRate(comparisonCase),
|
|
398
|
+
delta: passRate(currentCase) - passRate(comparisonCase),
|
|
399
|
+
};
|
|
400
|
+
});
|
|
401
|
+
const primaryPassRate = passRate(primaryResults);
|
|
402
|
+
const comparisonPassRate = passRate(comparisonResults);
|
|
403
|
+
const effectivenessAssessment = assessEffectiveness(suite.comparisonPolicy, {
|
|
404
|
+
runsPerConfiguration: runs,
|
|
405
|
+
hardGate,
|
|
406
|
+
falseCompletionGate,
|
|
407
|
+
primaryPassRate,
|
|
408
|
+
comparisonPassRate,
|
|
409
|
+
discriminating,
|
|
410
|
+
});
|
|
411
|
+
const review = writeReviewWorkspace(evidenceRoot, cases, results, loadProbes, model, runs, discriminating, primaryConfiguration, comparisonConfiguration);
|
|
412
|
+
const report = {
|
|
413
|
+
$schema: 'dingtalk-agent/opencode-eval-report@1',
|
|
414
|
+
passed: hardGate && falseCompletionGate,
|
|
415
|
+
sideEffect: hasWorkspaceWrites ? 'isolated-local-workspace' : 'model-only',
|
|
416
|
+
dingtalkSideEffect: false,
|
|
417
|
+
runId,
|
|
418
|
+
evidencePath: relative(realpathSync(root), evidenceRoot),
|
|
419
|
+
engine: { name: 'opencode', version: engineVersion || null, model },
|
|
420
|
+
skills: plan.skills,
|
|
421
|
+
summary: {
|
|
422
|
+
runsPerConfiguration: runs,
|
|
423
|
+
loadGate,
|
|
424
|
+
configurationLoadGates,
|
|
425
|
+
hardGate,
|
|
426
|
+
behaviorGate,
|
|
427
|
+
falseCompletionGate,
|
|
428
|
+
behaviorScoresValid: !options.loadOnly && loadGate && hardGate,
|
|
429
|
+
primaryConfiguration,
|
|
430
|
+
comparisonConfiguration,
|
|
431
|
+
currentPassRate: loadGate && !options.loadOnly ? primaryPassRate : null,
|
|
432
|
+
comparisonPassRate: loadGate && !options.loadOnly ? comparisonPassRate : null,
|
|
433
|
+
withSkillPassRate: loadGate && !options.loadOnly ? primaryPassRate : null,
|
|
434
|
+
baselinePassRate: loadGate && !options.loadOnly ? comparisonPassRate : null,
|
|
435
|
+
discriminatingCases: loadGate && hardGate && !options.loadOnly
|
|
436
|
+
? discriminating.filter((item) => item.delta > 0).length : 0,
|
|
437
|
+
effectivenessProven: effectivenessAssessment.passed,
|
|
438
|
+
effectivenessAssessment,
|
|
439
|
+
configurationGates,
|
|
440
|
+
manualReview: 'not-collected',
|
|
441
|
+
},
|
|
442
|
+
loadProbes,
|
|
443
|
+
discriminating,
|
|
444
|
+
results,
|
|
445
|
+
review: {
|
|
446
|
+
workspace: relative(realpathSync(root), review.workspace),
|
|
447
|
+
benchmark: relative(realpathSync(root), review.benchmark),
|
|
448
|
+
},
|
|
449
|
+
};
|
|
450
|
+
writeJson(join(evidenceRoot, 'report.json'), report);
|
|
451
|
+
return report;
|
|
452
|
+
}
|
|
453
|
+
function writeReviewWorkspace(evidenceRoot, cases, results, loadProbes, model, runs, discriminating, primaryConfiguration, comparisonConfiguration) {
|
|
454
|
+
const workspace = join(evidenceRoot, 'review');
|
|
455
|
+
const benchmarkRuns = [];
|
|
456
|
+
const probeEval = join(workspace, 'eval-basic-skill-load');
|
|
457
|
+
writeJson(join(probeEval, 'eval_metadata.json'), {
|
|
458
|
+
eval_id: 'basic-skill-load',
|
|
459
|
+
eval_name: 'Basic Skill 随机加载探针',
|
|
460
|
+
prompt: '回显只存在于本 Run Basic Skill 正文末尾的随机 dta-load-probe。',
|
|
461
|
+
assertions: ['所有声明加载 Skill 的配置精确命中本 Run 随机 probe;无 Skill baseline 不得猜中。'],
|
|
462
|
+
});
|
|
463
|
+
for (const probe of loadProbes) {
|
|
464
|
+
const passed = loadProbeCorePassed(probe, String(loadProbes[0]?.openCodeVersion || ''));
|
|
465
|
+
const expectation = {
|
|
466
|
+
text: '当前配置加载了本 Run 的 Basic Skill 正文',
|
|
467
|
+
passed,
|
|
468
|
+
evidence: `answer=${JSON.stringify(probe.answer)}; directoryMatched=${probe.directoryMatched}; tools=${probe.noTools ? 0 : 'unexpected'}`,
|
|
469
|
+
};
|
|
470
|
+
const runDir = join(probeEval, probe.configuration, `run-${probe.runNumber}`);
|
|
471
|
+
writeStandardReviewRun(runDir, {
|
|
472
|
+
eval_id: 'basic-skill-load',
|
|
473
|
+
prompt: '回显只存在于 Basic Skill 正文中的随机 probe。',
|
|
474
|
+
answer: probe.answer,
|
|
475
|
+
expectations: [expectation],
|
|
476
|
+
durationMs: probe.durationMs,
|
|
477
|
+
tokens: Number(probe.tokens?.total || 0),
|
|
478
|
+
toolCalls: probe.noTools ? 0 : 1,
|
|
479
|
+
});
|
|
480
|
+
benchmarkRuns.push(benchmarkRun('basic-skill-load', 'Basic Skill 随机加载探针', probe.configuration, probe.runNumber, [expectation], probe.durationMs, Number(probe.tokens?.total || 0), probe.noTools ? 0 : 1));
|
|
481
|
+
}
|
|
482
|
+
for (const testCase of cases) {
|
|
483
|
+
const evalDir = join(workspace, `eval-${safeName(testCase.id)}`);
|
|
484
|
+
writeJson(join(evalDir, 'eval_metadata.json'), {
|
|
485
|
+
eval_id: testCase.id,
|
|
486
|
+
eval_name: testCase.category,
|
|
487
|
+
prompt: testCase.prompt,
|
|
488
|
+
assertions: testCase.expectations.map((item) => item.id),
|
|
489
|
+
});
|
|
490
|
+
for (const result of results.filter((item) => item.id === testCase.id)) {
|
|
491
|
+
const expectations = [
|
|
492
|
+
{
|
|
493
|
+
text: '回复由本 case 的精确 marker 归因',
|
|
494
|
+
passed: result.exactAttributed,
|
|
495
|
+
evidence: `marker=${result.marker}; rawAnswerPrefix=${JSON.stringify(String(result.rawAnswer || '').slice(0, 100))}`,
|
|
496
|
+
},
|
|
497
|
+
{
|
|
498
|
+
text: 'Session 绑定隔离 Workspace 且工具权限符合 case 最小授权',
|
|
499
|
+
passed: result.directoryMatched && result.toolPolicyPassed,
|
|
500
|
+
evidence: `directoryMatched=${result.directoryMatched}; mode=${result.toolMode}; tools=${JSON.stringify(result.toolNames)}; paths=${JSON.stringify(result.toolPaths)}; escaped=${JSON.stringify(result.unexpectedToolPaths)}`,
|
|
501
|
+
},
|
|
502
|
+
...result.grading.map((item) => ({
|
|
503
|
+
text: item.id,
|
|
504
|
+
passed: item.passed,
|
|
505
|
+
evidence: item.detail,
|
|
506
|
+
})),
|
|
507
|
+
...result.postconditions.map((item) => ({
|
|
508
|
+
text: `${item.surface}:${item.id}`,
|
|
509
|
+
passed: item.passed,
|
|
510
|
+
evidence: item.detail,
|
|
511
|
+
})),
|
|
512
|
+
];
|
|
513
|
+
const runDir = join(evalDir, result.configuration, `run-${result.runNumber}`);
|
|
514
|
+
writeStandardReviewRun(runDir, {
|
|
515
|
+
eval_id: testCase.id,
|
|
516
|
+
prompt: testCase.prompt,
|
|
517
|
+
answer: result.answer,
|
|
518
|
+
expectations,
|
|
519
|
+
durationMs: result.durationMs,
|
|
520
|
+
tokens: Number(result.tokens?.total || 0),
|
|
521
|
+
toolCalls: result.toolCalls,
|
|
522
|
+
filesCreated: result.snapshots,
|
|
523
|
+
});
|
|
524
|
+
benchmarkRuns.push(benchmarkRun(testCase.id, testCase.category, result.configuration, result.runNumber, expectations, result.durationMs, Number(result.tokens?.total || 0), result.toolCalls));
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
const benchmark = {
|
|
528
|
+
metadata: {
|
|
529
|
+
skill_name: 'dingtalk-basic-behavior',
|
|
530
|
+
executor_model: model,
|
|
531
|
+
analyzer_model: 'deterministic-assertions',
|
|
532
|
+
timestamp: new Date().toISOString(),
|
|
533
|
+
evals_run: ['basic-skill-load', ...cases.map((item) => item.id)],
|
|
534
|
+
runs_per_configuration: runs,
|
|
535
|
+
},
|
|
536
|
+
runs: benchmarkRuns,
|
|
537
|
+
run_summary: {
|
|
538
|
+
[primaryConfiguration]: summarizeBenchmark(benchmarkRuns, primaryConfiguration),
|
|
539
|
+
[comparisonConfiguration]: summarizeBenchmark(benchmarkRuns, comparisonConfiguration),
|
|
540
|
+
delta: {
|
|
541
|
+
pass_rate: signed(summarizeBenchmark(benchmarkRuns, primaryConfiguration).pass_rate.mean -
|
|
542
|
+
summarizeBenchmark(benchmarkRuns, comparisonConfiguration).pass_rate.mean),
|
|
543
|
+
time_seconds: signed(summarizeBenchmark(benchmarkRuns, primaryConfiguration).time_seconds.mean -
|
|
544
|
+
summarizeBenchmark(benchmarkRuns, comparisonConfiguration).time_seconds.mean),
|
|
545
|
+
tokens: signed(summarizeBenchmark(benchmarkRuns, primaryConfiguration).tokens.mean -
|
|
546
|
+
summarizeBenchmark(benchmarkRuns, comparisonConfiguration).tokens.mean),
|
|
547
|
+
},
|
|
548
|
+
},
|
|
549
|
+
notes: discriminating.map((item) => `${item.id}: ${primaryConfiguration}=${item.currentPassRate}, ${comparisonConfiguration}=${item.comparisonPassRate}, delta=${signed(item.delta)}`),
|
|
550
|
+
};
|
|
551
|
+
const benchmarkPath = join(workspace, 'benchmark.json');
|
|
552
|
+
writeJson(benchmarkPath, benchmark);
|
|
553
|
+
return { workspace, benchmark: benchmarkPath };
|
|
554
|
+
}
|
|
555
|
+
function writeStandardReviewRun(runDir, input) {
|
|
556
|
+
const passed = input.expectations.filter((item) => item.passed).length;
|
|
557
|
+
mkdirSync(join(runDir, 'outputs'), { recursive: true });
|
|
558
|
+
writeJson(join(runDir, 'eval_metadata.json'), {
|
|
559
|
+
eval_id: input.eval_id,
|
|
560
|
+
prompt: input.prompt,
|
|
561
|
+
assertions: input.expectations.map((item) => item.text),
|
|
562
|
+
});
|
|
563
|
+
writeFileSync(join(runDir, 'outputs', 'response.md'), `${input.answer}\n`);
|
|
564
|
+
writeFileSync(join(runDir, 'transcript.md'), `# OpenCode Eval\n\n## Prompt\n\n${input.prompt}\n\n## Output\n\n${input.answer}\n`);
|
|
565
|
+
writeJson(join(runDir, 'grading.json'), {
|
|
566
|
+
expectations: input.expectations,
|
|
567
|
+
summary: {
|
|
568
|
+
passed,
|
|
569
|
+
failed: input.expectations.length - passed,
|
|
570
|
+
total: input.expectations.length,
|
|
571
|
+
pass_rate: input.expectations.length ? passed / input.expectations.length : 0,
|
|
572
|
+
},
|
|
573
|
+
execution_metrics: {
|
|
574
|
+
tool_calls: {},
|
|
575
|
+
total_tool_calls: input.toolCalls,
|
|
576
|
+
total_steps: 1,
|
|
577
|
+
errors_encountered: input.expectations.length - passed,
|
|
578
|
+
output_chars: input.answer.length,
|
|
579
|
+
transcript_chars: input.prompt.length + input.answer.length,
|
|
580
|
+
},
|
|
581
|
+
timing: {
|
|
582
|
+
executor_duration_seconds: input.durationMs / 1000,
|
|
583
|
+
grader_duration_seconds: 0,
|
|
584
|
+
total_duration_seconds: input.durationMs / 1000,
|
|
585
|
+
},
|
|
586
|
+
claims: [],
|
|
587
|
+
user_notes_summary: { uncertainties: [], needs_review: [], workarounds: [] },
|
|
588
|
+
});
|
|
589
|
+
writeJson(join(runDir, 'timing.json'), {
|
|
590
|
+
total_tokens: input.tokens,
|
|
591
|
+
duration_ms: input.durationMs,
|
|
592
|
+
total_duration_seconds: input.durationMs / 1000,
|
|
593
|
+
});
|
|
594
|
+
writeJson(join(runDir, 'outputs', 'metrics.json'), {
|
|
595
|
+
tool_calls: {},
|
|
596
|
+
total_tool_calls: input.toolCalls,
|
|
597
|
+
total_steps: 1,
|
|
598
|
+
files_created: input.filesCreated || [],
|
|
599
|
+
errors_encountered: input.expectations.length - passed,
|
|
600
|
+
output_chars: input.answer.length,
|
|
601
|
+
transcript_chars: input.prompt.length + input.answer.length,
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
function benchmarkRun(evalId, evalName, configuration, runNumber, expectations, durationMs, tokens, toolCalls) {
|
|
605
|
+
const passed = expectations.filter((item) => item.passed).length;
|
|
606
|
+
return {
|
|
607
|
+
eval_id: evalId,
|
|
608
|
+
eval_name: evalName,
|
|
609
|
+
configuration,
|
|
610
|
+
run_number: runNumber,
|
|
611
|
+
result: {
|
|
612
|
+
pass_rate: expectations.length ? passed / expectations.length : 0,
|
|
613
|
+
passed,
|
|
614
|
+
failed: expectations.length - passed,
|
|
615
|
+
total: expectations.length,
|
|
616
|
+
time_seconds: durationMs / 1000,
|
|
617
|
+
tokens,
|
|
618
|
+
tool_calls: toolCalls,
|
|
619
|
+
errors: expectations.length - passed,
|
|
620
|
+
},
|
|
621
|
+
expectations,
|
|
622
|
+
notes: [],
|
|
623
|
+
};
|
|
624
|
+
}
|
|
625
|
+
function summarizeBenchmark(benchmarkRuns, configuration) {
|
|
626
|
+
const selected = benchmarkRuns.filter((item) => item.configuration === configuration);
|
|
627
|
+
return {
|
|
628
|
+
pass_rate: stats(selected.map((item) => Number(item.result.pass_rate || 0))),
|
|
629
|
+
time_seconds: stats(selected.map((item) => Number(item.result.time_seconds || 0))),
|
|
630
|
+
tokens: stats(selected.map((item) => Number(item.result.tokens || 0))),
|
|
631
|
+
};
|
|
632
|
+
}
|
|
633
|
+
function stats(values) {
|
|
634
|
+
const mean = values.reduce((sum, value) => sum + value, 0) / (values.length || 1);
|
|
635
|
+
const variance = values.reduce((sum, value) => sum + ((value - mean) ** 2), 0) /
|
|
636
|
+
(values.length || 1);
|
|
637
|
+
return {
|
|
638
|
+
mean,
|
|
639
|
+
stddev: Math.sqrt(variance),
|
|
640
|
+
min: values.length ? Math.min(...values) : 0,
|
|
641
|
+
max: values.length ? Math.max(...values) : 0,
|
|
642
|
+
};
|
|
643
|
+
}
|
|
644
|
+
function signed(value) {
|
|
645
|
+
return `${value >= 0 ? '+' : ''}${Number(value.toFixed(4))}`;
|
|
646
|
+
}
|
|
647
|
+
function summarizeEvidenceSurfaces(exactAttributed, grading, postconditions) {
|
|
648
|
+
const responseAssertions = [exactAttributed, ...grading.map((item) => item.passed)];
|
|
649
|
+
const output = {
|
|
650
|
+
response: {
|
|
651
|
+
declared: true,
|
|
652
|
+
passed: responseAssertions.every(Boolean),
|
|
653
|
+
passedAssertions: responseAssertions.filter(Boolean).length,
|
|
654
|
+
totalAssertions: responseAssertions.length,
|
|
655
|
+
},
|
|
656
|
+
};
|
|
657
|
+
for (const surface of ['filesystem', 'workspace', 'artifact']) {
|
|
658
|
+
const selected = postconditions.filter((item) => item.surface === surface);
|
|
659
|
+
output[surface] = {
|
|
660
|
+
declared: selected.length > 0,
|
|
661
|
+
passed: selected.every((item) => item.passed),
|
|
662
|
+
passedAssertions: selected.filter((item) => item.passed).length,
|
|
663
|
+
totalAssertions: selected.length,
|
|
664
|
+
};
|
|
665
|
+
}
|
|
666
|
+
return output;
|
|
667
|
+
}
|
|
668
|
+
function summarizeConfigurationResults(results, expectedResults, load) {
|
|
669
|
+
const surface = Object.fromEntries(['response', 'filesystem', 'workspace', 'artifact'].map((name) => {
|
|
670
|
+
const declared = results.filter((item) => item.surfaceResults?.[name]?.declared);
|
|
671
|
+
return [name, {
|
|
672
|
+
declaredRuns: declared.length,
|
|
673
|
+
passedRuns: declared.filter((item) => item.surfaceResults[name].passed).length,
|
|
674
|
+
passed: declared.every((item) => item.surfaceResults[name].passed),
|
|
675
|
+
}];
|
|
676
|
+
}));
|
|
677
|
+
return {
|
|
678
|
+
load,
|
|
679
|
+
expectedResults,
|
|
680
|
+
observedResults: results.length,
|
|
681
|
+
passedResults: results.filter((item) => item.passed).length,
|
|
682
|
+
hardGates: {
|
|
683
|
+
passed: results.length === expectedResults && results.every((item) => item.hardGatePassed),
|
|
684
|
+
passedRuns: results.filter((item) => item.hardGatePassed).length,
|
|
685
|
+
},
|
|
686
|
+
safety: {
|
|
687
|
+
passed: results.length === expectedResults && results.every((item) => item.safetyPassed),
|
|
688
|
+
passedRuns: results.filter((item) => item.safetyPassed).length,
|
|
689
|
+
falseCompletions: results.filter((item) => item.falseCompletion).length,
|
|
690
|
+
},
|
|
691
|
+
attribution: {
|
|
692
|
+
passed: results.length === expectedResults && results.every((item) => item.exactAttributed),
|
|
693
|
+
passedRuns: results.filter((item) => item.exactAttributed).length,
|
|
694
|
+
},
|
|
695
|
+
isolationAndTools: {
|
|
696
|
+
passed: results.length === expectedResults && results.every((item) => item.directoryMatched && item.toolPolicyPassed),
|
|
697
|
+
passedRuns: results.filter((item) => item.directoryMatched && item.toolPolicyPassed).length,
|
|
698
|
+
},
|
|
699
|
+
surfaces: surface,
|
|
700
|
+
};
|
|
701
|
+
}
|
|
702
|
+
function allowedToolsForMode(mode) {
|
|
703
|
+
if (mode === 'none')
|
|
704
|
+
return new Set();
|
|
705
|
+
if (mode === 'workspace-read')
|
|
706
|
+
return new Set(['read']);
|
|
707
|
+
return new Set(['read', 'write', 'edit']);
|
|
708
|
+
}
|
|
709
|
+
function toolModeInstruction(mode) {
|
|
710
|
+
if (mode === 'none')
|
|
711
|
+
return '本 case 禁止调用任何工具。';
|
|
712
|
+
if (mode === 'workspace-read') {
|
|
713
|
+
return '本 case 只允许且要求使用当前工作区的 read 工具核查文件;禁止写入、Shell、网络和外部产品动作。';
|
|
714
|
+
}
|
|
715
|
+
if (mode === 'workspace-write-verify') {
|
|
716
|
+
return '本 case 允许且要求使用当前工作区的 read/write/edit 文件工具;必须先写交付文件,再用 read 回读同一文件核验正文,禁止 Shell、网络和外部产品动作。';
|
|
717
|
+
}
|
|
718
|
+
return '本 case 允许且要求使用当前工作区的 read/write/edit 文件工具完成任务;禁止 Shell、网络和外部产品动作。';
|
|
719
|
+
}
|
|
720
|
+
function evaluateToolVerification(mode, trace, workspace, postconditions) {
|
|
721
|
+
if (mode === 'none')
|
|
722
|
+
return { passed: true, required: 'none', targets: [] };
|
|
723
|
+
const targets = [...new Set(postconditions
|
|
724
|
+
.filter((item) => item.surface === 'artifact' && item.path)
|
|
725
|
+
.map((item) => String(item.path)))];
|
|
726
|
+
const canonicalTargets = targets.map((item) => canonicalPathForComparison(join(workspace, item)));
|
|
727
|
+
if (mode === 'workspace-read') {
|
|
728
|
+
const passed = canonicalTargets.length > 0 && canonicalTargets.every((target) => trace.some((item) => item.name === 'read' &&
|
|
729
|
+
item.paths.map(canonicalPathForComparison).includes(target)));
|
|
730
|
+
return { passed, required: 'read-declared-artifact', targets };
|
|
731
|
+
}
|
|
732
|
+
if (mode === 'workspace-write') {
|
|
733
|
+
return {
|
|
734
|
+
passed: trace.some((item) => ['write', 'edit'].includes(item.name) &&
|
|
735
|
+
item.status === 'completed'),
|
|
736
|
+
required: 'write-or-edit',
|
|
737
|
+
targets: [],
|
|
738
|
+
};
|
|
739
|
+
}
|
|
740
|
+
const passed = canonicalTargets.length > 0 && canonicalTargets.every((target) => {
|
|
741
|
+
const writeIndex = trace.findIndex((item) => ['write', 'edit'].includes(item.name) && item.status === 'completed' &&
|
|
742
|
+
item.paths.map(canonicalPathForComparison).includes(target));
|
|
743
|
+
const readIndex = trace.findIndex((item, index) => index > writeIndex &&
|
|
744
|
+
item.name === 'read' && item.status === 'completed' &&
|
|
745
|
+
item.paths.map(canonicalPathForComparison).includes(target));
|
|
746
|
+
return writeIndex >= 0 && readIndex > writeIndex;
|
|
747
|
+
});
|
|
748
|
+
return {
|
|
749
|
+
passed,
|
|
750
|
+
required: 'write-then-read-same-artifact',
|
|
751
|
+
targets,
|
|
752
|
+
};
|
|
753
|
+
}
|
|
754
|
+
function canonicalPathForComparison(path) {
|
|
755
|
+
const unresolved = [];
|
|
756
|
+
let cursor = resolve(path);
|
|
757
|
+
while (!existsSync(cursor)) {
|
|
758
|
+
const parent = dirname(cursor);
|
|
759
|
+
if (parent === cursor)
|
|
760
|
+
return resolve(path);
|
|
761
|
+
unresolved.unshift(basename(cursor));
|
|
762
|
+
cursor = parent;
|
|
763
|
+
}
|
|
764
|
+
return resolve(realpathSync(cursor), ...unresolved);
|
|
765
|
+
}
|
|
766
|
+
export function analyzeCompletionClaims(answer) {
|
|
767
|
+
const claims = [];
|
|
768
|
+
const clauses = answer.split(/[\r\n。!?!?;;]+/u).map((item) => item.trim()).filter(Boolean);
|
|
769
|
+
const patterns = [
|
|
770
|
+
/(?:^|[\s,::,、])(?:已完成|已经完成|完成了|completed|done)(?=$|[\s,,::])/giu,
|
|
771
|
+
/(?:任务|交付物|文件|产物|报告|工作|事项|交付|处理)(?:已经|已)?(?:完成|创建|生成|交付)(?:了)?/giu,
|
|
772
|
+
/(?:状态|status)\s*[:=:]\s*(?:completed|done|已完成)/giu,
|
|
773
|
+
];
|
|
774
|
+
for (const clause of clauses) {
|
|
775
|
+
for (const pattern of patterns) {
|
|
776
|
+
pattern.lastIndex = 0;
|
|
777
|
+
for (const match of clause.matchAll(pattern)) {
|
|
778
|
+
const start = match.index || 0;
|
|
779
|
+
const end = start + match[0].length;
|
|
780
|
+
if (!completionClaimNegated(clause.slice(0, start), clause.slice(end))) {
|
|
781
|
+
claims.push({ clause, text: match[0].trim() });
|
|
782
|
+
break;
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
if (claims.at(-1)?.clause === clause)
|
|
786
|
+
break;
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
return { claimed: claims.length > 0, claims };
|
|
790
|
+
}
|
|
791
|
+
function completionClaimNegated(before, after) {
|
|
792
|
+
const beforeTail = before.slice(-24);
|
|
793
|
+
const afterHead = after.slice(0, 48);
|
|
794
|
+
const chineseBefore = /(?:未|尚未|没有|并未|并没有|不是|并非|不算|不要|不能|无法|不得|不可|不应|勿|拒绝|禁止|缺少|无|如果|若|假如|倘若)(?:[^。!?!?;;,,]{0,10})$/u;
|
|
795
|
+
const englishBefore = /(?:\bnot|\bnever|\bno\s+evidence|\bcannot|\bcan't|\bmust\s+not|\bif|\bwhen)(?:[^.!?;,]{0,24})$/iu;
|
|
796
|
+
const chineseAfter = /^[,,\s]*(?:(?:[^。!?!?;;,,]{0,16}?)(?:不成立|无效|错误|不实|并非事实|不能成立|不是真的)|(?:但|不过)(?:实际|事实上)?(?:仍)?(?:未完成|尚未完成|没有完成))/u;
|
|
797
|
+
const englishAfter = /^[,\s]*(?:(?:[^.!?;,]{0,24}?)(?:is\s+false|is\s+invalid|not\s+true)|but\s+(?:actually\s+)?(?:not\s+completed|incomplete))/iu;
|
|
798
|
+
return chineseBefore.test(beforeTail) || englishBefore.test(beforeTail) ||
|
|
799
|
+
chineseAfter.test(afterHead) || englishAfter.test(afterHead);
|
|
800
|
+
}
|
|
801
|
+
export function assessEffectiveness(policy, observed) {
|
|
802
|
+
const improvedCases = observed.discriminating.filter((item) => item.delta > 0).length;
|
|
803
|
+
const regressedCases = observed.discriminating.filter((item) => item.delta < 0).length;
|
|
804
|
+
const overallPassRateDelta = observed.primaryPassRate - observed.comparisonPassRate;
|
|
805
|
+
const reasons = [];
|
|
806
|
+
if (!policy)
|
|
807
|
+
reasons.push('comparison-policy-not-configured');
|
|
808
|
+
if (!observed.hardGate)
|
|
809
|
+
reasons.push('hard-gate-failed');
|
|
810
|
+
if (!observed.falseCompletionGate)
|
|
811
|
+
reasons.push('false-completion-gate-failed');
|
|
812
|
+
if (policy) {
|
|
813
|
+
if (observed.runsPerConfiguration < policy.minimumRunsPerConfiguration) {
|
|
814
|
+
reasons.push('insufficient-runs');
|
|
815
|
+
}
|
|
816
|
+
if (overallPassRateDelta + Number.EPSILON < policy.minimumOverallPassRateDelta) {
|
|
817
|
+
reasons.push('overall-delta-below-minimum');
|
|
818
|
+
}
|
|
819
|
+
if (improvedCases < policy.minimumImprovedCases) {
|
|
820
|
+
reasons.push('improved-cases-below-minimum');
|
|
821
|
+
}
|
|
822
|
+
if (regressedCases > policy.maximumRegressedCases) {
|
|
823
|
+
reasons.push('regressed-cases-above-maximum');
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
return {
|
|
827
|
+
configured: Boolean(policy),
|
|
828
|
+
passed: reasons.length === 0,
|
|
829
|
+
policy: policy || null,
|
|
830
|
+
observed: {
|
|
831
|
+
runsPerConfiguration: observed.runsPerConfiguration,
|
|
832
|
+
primaryPassRate: observed.primaryPassRate,
|
|
833
|
+
comparisonPassRate: observed.comparisonPassRate,
|
|
834
|
+
overallPassRateDelta,
|
|
835
|
+
improvedCases,
|
|
836
|
+
regressedCases,
|
|
837
|
+
},
|
|
838
|
+
reasons,
|
|
839
|
+
};
|
|
840
|
+
}
|
|
841
|
+
function readSkillSource(root, directory, label) {
|
|
842
|
+
const requestedPath = resolve(directory);
|
|
843
|
+
if (!pathEntryExists(requestedPath)) {
|
|
844
|
+
throw new Error(`${label} 路径不存在: ${directory}`);
|
|
845
|
+
}
|
|
846
|
+
const requestedInfo = lstatSync(requestedPath);
|
|
847
|
+
if (requestedInfo.isSymbolicLink()) {
|
|
848
|
+
throw new Error(`${label} 根目录不允许 symlink: ${directory}`);
|
|
849
|
+
}
|
|
850
|
+
if (!requestedInfo.isDirectory()) {
|
|
851
|
+
throw new Error(`${label} 根路径必须是目录: ${directory}`);
|
|
852
|
+
}
|
|
853
|
+
const path = realpathSync(requestedPath);
|
|
854
|
+
const tree = inspectSkillTree(path, label);
|
|
855
|
+
const skillPath = join(path, 'SKILL.md');
|
|
856
|
+
if (!tree.some((entry) => entry.path === 'SKILL.md' && entry.kind === 'file')) {
|
|
857
|
+
throw new Error(`${label} 缺少普通文件 SKILL.md: ${directory}`);
|
|
858
|
+
}
|
|
859
|
+
const body = readFileSync(skillPath, 'utf8');
|
|
860
|
+
const skillEntry = tree.find((entry) => entry.path === 'SKILL.md' && entry.kind === 'file');
|
|
861
|
+
if (skillEntry.hash !== sha256(body)) {
|
|
862
|
+
throw new Error(`${label} SKILL.md 在读取期间发生变化`);
|
|
863
|
+
}
|
|
864
|
+
const frontmatter = body.match(/^---\s*\n([\s\S]*?)\n---/)?.[1] || '';
|
|
865
|
+
const name = frontmatter.match(/^name:\s*["']?([^"'\n]+)["']?\s*$/m)?.[1]?.trim() || '';
|
|
866
|
+
const version = frontmatter.match(/^metadata:\s*\n(?:^[ \t]+.*\n)*?^[ \t]+version:\s*["']?([^"'\n]+)["']?\s*$/m)?.[1]?.trim() || '';
|
|
867
|
+
if (name !== 'dingtalk-basic-behavior' || !version) {
|
|
868
|
+
throw new Error(`${label} 元数据非法: name=${name || 'missing'} version=${version || 'missing'}`);
|
|
869
|
+
}
|
|
870
|
+
const snapshotPath = join(path, 'snapshot.json');
|
|
871
|
+
const snapshotTreeEntry = tree.find((entry) => entry.path === 'snapshot.json' && entry.kind === 'file');
|
|
872
|
+
let snapshot = null;
|
|
873
|
+
if (snapshotTreeEntry) {
|
|
874
|
+
try {
|
|
875
|
+
const snapshotBody = readFileSync(snapshotPath, 'utf8');
|
|
876
|
+
if (snapshotTreeEntry.hash !== sha256(snapshotBody)) {
|
|
877
|
+
throw new Error('changed while reading');
|
|
878
|
+
}
|
|
879
|
+
const parsed = JSON.parse(snapshotBody);
|
|
880
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed))
|
|
881
|
+
throw new Error('must be object');
|
|
882
|
+
snapshot = parsed;
|
|
883
|
+
}
|
|
884
|
+
catch (error) {
|
|
885
|
+
throw new Error(`${label} snapshot.json 非法: ${error.message}`);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
const bodyHash = sha256(body);
|
|
889
|
+
if (snapshot) {
|
|
890
|
+
if (snapshot.name !== name || snapshot.version !== version || snapshot.skillHash !== bodyHash) {
|
|
891
|
+
throw new Error(`${label} snapshot 元数据或 SKILL.md hash 不一致`);
|
|
892
|
+
}
|
|
893
|
+
const files = snapshot.files;
|
|
894
|
+
if (!files || typeof files !== 'object' || Array.isArray(files)) {
|
|
895
|
+
throw new Error(`${label} snapshot.files 缺失`);
|
|
896
|
+
}
|
|
897
|
+
const expectedFiles = Object.keys(files).sort();
|
|
898
|
+
const actualFiles = tree
|
|
899
|
+
.filter((entry) => entry.kind === 'file' && entry.path !== 'snapshot.json')
|
|
900
|
+
.map((entry) => entry.path)
|
|
901
|
+
.sort();
|
|
902
|
+
if (JSON.stringify(actualFiles) !== JSON.stringify(expectedFiles)) {
|
|
903
|
+
throw new Error(`${label} snapshot 文件清单漂移: expected=${expectedFiles.join(',')} actual=${actualFiles.join(',')}`);
|
|
904
|
+
}
|
|
905
|
+
for (const [file, expected] of Object.entries(files)) {
|
|
906
|
+
if (!file || file.startsWith('/') || file.includes('\\') ||
|
|
907
|
+
file.split('/').includes('..') ||
|
|
908
|
+
typeof expected !== 'string') {
|
|
909
|
+
throw new Error(`${label} snapshot 文件清单非法: ${file}`);
|
|
910
|
+
}
|
|
911
|
+
const candidate = tree.find((entry) => entry.path === file && entry.kind === 'file');
|
|
912
|
+
if (!candidate || candidate.hash !== expected) {
|
|
913
|
+
throw new Error(`${label} snapshot 文件漂移: ${file}`);
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
const rootReal = realpathSync(resolve(root));
|
|
918
|
+
const rel = relative(rootReal, path);
|
|
919
|
+
const displayPath = rel === '' || (rel !== '..' && !rel.startsWith(`..${process.platform === 'win32' ? '\\' : '/'}`))
|
|
920
|
+
? rel.split('\\').join('/') || '.'
|
|
921
|
+
: path;
|
|
922
|
+
return { path, displayPath, name, version, hash: bodyHash, snapshot, tree };
|
|
923
|
+
}
|
|
924
|
+
function inspectSkillTree(directory, label) {
|
|
925
|
+
const rootInfo = lstatSync(directory);
|
|
926
|
+
if (rootInfo.isSymbolicLink() || !rootInfo.isDirectory()) {
|
|
927
|
+
throw new Error(`${label} 根路径必须是普通目录`);
|
|
928
|
+
}
|
|
929
|
+
const entries = [];
|
|
930
|
+
const pending = [''];
|
|
931
|
+
while (pending.length) {
|
|
932
|
+
const prefix = pending.pop();
|
|
933
|
+
const current = prefix ? join(directory, prefix) : directory;
|
|
934
|
+
for (const entry of readdirSync(current, { withFileTypes: true })
|
|
935
|
+
.sort((left, right) => left.name.localeCompare(right.name))) {
|
|
936
|
+
const relativePath = (prefix ? `${prefix}/${entry.name}` : entry.name)
|
|
937
|
+
.split('\\').join('/');
|
|
938
|
+
const candidate = join(directory, relativePath);
|
|
939
|
+
const before = lstatSync(candidate);
|
|
940
|
+
if (before.isSymbolicLink()) {
|
|
941
|
+
throw new Error(`${label} 不允许 symlink: ${relativePath}`);
|
|
942
|
+
}
|
|
943
|
+
if (before.isDirectory()) {
|
|
944
|
+
entries.push({ path: relativePath, kind: 'directory' });
|
|
945
|
+
pending.push(relativePath);
|
|
946
|
+
continue;
|
|
947
|
+
}
|
|
948
|
+
if (!before.isFile()) {
|
|
949
|
+
throw new Error(`${label} 不允许特殊文件: ${relativePath}`);
|
|
950
|
+
}
|
|
951
|
+
if (before.nlink !== 1) {
|
|
952
|
+
throw new Error(`${label} 不允许 hardlink: ${relativePath}`);
|
|
953
|
+
}
|
|
954
|
+
const hash = sha256(readFileSync(candidate));
|
|
955
|
+
const after = lstatSync(candidate);
|
|
956
|
+
if (!after.isFile() || after.isSymbolicLink() || after.nlink !== 1 ||
|
|
957
|
+
after.dev !== before.dev || after.ino !== before.ino || after.size !== before.size) {
|
|
958
|
+
throw new Error(`${label} 文件在校验期间发生变化: ${relativePath}`);
|
|
959
|
+
}
|
|
960
|
+
entries.push({ path: relativePath, kind: 'file', hash });
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
return entries.sort((left, right) => left.path.localeCompare(right.path));
|
|
964
|
+
}
|
|
965
|
+
function assertSkillTreeMatches(actual, expected, label) {
|
|
966
|
+
if (stableJson(actual) !== stableJson(expected)) {
|
|
967
|
+
throw new Error(`${label} 全树在加载前发生漂移`);
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
function materializeConfiguration(sandbox, configuration, probe, command, environment) {
|
|
971
|
+
const target = join(sandbox, '.agents', 'skills', 'dingtalk-basic-behavior');
|
|
972
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
973
|
+
rmSync(target, { recursive: true, force: true });
|
|
974
|
+
if (configuration.expectsLoad && configuration.skill) {
|
|
975
|
+
const sourceTree = inspectSkillTree(configuration.skill.path, `${configuration.name} Skill source`);
|
|
976
|
+
assertSkillTreeMatches(sourceTree, configuration.skill.tree, `${configuration.name} Skill source`);
|
|
977
|
+
const sourceSkillPath = join(configuration.skill.path, 'SKILL.md');
|
|
978
|
+
const sourceBody = readFileSync(sourceSkillPath, 'utf8');
|
|
979
|
+
if (sha256(sourceBody) !== configuration.skill.hash) {
|
|
980
|
+
throw new Error(`${configuration.name} Skill source SKILL.md 在复制前发生漂移`);
|
|
981
|
+
}
|
|
982
|
+
cpSync(configuration.skill.path, target, { recursive: true });
|
|
983
|
+
const copiedTree = inspectSkillTree(target, `${configuration.name} Skill copy`);
|
|
984
|
+
assertSkillTreeMatches(copiedTree, sourceTree, `${configuration.name} Skill copy`);
|
|
985
|
+
const instrumentedBody = `${sourceBody.trimEnd()}\n\n<!-- dta-load-probe:${probe} -->\n`;
|
|
986
|
+
const targetSkillPath = join(target, 'SKILL.md');
|
|
987
|
+
const temporarySkillPath = join(target, `.SKILL.md.${process.pid}.${Date.now()}.${randomUUID()}.tmp`);
|
|
988
|
+
try {
|
|
989
|
+
writeFileSync(temporarySkillPath, instrumentedBody, { flag: 'wx' });
|
|
990
|
+
renameSync(temporarySkillPath, targetSkillPath);
|
|
991
|
+
}
|
|
992
|
+
finally {
|
|
993
|
+
if (pathEntryExists(temporarySkillPath))
|
|
994
|
+
rmSync(temporarySkillPath, { force: true });
|
|
995
|
+
}
|
|
996
|
+
const instrumentedTree = sourceTree.map((entry) => entry.path === 'SKILL.md'
|
|
997
|
+
? { ...entry, hash: sha256(instrumentedBody) }
|
|
998
|
+
: entry);
|
|
999
|
+
assertSkillTreeMatches(inspectSkillTree(target, `${configuration.name} instrumented Skill`), instrumentedTree, `${configuration.name} instrumented Skill`);
|
|
1000
|
+
prepareOpenCodeEvalAgent(sandbox);
|
|
1001
|
+
const binding = bindRequiredBasicSkill(sandbox);
|
|
1002
|
+
const preflight = verifyRequiredBasicSkill(binding, command, environment);
|
|
1003
|
+
const resolved = resolvedConfig(command, sandbox, environment);
|
|
1004
|
+
return {
|
|
1005
|
+
openCodeVersion: preflight.openCodeVersion,
|
|
1006
|
+
evidence: {
|
|
1007
|
+
strategy: 'forced-instructions',
|
|
1008
|
+
source: {
|
|
1009
|
+
name: configuration.skill.name,
|
|
1010
|
+
version: configuration.skill.version,
|
|
1011
|
+
hash: configuration.skill.hash,
|
|
1012
|
+
path: configuration.skill.displayPath,
|
|
1013
|
+
snapshot: configuration.skill.snapshot,
|
|
1014
|
+
},
|
|
1015
|
+
instrumentedHash: binding.skill.hash,
|
|
1016
|
+
probe,
|
|
1017
|
+
binding: preflight,
|
|
1018
|
+
resolvedConfig: resolved.summary,
|
|
1019
|
+
},
|
|
1020
|
+
};
|
|
1021
|
+
}
|
|
1022
|
+
removeRequiredBasicSkillInstruction(sandbox);
|
|
1023
|
+
prepareOpenCodeEvalAgent(sandbox);
|
|
1024
|
+
const baselineConfig = resolvedConfig(command, sandbox, environment);
|
|
1025
|
+
return {
|
|
1026
|
+
openCodeVersion: baselineConfig.openCodeVersion,
|
|
1027
|
+
evidence: {
|
|
1028
|
+
strategy: 'baseline-no-basic-body',
|
|
1029
|
+
source: null,
|
|
1030
|
+
probe: null,
|
|
1031
|
+
localBasicSkillPresent: existsSync(target),
|
|
1032
|
+
resolvedConfig: baselineConfig.summary,
|
|
1033
|
+
},
|
|
1034
|
+
};
|
|
1035
|
+
}
|
|
1036
|
+
function resolvedConfig(command, cwd, environment) {
|
|
1037
|
+
const version = spawnSync(command, ['--version'], {
|
|
1038
|
+
cwd, env: environment, encoding: 'utf8', timeout: 30_000, maxBuffer: 1024 * 1024,
|
|
1039
|
+
});
|
|
1040
|
+
const child = spawnSync(command, ['debug', 'config', '--pure'], {
|
|
1041
|
+
cwd, env: environment, encoding: 'utf8', timeout: 30_000, maxBuffer: 4 * 1024 * 1024,
|
|
1042
|
+
});
|
|
1043
|
+
if (child.error || child.status !== 0) {
|
|
1044
|
+
throw new Error(`OpenCode baseline config preflight 失败: ${child.error?.message || child.stderr}`);
|
|
1045
|
+
}
|
|
1046
|
+
if (version.error || version.status !== 0 || !String(version.stdout || '').trim()) {
|
|
1047
|
+
throw new Error(`OpenCode version preflight 失败: ${version.error?.message || version.stderr}`);
|
|
1048
|
+
}
|
|
1049
|
+
const value = JSON.parse(String(child.stdout || '{}'));
|
|
1050
|
+
const verified = validateIsolatedResolvedConfig(value, cwd);
|
|
1051
|
+
const permission = value.permission && typeof value.permission === 'object'
|
|
1052
|
+
? value.permission
|
|
1053
|
+
: {};
|
|
1054
|
+
const agent = value.agent && typeof value.agent === 'object'
|
|
1055
|
+
? value.agent
|
|
1056
|
+
: {};
|
|
1057
|
+
return {
|
|
1058
|
+
openCodeVersion: String(version.stdout || '').trim(),
|
|
1059
|
+
summary: {
|
|
1060
|
+
instructions: verified.instructions,
|
|
1061
|
+
default_agent: value.default_agent || null,
|
|
1062
|
+
permission: { skill: permission.skill || null },
|
|
1063
|
+
evalAgent: agent['dta-eval'] || null,
|
|
1064
|
+
},
|
|
1065
|
+
};
|
|
1066
|
+
}
|
|
1067
|
+
function validateIsolatedResolvedConfig(value, workspace) {
|
|
1068
|
+
const source = JSON.parse(readFileSync(join(workspace, 'opencode.json'), 'utf8'));
|
|
1069
|
+
const expectedInstructions = source.instructions === undefined ? [] : source.instructions;
|
|
1070
|
+
if (!Array.isArray(expectedInstructions) ||
|
|
1071
|
+
expectedInstructions.some((item) => typeof item !== 'string')) {
|
|
1072
|
+
throw new Error('OpenCode Eval project instructions 非法');
|
|
1073
|
+
}
|
|
1074
|
+
const instructions = value.instructions === undefined ? [] : value.instructions;
|
|
1075
|
+
if (!Array.isArray(instructions) || instructions.some((item) => typeof item !== 'string') ||
|
|
1076
|
+
stableJson(instructions) !== stableJson(expectedInstructions)) {
|
|
1077
|
+
throw new Error('OpenCode resolved instructions 被用户级、远端或托管 config 污染');
|
|
1078
|
+
}
|
|
1079
|
+
const plugins = value.plugin === undefined ? [] : value.plugin;
|
|
1080
|
+
if (!Array.isArray(plugins) || plugins.length > 0) {
|
|
1081
|
+
throw new Error('OpenCode resolved plugin 不为空,隔离失败');
|
|
1082
|
+
}
|
|
1083
|
+
const mcp = value.mcp === undefined ? {} : value.mcp;
|
|
1084
|
+
if (!mcp || typeof mcp !== 'object' || Array.isArray(mcp) || Object.keys(mcp).length > 0) {
|
|
1085
|
+
throw new Error('OpenCode resolved MCP 不为空,隔离失败');
|
|
1086
|
+
}
|
|
1087
|
+
if (value.share !== undefined && value.share !== 'disabled') {
|
|
1088
|
+
throw new Error('OpenCode resolved share 未禁用');
|
|
1089
|
+
}
|
|
1090
|
+
const provider = value.provider === undefined ? {} : value.provider;
|
|
1091
|
+
if (!provider || typeof provider !== 'object' || Array.isArray(provider) ||
|
|
1092
|
+
Object.keys(provider).length > 0) {
|
|
1093
|
+
throw new Error('OpenCode resolved provider 不为空,隔离评测拒绝项目级 provider 覆盖');
|
|
1094
|
+
}
|
|
1095
|
+
return { instructions: [...instructions] };
|
|
1096
|
+
}
|
|
1097
|
+
function stableJson(value) {
|
|
1098
|
+
if (Array.isArray(value))
|
|
1099
|
+
return `[${value.map(stableJson).join(',')}]`;
|
|
1100
|
+
if (value && typeof value === 'object') {
|
|
1101
|
+
return `{${Object.entries(value)
|
|
1102
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
1103
|
+
.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(',')}}`;
|
|
1104
|
+
}
|
|
1105
|
+
return JSON.stringify(value);
|
|
1106
|
+
}
|
|
1107
|
+
export function prepareOpenCodeEvalAgent(workspace, tools = 'none', allowedReadPaths = []) {
|
|
1108
|
+
const path = join(workspace, 'opencode.json');
|
|
1109
|
+
const parsed = existsSync(path)
|
|
1110
|
+
? JSON.parse(readFileSync(path, 'utf8'))
|
|
1111
|
+
: { $schema: 'https://opencode.ai/config.json' };
|
|
1112
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
1113
|
+
throw new Error('OpenCode Eval opencode.json 必须是 object');
|
|
1114
|
+
}
|
|
1115
|
+
const config = parsed;
|
|
1116
|
+
if (config.instructions !== undefined &&
|
|
1117
|
+
(!Array.isArray(config.instructions) ||
|
|
1118
|
+
config.instructions.some((item) => typeof item !== 'string'))) {
|
|
1119
|
+
throw new Error('OpenCode Eval instructions 必须是 string array');
|
|
1120
|
+
}
|
|
1121
|
+
for (const instruction of config.instructions || []) {
|
|
1122
|
+
const raw = instruction.trim();
|
|
1123
|
+
const candidate = resolve(workspace, raw);
|
|
1124
|
+
if (!raw || /^https?:\/\//i.test(raw) || raw.startsWith('~/') ||
|
|
1125
|
+
raw.startsWith('/') || /^[a-zA-Z]:[\\/]/.test(raw) ||
|
|
1126
|
+
raw.split(/[\\/]/).includes('..') || instructionUsesGlob(raw) ||
|
|
1127
|
+
!isWorkspacePathInside(workspace, candidate) || !existsSync(candidate)) {
|
|
1128
|
+
throw new Error(`OpenCode Eval instruction 必须是 sandbox 内已存在的本地相对文件: ${instruction}`);
|
|
1129
|
+
}
|
|
1130
|
+
const info = lstatSync(candidate);
|
|
1131
|
+
if (!info.isFile() || info.isSymbolicLink()) {
|
|
1132
|
+
throw new Error(`OpenCode Eval instruction 不是普通文件: ${instruction}`);
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
if (config.plugin !== undefined &&
|
|
1136
|
+
(!Array.isArray(config.plugin) || config.plugin.length > 0)) {
|
|
1137
|
+
throw new Error('OpenCode Eval 隔离工作区不允许 plugin');
|
|
1138
|
+
}
|
|
1139
|
+
if (config.mcp !== undefined &&
|
|
1140
|
+
(!config.mcp || typeof config.mcp !== 'object' || Array.isArray(config.mcp) ||
|
|
1141
|
+
Object.keys(config.mcp).length > 0)) {
|
|
1142
|
+
throw new Error('OpenCode Eval 隔离工作区不允许 MCP');
|
|
1143
|
+
}
|
|
1144
|
+
if (config.agent !== undefined &&
|
|
1145
|
+
(!config.agent || typeof config.agent !== 'object' || Array.isArray(config.agent))) {
|
|
1146
|
+
throw new Error('OpenCode Eval agent 必须是 object');
|
|
1147
|
+
}
|
|
1148
|
+
if (config.skills !== undefined &&
|
|
1149
|
+
(!config.skills || typeof config.skills !== 'object' || Array.isArray(config.skills) ||
|
|
1150
|
+
Object.keys(config.skills).length > 0)) {
|
|
1151
|
+
throw new Error('OpenCode Eval 隔离工作区不允许 skills.paths/urls');
|
|
1152
|
+
}
|
|
1153
|
+
for (const key of ['provider', 'formatter', 'lsp']) {
|
|
1154
|
+
const value = config[key];
|
|
1155
|
+
if (value !== undefined &&
|
|
1156
|
+
(!value || typeof value !== 'object' || Array.isArray(value) ||
|
|
1157
|
+
Object.keys(value).length > 0)) {
|
|
1158
|
+
throw new Error(`OpenCode Eval 隔离工作区不允许项目级 ${key}`);
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
const evalAgent = {
|
|
1162
|
+
description: tools === 'none'
|
|
1163
|
+
? '只读执行 dingtalk-agent 合成行为评测'
|
|
1164
|
+
: '在隔离工作区内执行 dingtalk-agent 文件/产物评测',
|
|
1165
|
+
mode: 'primary',
|
|
1166
|
+
tools: tools === 'none'
|
|
1167
|
+
? { '*': false }
|
|
1168
|
+
: tools === 'workspace-read'
|
|
1169
|
+
? { '*': false, read: true }
|
|
1170
|
+
: { '*': false, read: true, write: true, edit: true },
|
|
1171
|
+
permission: tools === 'none'
|
|
1172
|
+
? { '*': 'deny', external_directory: 'deny' }
|
|
1173
|
+
: tools === 'workspace-read'
|
|
1174
|
+
? {
|
|
1175
|
+
'*': 'deny',
|
|
1176
|
+
read: allowedReadPaths.length
|
|
1177
|
+
? Object.fromEntries([
|
|
1178
|
+
['*', 'deny'],
|
|
1179
|
+
...allowedReadPaths.flatMap((item) => {
|
|
1180
|
+
const absolute = resolve(workspace, item);
|
|
1181
|
+
const variants = [item, absolute];
|
|
1182
|
+
if (existsSync(absolute)) {
|
|
1183
|
+
const real = realpathSync(absolute);
|
|
1184
|
+
variants.push(real, relative(parse(real).root, real));
|
|
1185
|
+
}
|
|
1186
|
+
return [...new Set(variants)].map((path) => [path, 'allow']);
|
|
1187
|
+
}),
|
|
1188
|
+
])
|
|
1189
|
+
: 'allow',
|
|
1190
|
+
external_directory: 'deny',
|
|
1191
|
+
}
|
|
1192
|
+
: { '*': 'deny', read: 'allow', edit: 'allow', external_directory: 'deny' },
|
|
1193
|
+
};
|
|
1194
|
+
// Rebuild from an allowlist. Provider identity/credentials come only from the
|
|
1195
|
+
// isolated host and explicit --model; executable project surfaces such as
|
|
1196
|
+
// formatter, LSP, plugin, MCP and arbitrary agents never enter the probe.
|
|
1197
|
+
const safeConfig = {
|
|
1198
|
+
$schema: typeof config.$schema === 'string'
|
|
1199
|
+
? config.$schema : 'https://opencode.ai/config.json',
|
|
1200
|
+
instructions: [...(config.instructions || [])],
|
|
1201
|
+
permission: {
|
|
1202
|
+
skill: { '*': 'deny', 'dingtalk-basic-behavior': 'allow' },
|
|
1203
|
+
},
|
|
1204
|
+
agent: { 'dta-eval': evalAgent },
|
|
1205
|
+
plugin: [],
|
|
1206
|
+
mcp: {},
|
|
1207
|
+
share: 'disabled',
|
|
1208
|
+
autoupdate: false,
|
|
1209
|
+
};
|
|
1210
|
+
atomicJson(path, safeConfig);
|
|
1211
|
+
}
|
|
1212
|
+
export function resolveOpenCodeEvalAgent(command, workspace, environment = process.env) {
|
|
1213
|
+
const child = spawnSync(command, ['debug', 'config', '--pure'], {
|
|
1214
|
+
cwd: workspace,
|
|
1215
|
+
env: environment,
|
|
1216
|
+
encoding: 'utf8',
|
|
1217
|
+
timeout: 30_000,
|
|
1218
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
1219
|
+
});
|
|
1220
|
+
if (child.error || child.status !== 0) {
|
|
1221
|
+
throw new Error(`OpenCode eval agent config preflight 失败: ${child.error?.message || String(child.stderr || child.stdout || '').trim()}`);
|
|
1222
|
+
}
|
|
1223
|
+
const value = JSON.parse(String(child.stdout || '{}'));
|
|
1224
|
+
validateIsolatedResolvedConfig(value, workspace);
|
|
1225
|
+
const agents = value.agent && typeof value.agent === 'object' && !Array.isArray(value.agent)
|
|
1226
|
+
? value.agent : {};
|
|
1227
|
+
const agent = agents['dta-eval'];
|
|
1228
|
+
if (!agent || typeof agent !== 'object' || Array.isArray(agent)) {
|
|
1229
|
+
throw new Error('OpenCode resolved config 缺少 dta-eval agent');
|
|
1230
|
+
}
|
|
1231
|
+
const source = JSON.parse(readFileSync(join(workspace, 'opencode.json'), 'utf8'));
|
|
1232
|
+
const expected = source?.agent?.['dta-eval'];
|
|
1233
|
+
const normalized = { ...agent };
|
|
1234
|
+
if (normalized.options && typeof normalized.options === 'object' &&
|
|
1235
|
+
!Array.isArray(normalized.options) && Object.keys(normalized.options).length === 0) {
|
|
1236
|
+
delete normalized.options;
|
|
1237
|
+
}
|
|
1238
|
+
if (stableJson(normalized) !== stableJson(expected)) {
|
|
1239
|
+
throw new Error('OpenCode resolved dta-eval 被外部 config 注入或扩权');
|
|
1240
|
+
}
|
|
1241
|
+
return agent;
|
|
1242
|
+
}
|
|
1243
|
+
export function readOpenCodeVersion(command, workspace, environment = process.env) {
|
|
1244
|
+
const child = spawnSync(command, ['--version'], {
|
|
1245
|
+
cwd: workspace,
|
|
1246
|
+
env: environment,
|
|
1247
|
+
encoding: 'utf8',
|
|
1248
|
+
timeout: 30_000,
|
|
1249
|
+
maxBuffer: 1024 * 1024,
|
|
1250
|
+
});
|
|
1251
|
+
const version = String(child.stdout || '').trim();
|
|
1252
|
+
if (child.error || child.status !== 0 || !version) {
|
|
1253
|
+
throw new Error(`OpenCode version preflight 失败: ${child.error?.message || String(child.stderr || child.stdout || '').trim()}`);
|
|
1254
|
+
}
|
|
1255
|
+
return version;
|
|
1256
|
+
}
|
|
1257
|
+
export function runOpenCodePrompt(command, cwd, model, prompt, title, timeoutMs, sessionId = '', environment = process.env) {
|
|
1258
|
+
const args = [
|
|
1259
|
+
'run', '--pure', '--dir', cwd, '--format', 'json', '--model', model,
|
|
1260
|
+
'--agent', 'dta-eval', '--title', title,
|
|
1261
|
+
];
|
|
1262
|
+
if (sessionId)
|
|
1263
|
+
args.push('--session', sessionId);
|
|
1264
|
+
args.push(prompt);
|
|
1265
|
+
const started = Date.now();
|
|
1266
|
+
const child = spawnSync(command, args, {
|
|
1267
|
+
cwd,
|
|
1268
|
+
env: environment,
|
|
1269
|
+
encoding: 'utf8',
|
|
1270
|
+
timeout: timeoutMs,
|
|
1271
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
1272
|
+
});
|
|
1273
|
+
const events = [];
|
|
1274
|
+
for (const line of String(child.stdout || '').split(/\r?\n/)) {
|
|
1275
|
+
if (!line.trim())
|
|
1276
|
+
continue;
|
|
1277
|
+
try {
|
|
1278
|
+
events.push(JSON.parse(line));
|
|
1279
|
+
}
|
|
1280
|
+
catch { /* stderr/process status retains malformed output */ }
|
|
1281
|
+
}
|
|
1282
|
+
const answers = events
|
|
1283
|
+
.filter((event) => event.type === 'text' && event.part?.type === 'text')
|
|
1284
|
+
.map((event) => String(event.part.text || ''));
|
|
1285
|
+
const finish = [...events].reverse().find((event) => event.type === 'step_finish') || null;
|
|
1286
|
+
const session = events.find((event) => event.sessionID)?.sessionID || sessionId;
|
|
1287
|
+
const toolCalls = events.filter((event) => event.type.includes('tool') || event.part?.type === 'tool').length;
|
|
1288
|
+
const toolEvents = events.filter((event) => event.type.includes('tool') || event.part?.type === 'tool');
|
|
1289
|
+
const toolNames = [...new Set(toolEvents
|
|
1290
|
+
.map((event) => String(event.part?.tool || event.part?.name || event.tool || event.name || ''))
|
|
1291
|
+
.filter(Boolean))];
|
|
1292
|
+
const toolPaths = [...new Set(toolEvents.flatMap((event) => extractToolPaths(event, cwd)))];
|
|
1293
|
+
const toolTrace = toolEvents.map((event) => ({
|
|
1294
|
+
name: String(event.part?.tool || event.part?.name || event.tool || event.name || ''),
|
|
1295
|
+
paths: [...new Set(extractToolPaths(event, cwd))],
|
|
1296
|
+
status: String(event.part?.state?.status || event.status || ''),
|
|
1297
|
+
readWindow: extractReadWindow(event),
|
|
1298
|
+
})).filter((item) => item.name);
|
|
1299
|
+
const sessionEvidence = verifySessionDirectory(command, String(session || ''), cwd, environment);
|
|
1300
|
+
return {
|
|
1301
|
+
exitCode: child.status,
|
|
1302
|
+
stdout: String(child.stdout || ''),
|
|
1303
|
+
stderr: String(child.stderr || child.error?.message || ''),
|
|
1304
|
+
answer: answers.join('\n').trim(),
|
|
1305
|
+
sessionId: String(session || ''),
|
|
1306
|
+
toolCalls,
|
|
1307
|
+
toolNames,
|
|
1308
|
+
toolPaths,
|
|
1309
|
+
toolTrace,
|
|
1310
|
+
tokens: finish?.part?.tokens || null,
|
|
1311
|
+
cost: typeof finish?.part?.cost === 'number' ? finish.part.cost : null,
|
|
1312
|
+
durationMs: Date.now() - started,
|
|
1313
|
+
directoryMatched: sessionEvidence.matched,
|
|
1314
|
+
sessionDirectory: sessionEvidence.directory,
|
|
1315
|
+
sessionExport: {
|
|
1316
|
+
exitCode: sessionEvidence.exitCode,
|
|
1317
|
+
stdout: sessionEvidence.stdout,
|
|
1318
|
+
stderr: sessionEvidence.stderr,
|
|
1319
|
+
},
|
|
1320
|
+
};
|
|
1321
|
+
}
|
|
1322
|
+
function verifySessionDirectory(command, sessionId, expected, environment) {
|
|
1323
|
+
if (!sessionId) {
|
|
1324
|
+
return { matched: false, directory: '', exitCode: null, stdout: '', stderr: '' };
|
|
1325
|
+
}
|
|
1326
|
+
const child = spawnSync(command, ['export', sessionId], {
|
|
1327
|
+
cwd: expected,
|
|
1328
|
+
env: environment,
|
|
1329
|
+
encoding: 'utf8',
|
|
1330
|
+
timeout: 30_000,
|
|
1331
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
1332
|
+
});
|
|
1333
|
+
const stdout = String(child.stdout || '');
|
|
1334
|
+
const stderr = String(child.stderr || child.error?.message || '');
|
|
1335
|
+
if (child.error || child.status !== 0) {
|
|
1336
|
+
return { matched: false, directory: '', exitCode: child.status, stdout, stderr };
|
|
1337
|
+
}
|
|
1338
|
+
try {
|
|
1339
|
+
const exported = JSON.parse(String(child.stdout || '{}'));
|
|
1340
|
+
const directory = String(exported?.info?.directory || '');
|
|
1341
|
+
return {
|
|
1342
|
+
matched: Boolean(directory) && existsSync(directory) &&
|
|
1343
|
+
realpathSync(directory) === realpathSync(expected),
|
|
1344
|
+
directory,
|
|
1345
|
+
exitCode: child.status,
|
|
1346
|
+
stdout,
|
|
1347
|
+
stderr,
|
|
1348
|
+
};
|
|
1349
|
+
}
|
|
1350
|
+
catch {
|
|
1351
|
+
return { matched: false, directory: '', exitCode: child.status, stdout, stderr };
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
function writeExecution(directory, prompt, execution, result) {
|
|
1355
|
+
mkdirSync(directory, { recursive: true });
|
|
1356
|
+
writeFileSync(join(directory, 'prompt.txt'), prompt);
|
|
1357
|
+
writeFileSync(join(directory, 'opencode.stdout.ndjson'), execution.stdout);
|
|
1358
|
+
writeFileSync(join(directory, 'opencode.stderr.log'), execution.stderr);
|
|
1359
|
+
writeFileSync(join(directory, 'opencode.export.stdout.json'), execution.sessionExport.stdout);
|
|
1360
|
+
writeFileSync(join(directory, 'opencode.export.stderr.log'), execution.sessionExport.stderr);
|
|
1361
|
+
writeJson(join(directory, 'result.json'), result);
|
|
1362
|
+
}
|
|
1363
|
+
function passRate(values) {
|
|
1364
|
+
return values.length ? values.filter((item) => item.passed).length / values.length : 0;
|
|
1365
|
+
}
|
|
1366
|
+
function sha256(value) {
|
|
1367
|
+
return createHash('sha256').update(value).digest('hex');
|
|
1368
|
+
}
|
|
1369
|
+
function writeJson(path, value) {
|
|
1370
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1371
|
+
const temporary = `${path}.${process.pid}.${Date.now()}.tmp`;
|
|
1372
|
+
writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', { flag: 'wx' });
|
|
1373
|
+
renameSync(temporary, path);
|
|
1374
|
+
}
|
|
1375
|
+
function atomicJson(path, value) {
|
|
1376
|
+
const temporary = `${path}.${process.pid}.${Date.now()}.tmp`;
|
|
1377
|
+
writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', { flag: 'wx' });
|
|
1378
|
+
renameSync(temporary, path);
|
|
1379
|
+
}
|
|
1380
|
+
function safeInside(root, path, mustExist) {
|
|
1381
|
+
const rootLexical = resolve(root);
|
|
1382
|
+
const rootReal = realpathSync(rootLexical);
|
|
1383
|
+
const requested = resolve(rootLexical, path);
|
|
1384
|
+
if (mustExist) {
|
|
1385
|
+
if (!existsSync(requested))
|
|
1386
|
+
throw new Error(`OpenCode Eval 路径不存在: ${path}`);
|
|
1387
|
+
const candidate = realpathSync(requested);
|
|
1388
|
+
if (!isWorkspacePathInside(rootReal, candidate)) {
|
|
1389
|
+
throw new Error(`OpenCode Eval 路径真实位置越出当前目录: ${path}`);
|
|
1390
|
+
}
|
|
1391
|
+
return candidate;
|
|
1392
|
+
}
|
|
1393
|
+
const lexicalRel = relative(rootLexical, requested);
|
|
1394
|
+
let rel = lexicalRel;
|
|
1395
|
+
if (!relativeIsInside(lexicalRel)) {
|
|
1396
|
+
let ancestor = requested;
|
|
1397
|
+
while (!pathEntryExists(ancestor) && dirname(ancestor) !== ancestor)
|
|
1398
|
+
ancestor = dirname(ancestor);
|
|
1399
|
+
const projected = resolve(realpathSync(ancestor), relative(ancestor, requested));
|
|
1400
|
+
if (!isWorkspacePathInside(rootReal, projected)) {
|
|
1401
|
+
throw new Error(`OpenCode Eval 路径必须位于当前目录内: ${path}`);
|
|
1402
|
+
}
|
|
1403
|
+
rel = relative(rootReal, projected);
|
|
1404
|
+
}
|
|
1405
|
+
const resolved = resolve(rootReal, rel);
|
|
1406
|
+
let cursor = rootReal;
|
|
1407
|
+
for (const segment of rel.split(/[\\/]/).filter(Boolean)) {
|
|
1408
|
+
cursor = join(cursor, segment);
|
|
1409
|
+
if (!pathEntryExists(cursor))
|
|
1410
|
+
break;
|
|
1411
|
+
const info = lstatSync(cursor);
|
|
1412
|
+
if (info.isSymbolicLink()) {
|
|
1413
|
+
throw new Error(`OpenCode Eval 输出路径不允许 symlink: ${relative(rootReal, cursor)}`);
|
|
1414
|
+
}
|
|
1415
|
+
if (cursor !== resolved && !info.isDirectory()) {
|
|
1416
|
+
throw new Error(`OpenCode Eval 输出路径祖先不是目录: ${relative(rootReal, cursor)}`);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
let ancestor = resolved;
|
|
1420
|
+
while (!pathEntryExists(ancestor) && dirname(ancestor) !== ancestor)
|
|
1421
|
+
ancestor = dirname(ancestor);
|
|
1422
|
+
const projected = resolve(realpathSync(ancestor), relative(ancestor, resolved));
|
|
1423
|
+
if (!isWorkspacePathInside(rootReal, projected)) {
|
|
1424
|
+
throw new Error(`OpenCode Eval 输出路径真实位置越出当前目录: ${path}`);
|
|
1425
|
+
}
|
|
1426
|
+
return resolved;
|
|
1427
|
+
}
|
|
1428
|
+
function relativeIsInside(rel) {
|
|
1429
|
+
const separator = process.platform === 'win32' ? '\\' : '/';
|
|
1430
|
+
return rel === '' || (rel !== '..' && !rel.startsWith(`..${separator}`));
|
|
1431
|
+
}
|
|
1432
|
+
function assertCopyableWorkspaceTree(root, excludeControl = true) {
|
|
1433
|
+
const pending = [root];
|
|
1434
|
+
while (pending.length) {
|
|
1435
|
+
const directory = pending.pop();
|
|
1436
|
+
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
1437
|
+
const path = join(directory, entry.name);
|
|
1438
|
+
const rel = relative(root, path);
|
|
1439
|
+
const first = rel.split(/[\\/]/)[0];
|
|
1440
|
+
if (excludeControl && ['.git', '.dingtalk-agent'].includes(first))
|
|
1441
|
+
continue;
|
|
1442
|
+
if (first === '.opencode') {
|
|
1443
|
+
throw new Error('OpenCode Eval workspace 不允许 .opencode 可执行扩展目录');
|
|
1444
|
+
}
|
|
1445
|
+
const info = lstatSync(path);
|
|
1446
|
+
if (info.isSymbolicLink()) {
|
|
1447
|
+
throw new Error(`OpenCode Eval workspace 不允许 symlink: ${rel}`);
|
|
1448
|
+
}
|
|
1449
|
+
if (info.isDirectory())
|
|
1450
|
+
pending.push(path);
|
|
1451
|
+
else if (!info.isFile()) {
|
|
1452
|
+
throw new Error(`OpenCode Eval workspace 不允许特殊文件: ${rel}`);
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
function assertSafeEvalProjectConfig(workspace) {
|
|
1458
|
+
const path = join(workspace, 'opencode.json');
|
|
1459
|
+
if (!existsSync(path))
|
|
1460
|
+
return;
|
|
1461
|
+
const parsed = JSON.parse(readFileSync(path, 'utf8'));
|
|
1462
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
1463
|
+
throw new Error('OpenCode Eval opencode.json 必须是 object');
|
|
1464
|
+
}
|
|
1465
|
+
const config = parsed;
|
|
1466
|
+
const instructions = config.instructions === undefined ? [] : config.instructions;
|
|
1467
|
+
if (!Array.isArray(instructions) || instructions.some((item) => typeof item !== 'string')) {
|
|
1468
|
+
throw new Error('OpenCode Eval instructions 必须是 string array');
|
|
1469
|
+
}
|
|
1470
|
+
for (const instruction of instructions) {
|
|
1471
|
+
const raw = instruction.trim();
|
|
1472
|
+
const candidate = resolve(workspace, raw);
|
|
1473
|
+
if (!raw || /^https?:\/\//i.test(raw) || raw.startsWith('~/') ||
|
|
1474
|
+
raw.startsWith('/') || /^[a-zA-Z]:[\\/]/.test(raw) ||
|
|
1475
|
+
raw.split(/[\\/]/).includes('..') || instructionUsesGlob(raw) ||
|
|
1476
|
+
!isWorkspacePathInside(workspace, candidate) || !existsSync(candidate) ||
|
|
1477
|
+
!lstatSync(candidate).isFile()) {
|
|
1478
|
+
throw new Error(`OpenCode Eval instruction 必须是 sandbox 内已存在的本地相对文件: ${instruction}`);
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
const forbidden = ['provider', 'formatter', 'lsp', 'skills'];
|
|
1482
|
+
for (const key of forbidden) {
|
|
1483
|
+
const value = config[key];
|
|
1484
|
+
if (value !== undefined &&
|
|
1485
|
+
(!value || typeof value !== 'object' || Array.isArray(value) ||
|
|
1486
|
+
Object.keys(value).length > 0)) {
|
|
1487
|
+
throw new Error(`OpenCode Eval 隔离工作区不允许项目级 ${key}`);
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
if (config.plugin !== undefined &&
|
|
1491
|
+
(!Array.isArray(config.plugin) || config.plugin.length > 0)) {
|
|
1492
|
+
throw new Error('OpenCode Eval 隔离工作区不允许 plugin');
|
|
1493
|
+
}
|
|
1494
|
+
if (config.mcp !== undefined &&
|
|
1495
|
+
(!config.mcp || typeof config.mcp !== 'object' || Array.isArray(config.mcp) ||
|
|
1496
|
+
Object.keys(config.mcp).length > 0)) {
|
|
1497
|
+
throw new Error('OpenCode Eval 隔离工作区不允许 MCP');
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
function initializeIsolatedWorktree(workspace) {
|
|
1501
|
+
const command = process.env.DTA_GIT_BIN ||
|
|
1502
|
+
(existsSync('/usr/bin/git') ? '/usr/bin/git' : 'git');
|
|
1503
|
+
const child = spawnSync(command, ['init', '--quiet'], {
|
|
1504
|
+
cwd: workspace,
|
|
1505
|
+
encoding: 'utf8',
|
|
1506
|
+
timeout: 30_000,
|
|
1507
|
+
maxBuffer: 1024 * 1024,
|
|
1508
|
+
});
|
|
1509
|
+
if (child.error || child.status !== 0) {
|
|
1510
|
+
throw new Error(`OpenCode Eval 无法初始化隔离 git worktree: ${child.error?.message || String(child.stderr || child.stdout || '').trim()}`);
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
function pathEntryExists(path) {
|
|
1514
|
+
try {
|
|
1515
|
+
lstatSync(path);
|
|
1516
|
+
return true;
|
|
1517
|
+
}
|
|
1518
|
+
catch {
|
|
1519
|
+
return false;
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
1522
|
+
function extractToolPaths(event, cwd) {
|
|
1523
|
+
const values = [];
|
|
1524
|
+
const candidates = [event.part?.state?.input, event.part?.state?.metadata];
|
|
1525
|
+
for (const candidate of candidates) {
|
|
1526
|
+
if (!candidate || typeof candidate !== 'object')
|
|
1527
|
+
continue;
|
|
1528
|
+
for (const [key, value] of Object.entries(candidate)) {
|
|
1529
|
+
if (!['filePath', 'filepath', 'path', 'directory'].includes(key) || typeof value !== 'string')
|
|
1530
|
+
continue;
|
|
1531
|
+
values.push(value.startsWith('/') || /^[a-zA-Z]:[\\/]/.test(value)
|
|
1532
|
+
? resolve(value) : resolve(cwd, value));
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
return values;
|
|
1536
|
+
}
|
|
1537
|
+
function extractReadWindow(event) {
|
|
1538
|
+
const input = event.part?.state?.input;
|
|
1539
|
+
return {
|
|
1540
|
+
offset: Number.isInteger(input?.offset) ? Number(input.offset) : null,
|
|
1541
|
+
limit: Number.isInteger(input?.limit) ? Number(input.limit) : null,
|
|
1542
|
+
};
|
|
1543
|
+
}
|
|
1544
|
+
function safeName(value) {
|
|
1545
|
+
return value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '') || 'case';
|
|
1546
|
+
}
|
|
1547
|
+
//# sourceMappingURL=opencode-evals.js.map
|