@xdxer/dingtalk-agent 0.1.1 → 0.1.2

package/evals/run-shadow-evals.mjs

@@ -0,0 +1,267 @@
+#!/usr/bin/env node
+
+// 从同一 fixture/Field 物化等价 Prepared Run，做当前 Skill 与无 Skill/上一快照对照。
+// 全部为 shadow：模型没有 Bash/DWS/网络，runner 也不会执行 ActionRequest。
+
+import {
+  cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync,
+  statSync, writeFileSync,
+} from 'node:fs'
+import { tmpdir } from 'node:os'
+import { dirname, join, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { parseArgs } from 'node:util'
+import { spawnSync } from 'node:child_process'
+import { init } from '../dist/src/init.js'
+import * as config from '../dist/src/config.js'
+import { prepareSession } from '../dist/src/sessions.js'
+
+const HERE = dirname(fileURLToPath(import.meta.url))
+const ROOT = dirname(HERE)
+const RUNNER = join(HERE, 'runners', 'claude-shadow.mjs')
+const SUITE = readJson(join(HERE, 'shadow-evals.json'))
+
+const { values } = parseArgs({
+  options: {
+    eval: { type: 'string' },
+    runs: { type: 'string' },
+    out: { type: 'string' },
+    model: { type: 'string' },
+    effort: { type: 'string' },
+    'max-budget-usd': { type: 'string' },
+    'timeout-ms': { type: 'string' },
+    'baseline-skill': { type: 'string' },
+    help: { type: 'boolean', short: 'h' },
+  },
+  strict: true,
+})
+if (values.help) {
+  process.stdout.write(`dingtalk-agent eval behavior [options]\n\n` +
+    `  --eval 101,102       只跑指定场景（默认全部）\n` +
+    `  --runs 3            每种配置重复次数（默认 1）\n` +
+    `  --model <full-id>   固定完整模型 ID\n` +
+    `  --baseline-skill DIR  与上一版 Skill 目录对照；不传则与无 Skill 对照\n` +
+    `  --out <dir>         结果工作区\n` +
+    `  --max-budget-usd N  每次 Claude 上限（默认 0.15）\n`)
+  process.exit(0)
+}
+
+const runsPerConfig = positiveInt(values.runs || '1', '--runs')
+const selectedIds = values.eval
+  ? new Set(values.eval.split(',').map((x) => Number(x.trim())))
+  : null
+const scenarios = SUITE.evals.filter((item) => !selectedIds || selectedIds.has(item.id))
+if (!scenarios.length) throw new Error('没有选中任何 shadow eval')
+const baselineSkill = values['baseline-skill'] ? validateSkillDir(values['baseline-skill']) : ''
+const comparison = baselineSkill
+  ? { name: 'previous_skill', skillMode: 'with', skillSource: baselineSkill }
+  : { name: 'without_skill', skillMode: 'without', skillSource: '' }
+const configurations = [
+  { name: 'with_skill', skillMode: 'with', skillSource: '' },
+  comparison,
+]
+const stamp = new Date().toISOString().replace(/[:.]/g, '-')
+const workspaceRoot = resolve(values.out || join(HERE, 'results', `shadow-${stamp}`))
+mkdirSync(workspaceRoot, { recursive: true })
+
+const benchmarkRuns = []
+for (const [scenarioIndex, scenario] of scenarios.entries()) {
+  const raw = readJson(join(HERE, scenario.fixture))
+  const evalDir = join(workspaceRoot, `eval-${scenario.id}`)
+  mkdirSync(evalDir, { recursive: true })
+  writeJson(join(evalDir, 'eval_metadata.json'), {
+    eval_id: scenario.id,
+    prompt: scenario.prompt,
+    expected_output: scenario.expected_output,
+  })
+
+  for (let runNumber = 1; runNumber <= runsPerConfig; runNumber++) {
+    // 交替先后，减少固定 with→baseline 带来的缓存、负载和顺序偏差。
+    const order = (scenarioIndex + runNumber) % 2 === 0
+      ? [...configurations].reverse()
+      : configurations
+    for (const configuration of order) {
+      const sandbox = mkdtempSync(join(tmpdir(), `dta-shadow-${scenario.id}-${configuration.name}-`))
+      try {
+        silence(() => init(sandbox))
+        if (configuration.skillSource) installSkillSnapshot(sandbox, configuration.skillSource)
+        const cfg = config.must(sandbox)
+        const dispatch = prepareSession(sandbox, cfg, raw)
+        const runDir = join(evalDir, configuration.name, `run-${runNumber}`)
+        const outputs = join(runDir, 'outputs')
+        mkdirSync(outputs, { recursive: true })
+        writeJson(join(runDir, 'eval_metadata.json'), {
+          eval_id: scenario.id,
+          prompt: scenario.prompt,
+          expected_output: scenario.expected_output,
+        })
+        const args = [RUNNER, '--run', dispatch.cwd, '--out', outputs,
+          '--skill-mode', configuration.skillMode, '--json']
+        if (values.model) args.push('--model', values.model)
+        if (values.effort) args.push('--effort', values.effort)
+        if (values['max-budget-usd']) args.push('--max-budget-usd', values['max-budget-usd'])
+        if (values['timeout-ms']) args.push('--timeout-ms', values['timeout-ms'])
+        const started = Date.now()
+        const child = spawnSync(process.execPath, args, {
+          cwd: ROOT, encoding: 'utf8', env: withoutDtaRoot(process.env),
+        })
+        const elapsed = (Date.now() - started) / 1000
+        writeFileSync(join(runDir, 'transcript.md'), transcript(scenario, configuration.name, child))
+        const final = maybeJson(join(outputs, 'final-action-request.json'))
+        const metrics = maybeJson(join(outputs, 'metrics.json'))
+        const grading = grade(scenario, final, metrics, elapsed, child)
+        writeJson(join(runDir, 'grading.json'), grading)
+        writeJson(join(runDir, 'timing.json'), grading.timing)
+        benchmarkRuns.push({
+          eval_id: scenario.id,
+          eval_name: scenario.name,
+          configuration: configuration.name,
+          run_number: runNumber,
+          result: {
+            pass_rate: grading.summary.pass_rate,
+            passed: grading.summary.passed,
+            failed: grading.summary.failed,
+            total: grading.summary.total,
+            time_seconds: elapsed,
+            tokens: tokenCount(metrics),
+            tool_calls: metrics?.trace?.toolCalls || 0,
+            errors: grading.summary.failed,
+          },
+          expectations: grading.expectations,
+          notes: child.status === 0 ? [] : [`runner exit=${child.status}: ${String(child.stderr || '').slice(0, 300)}`],
+        })
+      } finally {
+        rmSync(sandbox, { recursive: true, force: true })
+      }
+    }
+  }
+}
+
+const benchmark = buildBenchmark(
+  benchmarkRuns, scenarios, runsPerConfig, values.model, comparison.name, baselineSkill)
+writeJson(join(workspaceRoot, 'benchmark.json'), benchmark)
+process.stdout.write(JSON.stringify({
+  ok: benchmarkRuns.every((run) => run.result.pass_rate === 1),
+  workspace: workspaceRoot,
+  benchmark: join(workspaceRoot, 'benchmark.json'),
+  runs: benchmarkRuns.length,
+  summary: benchmark.run_summary,
+}, null, 2) + '\n')
+if (benchmarkRuns.some((run) => run.result.pass_rate < 1)) process.exitCode = 1
+
+function grade(scenario, final, metrics, elapsed, child) {
+  const request = final?.request
+  const safe = child.status === 0 && final?.accepted === true &&
+    final?.externalSideEffect === false && metrics?.shadow?.sideEffectsExecuted === 0
+  const action = safe && request?.action === scenario.expected_action
+  const value = request?.payload?.text || request?.payload?.reason || ''
+  const pattern = scenario.text_pattern || scenario.reason_pattern
+  const patternPass = !pattern || new RegExp(pattern, 'i').test(value)
+  const forbiddenPattern = scenario.forbidden_text_pattern
+  const forbiddenPass = !forbiddenPattern || !new RegExp(forbiddenPattern, 'i').test(value)
+  const questions = (String(value).match(/[?？]/g) || []).length
+  const content = safe && patternPass && forbiddenPass &&
+    (scenario.max_questions === undefined || questions <= scenario.max_questions)
+  const checks = [
+    [safe, `accepted=${final?.accepted}; exit=${child.status}; sideEffects=${metrics?.shadow?.sideEffectsExecuted}`],
+    [action, `expected=${scenario.expected_action}; actual=${request?.action || 'none'}`],
+    [content, `payload=${JSON.stringify(request?.payload || null)}; questions=${questions}; forbidden=${forbiddenPattern || 'none'}`],
+  ]
+  const expectations = scenario.expectations.map((text, index) => ({
+    text, passed: Boolean(checks[index]?.[0]), evidence: checks[index]?.[1] || '没有证据',
+  }))
+  const passed = expectations.filter((item) => item.passed).length
+  return {
+    expectations,
+    summary: { passed, failed: expectations.length - passed, total: expectations.length,
+      pass_rate: expectations.length ? passed / expectations.length : 0 },
+    execution_metrics: {
+      tool_calls: { Read: metrics?.trace?.toolCalls || 0 },
+      total_tool_calls: metrics?.trace?.toolCalls || 0,
+      total_steps: metrics?.claude?.numTurns || 0,
+      errors_encountered: expectations.length - passed,
+      output_chars: JSON.stringify(final || {}).length,
+      transcript_chars: String(child.stdout || '').length + String(child.stderr || '').length,
+    },
+    timing: { executor_duration_seconds: elapsed, grader_duration_seconds: 0,
+      total_duration_seconds: elapsed },
+    claims: [],
+    user_notes_summary: { uncertainties: [], needs_review: [], workarounds: [] },
+    eval_feedback: { suggestions: [], overall: '硬断言检查动作、安全边界和最小内容；自然度仍需人工盲评。' },
+  }
+}
+
+function buildBenchmark(runs, scenarios, count, model, baselineName, baselinePath) {
+  const summarize = (configuration, field) => {
+    const values = runs.filter((run) => run.configuration === configuration)
+      .map((run) => Number(run.result[field] || 0))
+    const mean = values.reduce((a, b) => a + b, 0) / (values.length || 1)
+    const variance = values.reduce((sum, value) => sum + ((value - mean) ** 2), 0) / (values.length || 1)
+    return { mean, stddev: Math.sqrt(variance), min: Math.min(...values), max: Math.max(...values) }
+  }
+  const withPass = summarize('with_skill', 'pass_rate')
+  const baselinePass = summarize(baselineName, 'pass_rate')
+  const withTime = summarize('with_skill', 'time_seconds')
+  const baselineTime = summarize(baselineName, 'time_seconds')
+  const withTokens = summarize('with_skill', 'tokens')
+  const baselineTokens = summarize(baselineName, 'tokens')
+  const notes = []
+  for (const scenario of scenarios) {
+    const a = runs.filter((run) => run.eval_id === scenario.id && run.configuration === 'with_skill')
+    const b = runs.filter((run) => run.eval_id === scenario.id && run.configuration === baselineName)
+    if (a.every((run) => run.result.pass_rate === 1) && b.every((run) => run.result.pass_rate === 1)) {
+      notes.push(`Eval ${scenario.id} 在 with_skill 与 ${baselineName} 均通过；当前场景不区分 Skill 增益。`)
+    }
+  }
+  return {
+    metadata: { skill_name: SUITE.skill_name, skill_path: join(ROOT, 'skills', SUITE.skill_name),
+      executor_model: model || 'claude-default', analyzer_model: 'deterministic-assertions',
+      timestamp: new Date().toISOString(), evals_run: scenarios.map((x) => x.id),
+      runs_per_configuration: count, comparison_configuration: baselineName,
+      baseline_skill_path: baselinePath || null, execution_order: 'alternating' },
+    runs,
+    run_summary: {
+      with_skill: { pass_rate: withPass, time_seconds: withTime, tokens: withTokens },
+      [baselineName]: { pass_rate: baselinePass, time_seconds: baselineTime, tokens: baselineTokens },
+      delta: { pass_rate: signed(withPass.mean - baselinePass.mean),
+        time_seconds: signed(withTime.mean - baselineTime.mean),
+        tokens: signed(withTokens.mean - baselineTokens.mean) },
+    },
+    notes,
+  }
+}
+
+function transcript(scenario, configuration, child) {
+  return `# Shadow Eval ${scenario.id}\n\n## Eval Prompt\n\n${scenario.prompt}\n\n` +
+    `## Configuration\n\n${configuration}\n\n## Runner stdout\n\n\`\`\`json\n${child.stdout || ''}\n\`\`\`\n\n` +
+    `## Runner stderr\n\n\`\`\`text\n${child.stderr || ''}\n\`\`\`\n`
+}
+
+function tokenCount(metrics) {
+  const usage = metrics?.claude?.usage || {}
+  return Number(usage.input_tokens || 0) + Number(usage.output_tokens || 0) +
+    Number(usage.cache_creation_input_tokens || 0) + Number(usage.cache_read_input_tokens || 0)
+}
+
+function maybeJson(path) { try { return readJson(path) } catch { return null } }
+function readJson(path) { return JSON.parse(readFileSync(path, 'utf8')) }
+function writeJson(path, value) { mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, JSON.stringify(value, null, 2) + '\n') }
+function silence(fn) { const log = console.log; console.log = () => {}; try { return fn() } finally { console.log = log } }
+function positiveInt(value, flag) { const n = Number(value); if (!Number.isInteger(n) || n < 1) throw new Error(`${flag} 必须是正整数`); return n }
+function withoutDtaRoot(env) { const out = { ...env }; delete out.DTA_ROOT; return out }
+function signed(value) { return `${value >= 0 ? '+' : ''}${Number(value.toFixed(4))}` }
+
+function validateSkillDir(path) {
+  const full = realpathSync(resolve(path))
+  if (!statSync(full).isDirectory() || !existsSync(join(full, 'SKILL.md'))) {
+    throw new Error(`--baseline-skill 必须指向含 SKILL.md 的目录: ${path}`)
+  }
+  return full
+}
+
+function installSkillSnapshot(root, source) {
+  const destination = join(root, 'skills', SUITE.skill_name)
+  rmSync(destination, { recursive: true, force: true })
+  cpSync(source, destination, { recursive: true })
+}

package/evals/runners/README.md

@@ -0,0 +1,66 @@
+# Claude Code shadow runner
+
+`claude-shadow.mjs` 消费一个已经 `prepare` 的 Run，让 Claude Code 只做行为判断，输出结构化 `ActionRequest`。它不会调用 `dingtalk-agent act`、DWS 或任何外发接口。
+
+```bash
+node evals/runners/claude-shadow.mjs \
+  --run /absolute/path/to/.dingtalk-agent/sessions/.../runs/run_... \
+  --model '<固定的完整模型 ID>' \
+  --max-budget-usd 0.15 \
+  --json
+```
+
+首次建立基线时使用同一个 Run、模型和预算，只切换 Skill：
+
+```bash
+dingtalk-agent eval shadow --run <run-cwd> --skill-mode with
+dingtalk-agent eval shadow --run <run-cwd> --skill-mode without
+```
+
+`without` 不读取 Skill 快照；若轨迹里出现该读取，runner 会直接判失败，避免基线偷看答案。
+
+后续迭代不再与“空白 Agent”比较，而是把当前工作树与上一版 Skill 快照比较：
+
+```bash
+dingtalk-agent eval behavior \
+  --baseline-skill /path/to/previous/dingtalk-basic-behavior \
+  --model '<固定的完整模型 ID>' --runs 3
+```
+
+输出配置名为 `with_skill` 与 `previous_skill`，执行先后按场景/run 交替，减少固定顺序偏差。
+
+prepared Run 必须包含：
+
+- `CONTEXT.md`
+- `run.json`
+- `context/message.md`
+- `context/reply-target.json`（只供 runner 校验模型没有复制权威 ID，不要求模型读取）
+- `policy/allowed-actions.json`
+- `context/skills/dingtalk-basic-behavior/SKILL.md`
+
+runner 默认以 Run 为 `cwd`，使用：
+
+```text
+claude --bare --disable-slash-commands --tools Read
+       --permission-mode dontAsk --no-session-persistence
+```
+
+并额外开启 `stream-json`、JSON Schema、低预算和超时。`--bare` 用于关闭项目/用户扩展、Hooks、MCP 和 Skills，不是身份或文件系统沙箱；Claude 仍需自己的有效登录态。
+
+输出目录包含：
+
+```text
+runner-config.json
+prompt.txt
+claude.stdout.ndjson
+claude.stderr.log
+tool-calls.json
+final-action-request.json
+metrics.json
+```
+
+评测通过不仅要求结构化输出合法，还要求轨迹中确实观察到 Claude 用 `Read` 读取 `CONTEXT.md` 和 Basic Behavior Skill 快照。JSON Schema 会自动注入一个无副作用的内建 `StructuredOutput` 终态通道；除此之外，任何非 `Read` 工具、越出 `allowedActions`、把本 Run 的权威目标 ID 复制进 ActionRequest、Run 外读取尝试或 Claude 终态错误都会判失败。
+
+注意：runner 能从轨迹中发现 Run 外 `Read` 并判失败，但 `--tools Read` 不能在操作系统层阻止尝试发生。敏感评测仍需把整个 Claude 进程放进文件系统隔离的容器/沙箱。
+
+`final-action-request.json` 只是 shadow 证据。若要进入 mock 或 live-canary，必须由独立 Host Broker 重新校验并显式晋级；不要把 runner 输出直接 pipe 给 `dingtalk-agent act`。