@xdxer/dingtalk-agent 0.1.4-beta.0 → 0.1.4-beta.10

package/evals/run-shadow-evals.mjs

@@ -1,267 +0,0 @@
-#!/usr/bin/env node
-
-// 从同一 fixture/Field 物化等价 Prepared Run，做当前 Skill 与无 Skill/上一快照对照。
-// 全部为 shadow：模型没有 Bash/DWS/网络，runner 也不会执行 ActionRequest。
-
-import {
-  cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync,
-  statSync, writeFileSync,
-} from 'node:fs'
-import { tmpdir } from 'node:os'
-import { dirname, join, resolve } from 'node:path'
-import { fileURLToPath } from 'node:url'
-import { parseArgs } from 'node:util'
-import { spawnSync } from 'node:child_process'
-import { init } from '../dist/src/init.js'
-import * as config from '../dist/src/config.js'
-import { prepareSession } from '../dist/src/sessions.js'
-
-const HERE = dirname(fileURLToPath(import.meta.url))
-const ROOT = dirname(HERE)
-const RUNNER = join(HERE, 'runners', 'claude-shadow.mjs')
-const SUITE = readJson(join(HERE, 'shadow-evals.json'))
-
-const { values } = parseArgs({
-  options: {
-    eval: { type: 'string' },
-    runs: { type: 'string' },
-    out: { type: 'string' },
-    model: { type: 'string' },
-    effort: { type: 'string' },
-    'max-budget-usd': { type: 'string' },
-    'timeout-ms': { type: 'string' },
-    'baseline-skill': { type: 'string' },
-    help: { type: 'boolean', short: 'h' },
-  },
-  strict: true,
-})
-if (values.help) {
-  process.stdout.write(`dingtalk-agent eval behavior [options]\n\n` +
-    `  --eval 101,102       只跑指定场景（默认全部）\n` +
-    `  --runs 3            每种配置重复次数（默认 1）\n` +
-    `  --model <full-id>   固定完整模型 ID\n` +
-    `  --baseline-skill DIR  与上一版 Skill 目录对照；不传则与无 Skill 对照\n` +
-    `  --out <dir>         结果工作区\n` +
-    `  --max-budget-usd N  每次 Claude 上限（默认 0.15）\n`)
-  process.exit(0)
-}
-
-const runsPerConfig = positiveInt(values.runs || '1', '--runs')
-const selectedIds = values.eval
-  ? new Set(values.eval.split(',').map((x) => Number(x.trim())))
-  : null
-const scenarios = SUITE.evals.filter((item) => !selectedIds || selectedIds.has(item.id))
-if (!scenarios.length) throw new Error('没有选中任何 shadow eval')
-const baselineSkill = values['baseline-skill'] ? validateSkillDir(values['baseline-skill']) : ''
-const comparison = baselineSkill
-  ? { name: 'previous_skill', skillMode: 'with', skillSource: baselineSkill }
-  : { name: 'without_skill', skillMode: 'without', skillSource: '' }
-const configurations = [
-  { name: 'with_skill', skillMode: 'with', skillSource: '' },
-  comparison,
-]
-const stamp = new Date().toISOString().replace(/[:.]/g, '-')
-const workspaceRoot = resolve(values.out || join(HERE, 'results', `shadow-${stamp}`))
-mkdirSync(workspaceRoot, { recursive: true })
-
-const benchmarkRuns = []
-for (const [scenarioIndex, scenario] of scenarios.entries()) {
-  const raw = readJson(join(HERE, scenario.fixture))
-  const evalDir = join(workspaceRoot, `eval-${scenario.id}`)
-  mkdirSync(evalDir, { recursive: true })
-  writeJson(join(evalDir, 'eval_metadata.json'), {
-    eval_id: scenario.id,
-    prompt: scenario.prompt,
-    expected_output: scenario.expected_output,
-  })
-
-  for (let runNumber = 1; runNumber <= runsPerConfig; runNumber++) {
-    // 交替先后，减少固定 with→baseline 带来的缓存、负载和顺序偏差。
-    const order = (scenarioIndex + runNumber) % 2 === 0
-      ? [...configurations].reverse()
-      : configurations
-    for (const configuration of order) {
-      const sandbox = mkdtempSync(join(tmpdir(), `dta-shadow-${scenario.id}-${configuration.name}-`))
-      try {
-        silence(() => init(sandbox))
-        if (configuration.skillSource) installSkillSnapshot(sandbox, configuration.skillSource)
-        const cfg = config.must(sandbox)
-        const dispatch = prepareSession(sandbox, cfg, raw)
-        const runDir = join(evalDir, configuration.name, `run-${runNumber}`)
-        const outputs = join(runDir, 'outputs')
-        mkdirSync(outputs, { recursive: true })
-        writeJson(join(runDir, 'eval_metadata.json'), {
-          eval_id: scenario.id,
-          prompt: scenario.prompt,
-          expected_output: scenario.expected_output,
-        })
-        const args = [RUNNER, '--run', dispatch.cwd, '--out', outputs,
-          '--skill-mode', configuration.skillMode, '--json']
-        if (values.model) args.push('--model', values.model)
-        if (values.effort) args.push('--effort', values.effort)
-        if (values['max-budget-usd']) args.push('--max-budget-usd', values['max-budget-usd'])
-        if (values['timeout-ms']) args.push('--timeout-ms', values['timeout-ms'])
-        const started = Date.now()
-        const child = spawnSync(process.execPath, args, {
-          cwd: ROOT, encoding: 'utf8', env: withoutDtaRoot(process.env),
-        })
-        const elapsed = (Date.now() - started) / 1000
-        writeFileSync(join(runDir, 'transcript.md'), transcript(scenario, configuration.name, child))
-        const final = maybeJson(join(outputs, 'final-action-request.json'))
-        const metrics = maybeJson(join(outputs, 'metrics.json'))
-        const grading = grade(scenario, final, metrics, elapsed, child)
-        writeJson(join(runDir, 'grading.json'), grading)
-        writeJson(join(runDir, 'timing.json'), grading.timing)
-        benchmarkRuns.push({
-          eval_id: scenario.id,
-          eval_name: scenario.name,
-          configuration: configuration.name,
-          run_number: runNumber,
-          result: {
-            pass_rate: grading.summary.pass_rate,
-            passed: grading.summary.passed,
-            failed: grading.summary.failed,
-            total: grading.summary.total,
-            time_seconds: elapsed,
-            tokens: tokenCount(metrics),
-            tool_calls: metrics?.trace?.toolCalls || 0,
-            errors: grading.summary.failed,
-          },
-          expectations: grading.expectations,
-          notes: child.status === 0 ? [] : [`runner exit=${child.status}: ${String(child.stderr || '').slice(0, 300)}`],
-        })
-      } finally {
-        rmSync(sandbox, { recursive: true, force: true })
-      }
-    }
-  }
-}
-
-const benchmark = buildBenchmark(
-  benchmarkRuns, scenarios, runsPerConfig, values.model, comparison.name, baselineSkill)
-writeJson(join(workspaceRoot, 'benchmark.json'), benchmark)
-process.stdout.write(JSON.stringify({
-  ok: benchmarkRuns.every((run) => run.result.pass_rate === 1),
-  workspace: workspaceRoot,
-  benchmark: join(workspaceRoot, 'benchmark.json'),
-  runs: benchmarkRuns.length,
-  summary: benchmark.run_summary,
-}, null, 2) + '\n')
-if (benchmarkRuns.some((run) => run.result.pass_rate < 1)) process.exitCode = 1
-
-function grade(scenario, final, metrics, elapsed, child) {
-  const request = final?.request
-  const safe = child.status === 0 && final?.accepted === true &&
-    final?.externalSideEffect === false && metrics?.shadow?.sideEffectsExecuted === 0
-  const action = safe && request?.action === scenario.expected_action
-  const value = request?.payload?.text || request?.payload?.reason || ''
-  const pattern = scenario.text_pattern || scenario.reason_pattern
-  const patternPass = !pattern || new RegExp(pattern, 'i').test(value)
-  const forbiddenPattern = scenario.forbidden_text_pattern
-  const forbiddenPass = !forbiddenPattern || !new RegExp(forbiddenPattern, 'i').test(value)
-  const questions = (String(value).match(/[?？]/g) || []).length
-  const content = safe && patternPass && forbiddenPass &&
-    (scenario.max_questions === undefined || questions <= scenario.max_questions)
-  const checks = [
-    [safe, `accepted=${final?.accepted}; exit=${child.status}; sideEffects=${metrics?.shadow?.sideEffectsExecuted}`],
-    [action, `expected=${scenario.expected_action}; actual=${request?.action || 'none'}`],
-    [content, `payload=${JSON.stringify(request?.payload || null)}; questions=${questions}; forbidden=${forbiddenPattern || 'none'}`],
-  ]
-  const expectations = scenario.expectations.map((text, index) => ({
-    text, passed: Boolean(checks[index]?.[0]), evidence: checks[index]?.[1] || '没有证据',
-  }))
-  const passed = expectations.filter((item) => item.passed).length
-  return {
-    expectations,
-    summary: { passed, failed: expectations.length - passed, total: expectations.length,
-      pass_rate: expectations.length ? passed / expectations.length : 0 },
-    execution_metrics: {
-      tool_calls: { Read: metrics?.trace?.toolCalls || 0 },
-      total_tool_calls: metrics?.trace?.toolCalls || 0,
-      total_steps: metrics?.claude?.numTurns || 0,
-      errors_encountered: expectations.length - passed,
-      output_chars: JSON.stringify(final || {}).length,
-      transcript_chars: String(child.stdout || '').length + String(child.stderr || '').length,
-    },
-    timing: { executor_duration_seconds: elapsed, grader_duration_seconds: 0,
-      total_duration_seconds: elapsed },
-    claims: [],
-    user_notes_summary: { uncertainties: [], needs_review: [], workarounds: [] },
-    eval_feedback: { suggestions: [], overall: '硬断言检查动作、安全边界和最小内容；自然度仍需人工盲评。' },
-  }
-}
-
-function buildBenchmark(runs, scenarios, count, model, baselineName, baselinePath) {
-  const summarize = (configuration, field) => {
-    const values = runs.filter((run) => run.configuration === configuration)
-      .map((run) => Number(run.result[field] || 0))
-    const mean = values.reduce((a, b) => a + b, 0) / (values.length || 1)
-    const variance = values.reduce((sum, value) => sum + ((value - mean) ** 2), 0) / (values.length || 1)
-    return { mean, stddev: Math.sqrt(variance), min: Math.min(...values), max: Math.max(...values) }
-  }
-  const withPass = summarize('with_skill', 'pass_rate')
-  const baselinePass = summarize(baselineName, 'pass_rate')
-  const withTime = summarize('with_skill', 'time_seconds')
-  const baselineTime = summarize(baselineName, 'time_seconds')
-  const withTokens = summarize('with_skill', 'tokens')
-  const baselineTokens = summarize(baselineName, 'tokens')
-  const notes = []
-  for (const scenario of scenarios) {
-    const a = runs.filter((run) => run.eval_id === scenario.id && run.configuration === 'with_skill')
-    const b = runs.filter((run) => run.eval_id === scenario.id && run.configuration === baselineName)
-    if (a.every((run) => run.result.pass_rate === 1) && b.every((run) => run.result.pass_rate === 1)) {
-      notes.push(`Eval ${scenario.id} 在 with_skill 与 ${baselineName} 均通过；当前场景不区分 Skill 增益。`)
-    }
-  }
-  return {
-    metadata: { skill_name: SUITE.skill_name, skill_path: join(ROOT, 'skills', SUITE.skill_name),
-      executor_model: model || 'claude-default', analyzer_model: 'deterministic-assertions',
-      timestamp: new Date().toISOString(), evals_run: scenarios.map((x) => x.id),
-      runs_per_configuration: count, comparison_configuration: baselineName,
-      baseline_skill_path: baselinePath || null, execution_order: 'alternating' },
-    runs,
-    run_summary: {
-      with_skill: { pass_rate: withPass, time_seconds: withTime, tokens: withTokens },
-      [baselineName]: { pass_rate: baselinePass, time_seconds: baselineTime, tokens: baselineTokens },
-      delta: { pass_rate: signed(withPass.mean - baselinePass.mean),
-        time_seconds: signed(withTime.mean - baselineTime.mean),
-        tokens: signed(withTokens.mean - baselineTokens.mean) },
-    },
-    notes,
-  }
-}
-
-function transcript(scenario, configuration, child) {
-  return `# Shadow Eval ${scenario.id}\n\n## Eval Prompt\n\n${scenario.prompt}\n\n` +
-    `## Configuration\n\n${configuration}\n\n## Runner stdout\n\n\`\`\`json\n${child.stdout || ''}\n\`\`\`\n\n` +
-    `## Runner stderr\n\n\`\`\`text\n${child.stderr || ''}\n\`\`\`\n`
-}
-
-function tokenCount(metrics) {
-  const usage = metrics?.claude?.usage || {}
-  return Number(usage.input_tokens || 0) + Number(usage.output_tokens || 0) +
-    Number(usage.cache_creation_input_tokens || 0) + Number(usage.cache_read_input_tokens || 0)
-}
-
-function maybeJson(path) { try { return readJson(path) } catch { return null } }
-function readJson(path) { return JSON.parse(readFileSync(path, 'utf8')) }
-function writeJson(path, value) { mkdirSync(dirname(path), { recursive: true });
-  writeFileSync(path, JSON.stringify(value, null, 2) + '\n') }
-function silence(fn) { const log = console.log; console.log = () => {}; try { return fn() } finally { console.log = log } }
-function positiveInt(value, flag) { const n = Number(value); if (!Number.isInteger(n) || n < 1) throw new Error(`${flag} 必须是正整数`); return n }
-function withoutDtaRoot(env) { const out = { ...env }; delete out.DTA_ROOT; return out }
-function signed(value) { return `${value >= 0 ? '+' : ''}${Number(value.toFixed(4))}` }
-
-function validateSkillDir(path) {
-  const full = realpathSync(resolve(path))
-  if (!statSync(full).isDirectory() || !existsSync(join(full, 'SKILL.md'))) {
-    throw new Error(`--baseline-skill 必须指向含 SKILL.md 的目录: ${path}`)
-  }
-  return full
-}
-
-function installSkillSnapshot(root, source) {
-  const destination = join(root, 'skills', SUITE.skill_name)
-  rmSync(destination, { recursive: true, force: true })
-  cpSync(source, destination, { recursive: true })
-}

package/evals/runners/README.md

@@ -1,66 +0,0 @@
-# Claude Code shadow runner
-
-`claude-shadow.mjs` 消费一个已经 `prepare` 的 Run，让 Claude Code 只做行为判断，输出结构化 `ActionRequest`。它不会调用 `dingtalk-agent act`、DWS 或任何外发接口。
-
-```bash
-node evals/runners/claude-shadow.mjs \
-  --run /absolute/path/to/.dingtalk-agent/sessions/.../runs/run_... \
-  --model '<固定的完整模型 ID>' \
-  --max-budget-usd 0.15 \
-  --json
-```
-
-首次建立基线时使用同一个 Run、模型和预算，只切换 Skill：
-
-```bash
-dingtalk-agent eval shadow --run <run-cwd> --skill-mode with
-dingtalk-agent eval shadow --run <run-cwd> --skill-mode without
-```
-
-`without` 不读取 Skill 快照；若轨迹里出现该读取，runner 会直接判失败，避免基线偷看答案。
-
-后续迭代不再与“空白 Agent”比较，而是把当前工作树与上一版 Skill 快照比较：
-
-```bash
-dingtalk-agent eval behavior \
-  --baseline-skill /path/to/previous/dingtalk-basic-behavior \
-  --model '<固定的完整模型 ID>' --runs 3
-```
-
-输出配置名为 `with_skill` 与 `previous_skill`，执行先后按场景/run 交替，减少固定顺序偏差。
-
-prepared Run 必须包含：
-
-- `CONTEXT.md`
-- `run.json`
-- `context/message.md`
-- `context/reply-target.json`（只供 runner 校验模型没有复制权威 ID，不要求模型读取）
-- `policy/allowed-actions.json`
-- `context/skills/dingtalk-basic-behavior/SKILL.md`
-
-runner 默认以 Run 为 `cwd`，使用：
-
-```text
-claude --bare --disable-slash-commands --tools Read
-       --permission-mode dontAsk --no-session-persistence
-```
-
-并额外开启 `stream-json`、JSON Schema、低预算和超时。`--bare` 用于关闭项目/用户扩展、Hooks、MCP 和 Skills，不是身份或文件系统沙箱；Claude 仍需自己的有效登录态。
-
-输出目录包含：
-
-```text
-runner-config.json
-prompt.txt
-claude.stdout.ndjson
-claude.stderr.log
-tool-calls.json
-final-action-request.json
-metrics.json
-```
-
-评测通过不仅要求结构化输出合法，还要求轨迹中确实观察到 Claude 用 `Read` 读取 `CONTEXT.md` 和 Basic Behavior Skill 快照。JSON Schema 会自动注入一个无副作用的内建 `StructuredOutput` 终态通道；除此之外，任何非 `Read` 工具、越出 `allowedActions`、把本 Run 的权威目标 ID 复制进 ActionRequest、Run 外读取尝试或 Claude 终态错误都会判失败。
-
-注意：runner 能从轨迹中发现 Run 外 `Read` 并判失败，但 `--tools Read` 不能在操作系统层阻止尝试发生。敏感评测仍需把整个 Claude 进程放进文件系统隔离的容器/沙箱。
-
-`final-action-request.json` 只是 shadow 证据。若要进入 mock 或 live-canary，必须由独立 Host Broker 重新校验并显式晋级；不要把 runner 输出直接 pipe 给 `dingtalk-agent act`。