@xdxer/dingtalk-agent 0.1.4-beta.7 → 0.1.4-beta.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. package/CHANGELOG.md +35 -0
  2. package/README.en.md +395 -0
  3. package/README.md +466 -36
  4. package/dist/bin/dingtalk-agent.js +1071 -340
  5. package/dist/bin/dingtalk-agent.js.map +1 -1
  6. package/dist/src/actions.js +98 -14
  7. package/dist/src/actions.js.map +1 -1
  8. package/dist/src/agent-audit.js +460 -0
  9. package/dist/src/agent-audit.js.map +1 -0
  10. package/dist/src/agent-bindings.js +132 -0
  11. package/dist/src/agent-bindings.js.map +1 -0
  12. package/dist/src/agent-definition.js +182 -0
  13. package/dist/src/agent-definition.js.map +1 -0
  14. package/dist/src/agent-enhance.js +678 -0
  15. package/dist/src/agent-enhance.js.map +1 -0
  16. package/dist/src/bootstrap.js +125 -17
  17. package/dist/src/bootstrap.js.map +1 -1
  18. package/dist/src/development-workspace.js +729 -0
  19. package/dist/src/development-workspace.js.map +1 -0
  20. package/dist/src/dws.js +145 -0
  21. package/dist/src/dws.js.map +1 -1
  22. package/dist/src/eval-evidence.js +193 -0
  23. package/dist/src/eval-evidence.js.map +1 -0
  24. package/dist/src/init.js +1 -1
  25. package/dist/src/init.js.map +1 -1
  26. package/dist/src/invocation.js +36 -0
  27. package/dist/src/invocation.js.map +1 -0
  28. package/dist/src/lab.js +679 -0
  29. package/dist/src/lab.js.map +1 -0
  30. package/dist/src/lease.js +100 -0
  31. package/dist/src/lease.js.map +1 -0
  32. package/dist/src/memory/candidates.js +451 -0
  33. package/dist/src/memory/candidates.js.map +1 -0
  34. package/dist/src/memory/completion-evidence.js +536 -0
  35. package/dist/src/memory/completion-evidence.js.map +1 -0
  36. package/dist/src/memory/operational.js +263 -0
  37. package/dist/src/memory/operational.js.map +1 -0
  38. package/dist/src/memory/remote-state.js +478 -0
  39. package/dist/src/memory/remote-state.js.map +1 -0
  40. package/dist/src/memory/task-checkpoints.js +204 -0
  41. package/dist/src/memory/task-checkpoints.js.map +1 -0
  42. package/dist/src/multica-deploy.js +1480 -0
  43. package/dist/src/multica-deploy.js.map +1 -0
  44. package/dist/src/multica-provider.js +685 -0
  45. package/dist/src/multica-provider.js.map +1 -0
  46. package/dist/src/opencode-evals.js +1062 -0
  47. package/dist/src/opencode-evals.js.map +1 -0
  48. package/dist/src/opencode-provider.js +531 -0
  49. package/dist/src/opencode-provider.js.map +1 -0
  50. package/dist/src/opencode-workspace.js +197 -0
  51. package/dist/src/opencode-workspace.js.map +1 -0
  52. package/dist/src/perception.js +225 -0
  53. package/dist/src/perception.js.map +1 -0
  54. package/dist/src/personal-event-evals.js +595 -0
  55. package/dist/src/personal-event-evals.js.map +1 -0
  56. package/dist/src/promotion.js +786 -0
  57. package/dist/src/promotion.js.map +1 -0
  58. package/dist/src/remote-semantic-state-live-evals.js +888 -0
  59. package/dist/src/remote-semantic-state-live-evals.js.map +1 -0
  60. package/dist/src/remote-semantic-state-worker.js +38 -0
  61. package/dist/src/remote-semantic-state-worker.js.map +1 -0
  62. package/dist/src/remote-state-evals.js +501 -0
  63. package/dist/src/remote-state-evals.js.map +1 -0
  64. package/dist/src/response-gate.js +51 -0
  65. package/dist/src/response-gate.js.map +1 -0
  66. package/dist/src/robot-evals.js +770 -0
  67. package/dist/src/robot-evals.js.map +1 -0
  68. package/dist/src/sessions.js +66 -105
  69. package/dist/src/sessions.js.map +1 -1
  70. package/dist/src/skill-manager.js +25 -16
  71. package/dist/src/skill-manager.js.map +1 -1
  72. package/dist/src/skills.js.map +1 -1
  73. package/dist/src/storage-evals.js +26 -0
  74. package/dist/src/storage-evals.js.map +1 -0
  75. package/dist/src/types.js.map +1 -1
  76. package/dist/src/waits.js +5 -1
  77. package/dist/src/waits.js.map +1 -1
  78. package/dist/src/workspace.js +28 -3
  79. package/dist/src/workspace.js.map +1 -1
  80. package/docs/INSTALLATION.md +47 -0
  81. package/docs/SECOND-AGENT-ACCEPTANCE.md +62 -0
  82. package/docs/architecture/agent-memory-topology.png +0 -0
  83. package/docs/architecture/agent-memory-topology.svg +132 -0
  84. package/docs/architecture/general-agent-kernel-topology.png +0 -0
  85. package/docs/architecture/general-agent-kernel-topology.svg +149 -0
  86. package/docs/architecture/provider-bound-development-workspace.png +0 -0
  87. package/docs/architecture/provider-bound-development-workspace.svg +141 -0
  88. package/docs/architecture/task-completion-gate.png +0 -0
  89. package/docs/architecture/task-completion-gate.svg +191 -0
  90. package/docs/schemas/agent-audit-load-evidence.schema.json +14 -0
  91. package/docs/schemas/agent-audit.schema.json +92 -0
  92. package/docs/schemas/agent-bindings.schema.json +54 -0
  93. package/docs/schemas/agent-definition.schema.json +78 -0
  94. package/docs/schemas/agent-enhancement-plan.schema.json +88 -0
  95. package/docs/schemas/agent-enhancement-receipt.schema.json +37 -0
  96. package/docs/schemas/enriched-invocation.schema.json +46 -0
  97. package/docs/schemas/eval-candidate-plan.schema.json +28 -0
  98. package/docs/schemas/eval-candidate-result.schema.json +20 -0
  99. package/docs/schemas/eval-candidate.schema.json +30 -0
  100. package/docs/schemas/invocation.schema.json +19 -0
  101. package/docs/schemas/memory-candidate-proposal.schema.json +18 -0
  102. package/docs/schemas/memory-candidate.schema.json +76 -0
  103. package/docs/schemas/memory-publish-target.schema.json +25 -0
  104. package/docs/schemas/multica-deployment-list.schema.json +29 -0
  105. package/docs/schemas/multica-deployment-operation.schema.json +51 -0
  106. package/docs/schemas/multica-deployment-plan.schema.json +70 -0
  107. package/docs/schemas/multica-deployment-receipt.schema.json +87 -0
  108. package/docs/schemas/multica-deployment-status.schema.json +23 -0
  109. package/docs/schemas/multica-workspace-inspection.schema.json +77 -0
  110. package/docs/schemas/multica-workspace-plan.schema.json +68 -0
  111. package/docs/schemas/multica-workspace-resource-list.schema.json +27 -0
  112. package/docs/schemas/multica-workspace-status.schema.json +34 -0
  113. package/docs/schemas/observation.schema.json +21 -0
  114. package/docs/schemas/operational-memory-provider.schema.json +36 -0
  115. package/docs/schemas/operational-memory-record.schema.json +24 -0
  116. package/docs/schemas/perception-input.schema.json +56 -0
  117. package/docs/schemas/project.schema.json +112 -0
  118. package/docs/schemas/promotion-list.schema.json +36 -0
  119. package/docs/schemas/promotion-plan.schema.json +60 -0
  120. package/docs/schemas/promotion-policy.schema.json +29 -0
  121. package/docs/schemas/promotion-receipt.schema.json +43 -0
  122. package/docs/schemas/promotion-status.schema.json +23 -0
  123. package/docs/schemas/release-readiness.schema.json +65 -0
  124. package/docs/schemas/remote-semantic-state-live-eval.schema.json +60 -0
  125. package/docs/schemas/remote-semantic-state-manifest.schema.json +79 -0
  126. package/docs/schemas/remote-semantic-state-provider.schema.json +98 -0
  127. package/docs/schemas/response-gate.schema.json +20 -0
  128. package/docs/schemas/task-checkpoint.schema.json +71 -0
  129. package/docs/schemas/task-completion-evidence.schema.json +154 -0
  130. package/docs/schemas/workspace-doctor.schema.json +56 -0
  131. package/docs/schemas/workspace-state.schema.json +39 -0
  132. package/evals/baselines/2026-07-16/opencode-basic-010-completion-summary.json +123 -0
  133. package/evals/baselines/2026-07-16/opencode-basic-skill-required-summary.json +69 -0
  134. package/evals/baselines/2026-07-16/opencode-multi-surface-summary.json +63 -0
  135. package/evals/baselines/2026-07-16/remote-state-live-summary.json +70 -0
  136. package/evals/baselines/2026-07-17/agent-enhance-opencode-dogfood-summary.json +98 -0
  137. package/evals/baselines/2026-07-17/personal-event-live-readiness-summary.json +68 -0
  138. package/examples/agents/fde-coach/AGENTS.md +26 -0
  139. package/examples/agents/fde-coach/MEMORY.md +3 -0
  140. package/examples/agents/fde-coach/fields/default/field.json +24 -0
  141. package/examples/agents/fde-coach/knowledge/INDEX.md +4 -0
  142. package/examples/agents/fde-coach/skills/fde-coach/SKILL.md +13 -0
  143. package/examples/agents/release-manager/AGENTS.md +26 -0
  144. package/examples/agents/release-manager/MEMORY.md +3 -0
  145. package/examples/agents/release-manager/fields/default/field.json +24 -0
  146. package/examples/agents/release-manager/knowledge/INDEX.md +4 -0
  147. package/examples/agents/release-manager/skills/release-manager/SKILL.md +13 -0
  148. package/lab/README.md +109 -0
  149. package/lab/agent-eval/catalog.json +91 -0
  150. package/lab/agent-eval/classic-failures.json +177 -0
  151. package/lab/agent-eval/completion-gate-regression.json +99 -0
  152. package/lab/agent-eval/personal-event-live.example.json +94 -0
  153. package/lab/agent-eval/remote-semantic-state-live.example.json +70 -0
  154. package/lab/agent-eval/remote-semantic-state-provider.fixture.json +47 -0
  155. package/lab/agent-eval/remote-state-workspace/AGENTS.md +8 -0
  156. package/lab/agent-eval/remote-state-workspace/opencode.json +7 -0
  157. package/lab/agent-eval/remote-state-workspace/skills/remote-state-operator/SKILL.md +13 -0
  158. package/lab/agent-eval/remote-state.example.json +31 -0
  159. package/lab/agent-eval/workspace/AGENTS.md +7 -0
  160. package/lab/agent-eval/workspace/MEMORY.md +4 -0
  161. package/lab/agent-eval/workspace/artifacts/pending-review.md +3 -0
  162. package/lab/agent-eval/workspace/knowledge/INDEX.md +4 -0
  163. package/lab/agent-eval/workspace/opencode.json +20 -0
  164. package/lab/manifest.example.json +27 -0
  165. package/lab/manifest.personal-event.example.json +27 -0
  166. package/lab/project-workspace/README.md +11 -0
  167. package/lab/project-workspace/fake-multica-provider.mjs +266 -0
  168. package/lab/project-workspace/multica-deploy.fixture.json +29 -0
  169. package/lab/project-workspace/multica-readonly.fixture.json +69 -0
  170. package/lab/project-workspace/observation.fixture.json +14 -0
  171. package/lab/project-workspace/opencode-provider-suite.json +65 -0
  172. package/lab/project-workspace/project.fixture.json +44 -0
  173. package/lab/project-workspace/promotion-policy.fixture.json +15 -0
  174. package/lab/robot-eval/pool.example.json +30 -0
  175. package/lab/robot-eval/suite.json +123 -0
  176. package/lab/robot-eval/workspace/AGENTS.md +21 -0
  177. package/lab/robot-eval/workspace/MEMORY.md +3 -0
  178. package/lab/robot-eval/workspace/knowledge/INDEX.md +5 -0
  179. package/lab/robot-eval/workspace/opencode.json +22 -0
  180. package/lab/schemas/agent-eval-catalog.schema.json +47 -0
  181. package/lab/schemas/lab-manifest.schema.json +70 -0
  182. package/lab/schemas/personal-event-eval.schema.json +91 -0
  183. package/lab/schemas/remote-state-eval.schema.json +66 -0
  184. package/lab/schemas/robot-eval-suite.schema.json +184 -0
  185. package/lab/schemas/robot-pool.schema.json +56 -0
  186. package/package.json +30 -14
  187. package/skills/dingtalk-agent-boot-multica/SKILL.md +40 -0
  188. package/skills/dingtalk-agent-compose/SKILL.md +110 -0
  189. package/skills/dingtalk-agent-compose/assets/AGENTS.template.md +26 -0
  190. package/skills/dingtalk-agent-compose/assets/agent.bindings.dingtalk-doc.template.json +13 -0
  191. package/skills/dingtalk-agent-compose/assets/agent.bindings.local.template.json +13 -0
  192. package/skills/dingtalk-agent-compose/assets/opencode.template.json +12 -0
  193. package/skills/dingtalk-agent-compose/assets/role-skill.template.md +24 -0
  194. package/skills/dingtalk-agent-compose/evals/evals.json +94 -0
  195. package/skills/dingtalk-agent-compose/references/agent-definition-contract.md +55 -0
  196. package/skills/dingtalk-agent-compose/references/opencode-host-contract.md +65 -0
  197. package/skills/dingtalk-agent-compose/references/storage-routing.md +20 -0
  198. package/skills/dingtalk-agent-deploy/SKILL.md +60 -0
  199. package/skills/dingtalk-agent-deploy/references/multica-deployment-contract.md +49 -0
  200. package/skills/dingtalk-agent-deploy/references/promotion-observation-contract.md +49 -0
  201. package/skills/dingtalk-agent-eval/SKILL.md +116 -0
  202. package/skills/dingtalk-agent-eval/assets/eval-catalog.template.json +18 -0
  203. package/skills/dingtalk-agent-eval/evals/evals.json +61 -0
  204. package/skills/dingtalk-agent-eval/references/eval-topology.md +34 -0
  205. package/skills/dingtalk-agent-eval/references/evidence-contract.md +31 -0
  206. package/skills/dingtalk-agent-eval/references/scenario-taxonomy.md +25 -0
  207. package/skills/dingtalk-agent-eval/references/storage-modes.md +75 -0
  208. package/skills/dingtalk-basic-behavior/SKILL.md +63 -3
  209. package/skills/dingtalk-basic-behavior/assets/memory-candidate-proposal.json +10 -0
  210. package/skills/dingtalk-basic-behavior/assets/{task-checkpoint.md → task-checkpoint.json} +2 -21
  211. package/skills/dingtalk-basic-behavior/references/action-contract.md +2 -0
  212. package/skills/dingtalk-basic-behavior/references/memory-and-evolution.md +15 -1
  213. package/skills/dingtalk-basic-behavior/references/perception-and-gates.md +28 -0
  214. package/skills/dingtalk-basic-behavior/references/task-lifecycle.md +32 -7
  215. package/dist/src/boot.js +0 -70
  216. package/dist/src/boot.js.map +0 -1
  217. package/dist/src/duty.js +0 -74
  218. package/dist/src/duty.js.map +0 -1
  219. package/dist/src/kb.js +0 -240
  220. package/dist/src/kb.js.map +0 -1
  221. package/dist/src/runs.js +0 -79
  222. package/dist/src/runs.js.map +0 -1
  223. package/docs/ARCHITECTURE.md +0 -219
  224. package/docs/MINIMAL-WORKSPACE-V1.md +0 -172
  225. package/docs/OPEN-SOURCE-REFERENCES.md +0 -107
  226. package/docs/SELF-TEST.md +0 -252
  227. package/docs//345/206/205/347/275/221/345/256/236/347/233/270.md +0 -77
  228. package/evals/baselines/2026-07-14/behavior-summary.json +0 -28
  229. package/evals/baselines/2026-07-14/contract-summary.json +0 -18
  230. package/evals/baselines/2026-07-14/live-canary-summary.json +0 -25
  231. package/evals/baselines/2026-07-15/dingtalk-basic-behavior-0.3.0/SKILL.md +0 -72
  232. package/evals/baselines/2026-07-15/dingtalk-basic-behavior-0.3.0/references/action-contract.md +0 -31
  233. package/evals/baselines/2026-07-15/dingtalk-basic-behavior-0.3.0/references/event-to-behavior.md +0 -22
  234. package/evals/baselines/2026-07-15/dingtalk-basic-behavior-0.3.0/references/memory-and-evolution.md +0 -25
  235. package/evals/baselines/2026-07-15/dingtalk-basic-behavior-0.3.0/references/runtime-modes.md +0 -34
  236. package/evals/baselines/2026-07-15/task-lifecycle-summary.json +0 -50
  237. package/evals/evals.json +0 -339
  238. package/evals/fixtures/dm-ambiguous-send.json +0 -4
  239. package/evals/fixtures/dm-blocked.json +0 -4
  240. package/evals/fixtures/dm-clear.json +0 -4
  241. package/evals/fixtures/dm-discussion.json +0 -4
  242. package/evals/fixtures/dm-doc-write-no-tool.json +0 -4
  243. package/evals/fixtures/dm-long-task-ack.json +0 -4
  244. package/evals/fixtures/dm-nonblocking-gap.json +0 -4
  245. package/evals/fixtures/dm-structured-task.json +0 -4
  246. package/evals/fixtures/group.json +0 -10
  247. package/evals/fixtures/mentioned.json +0 -3
  248. package/evals/run-contract-evals.mjs +0 -1169
  249. package/evals/run-shadow-evals.mjs +0 -267
  250. package/evals/runners/README.md +0 -66
  251. package/evals/runners/claude-shadow.mjs +0 -533
  252. package/evals/schemas/action-request.schema.json +0 -77
  253. package/evals/shadow-evals.json +0 -133
  254. package/skills/AGENTS.md +0 -104
  255. package/skills//344/273/273/345/212/241.md +0 -48
  256. package/skills//345/237/272/347/241/200/350/241/214/344/270/272.md +0 -44
  257. package/skills//345/277/203/350/267/263.md +0 -79
  258. package/skills//346/266/210/346/201/257.md +0 -50
  259. package/skills//347/237/245/350/257/206.md +0 -55
  260. package/skills//350/257/204/346/265/213.md +0 -62
  261. package/skills//351/222/211/351/222/211.md +0 -64
  262. package/templates/ontology/index.md +0 -24
  263. package/templates/ontology/self/access.md +0 -38
  264. package/templates/ontology/self/role-spec.md +0 -34
  265. package/templates/ontology/self/workspace.md +0 -41
@@ -1,1169 +0,0 @@
1
- #!/usr/bin/env node
2
-
3
- // 纯本地合同评测:不调用真实 DWS 写能力。
4
- import {
5
- mkdtempSync, readFileSync, writeFileSync, mkdirSync, rmSync, chmodSync,
6
- readdirSync, cpSync, existsSync, lstatSync, readlinkSync,
7
- } from 'node:fs'
8
- import { tmpdir } from 'node:os'
9
- import { join, dirname, resolve } from 'node:path'
10
- import { fileURLToPath } from 'node:url'
11
- import { spawnSync } from 'node:child_process'
12
- import { init } from '../dist/src/init.js'
13
- import * as config from '../dist/src/config.js'
14
- import { ackDispatch, listPendingDispatches, prepareSession, readJson } from '../dist/src/sessions.js'
15
- import { previewAction, executeAction } from '../dist/src/actions.js'
16
- import { heartbeatEvent } from '../dist/src/driver.js'
17
- import { digest, normalizeEvent } from '../dist/src/events.js'
18
- import {
19
- cancelWaitForEvent, claimWait, readWaitForEvent, waitRoot,
20
- } from '../dist/src/waits.js'
21
-
22
- const HERE = dirname(fileURLToPath(import.meta.url))
23
- const ROOT = dirname(HERE)
24
- // 合同结果只拥有自己的子目录,不能清掉同级 Claude/live 评测证据。
25
- const RESULTS = join(HERE, 'results', 'contract')
26
- const EVALS = readJson(join(HERE, 'evals.json')).evals
27
- const CLI = join(ROOT, 'dist', 'bin', 'dingtalk-agent.js')
28
- const WORKSPACES = []
29
- process.on('exit', () => {
30
- for (const root of WORKSPACES) rmSync(root, { recursive: true, force: true })
31
- })
32
-
33
- rmSync(RESULTS, { recursive: true, force: true })
34
- mkdirSync(RESULTS, { recursive: true })
35
-
36
- const cases = [
37
- evalMentioned, evalAmbientGroup, evalDuplicate, evalTamper, evalInboxRecovery,
38
- evalDwsStringData, evalSessionScope, evalSkillSnapshot, evalSingleEgress,
39
- evalHeartbeatOccurrences, evalAskWaitResume, evalWaitScopeIsolation,
40
- evalWaitConflict, evalWaitCancelFence, evalWaitReceiptRepair,
41
- evalClaimSupersededByCancel, evalNegativeWaitRouting,
42
- evalWorkspaceAuthority, evalSessionSkillBinding,
43
- evalGlobalSkillInstall, evalBootstrapLocal, evalBootstrapRemote, evalOptionalInit,
44
- evalManagedSkillLifecycle, evalRemoteStorageTypeGate, evalDirectActGate,
45
- evalDoctorMissingDependencies, evalFirstRunSetup, evalCliErrorHints,
46
- evalSelfUpgrade,
47
- ]
48
- const benchmarkRuns = []
49
- for (let i = 0; i < cases.length; i++) {
50
- const started = Date.now()
51
- const result = cases[i]()
52
- const expectations = EVALS[i].expectations.map((text, n) => ({
53
- text,
54
- passed: Boolean(result.checks[n]?.passed),
55
- evidence: result.checks[n]?.evidence || '没有证据',
56
- }))
57
- const passed = expectations.filter((x) => x.passed).length
58
- const runDir = join(RESULTS, `eval-${i + 1}`, 'with_skill')
59
- mkdirSync(join(runDir, 'outputs'), { recursive: true })
60
- writeJson(join(runDir, 'eval_metadata.json'), {
61
- eval_id: i + 1,
62
- prompt: EVALS[i].prompt,
63
- expected_output: EVALS[i].expected_output,
64
- })
65
- writeJson(join(runDir, 'outputs', 'result.json'), result.output)
66
- writeFileSync(join(runDir, 'outputs', 'summary.md'),
67
- `# Eval ${i + 1}\n\n${EVALS[i].expected_output}\n\n` +
68
- expectations.map((x) => `- ${x.passed ? '✅' : '❌'} ${x.text}: ${x.evidence}`).join('\n') + '\n')
69
- writeJson(join(runDir, 'grading.json'), {
70
- expectations,
71
- summary: { passed, failed: expectations.length - passed,
72
- total: expectations.length, pass_rate: passed / expectations.length },
73
- execution_metrics: { tool_calls: { cli: result.toolCalls }, total_tool_calls: result.toolCalls,
74
- total_steps: result.steps, errors_encountered: expectations.length - passed,
75
- output_chars: JSON.stringify(result.output).length, transcript_chars: 0 },
76
- timing: { executor_duration_seconds: (Date.now() - started) / 1000,
77
- grader_duration_seconds: 0, total_duration_seconds: (Date.now() - started) / 1000 },
78
- })
79
- benchmarkRuns.push({
80
- eval_id: i + 1,
81
- eval_name: `Contract ${i + 1}`,
82
- configuration: 'with_skill',
83
- run_number: 1,
84
- result: { pass_rate: passed / expectations.length, passed,
85
- failed: expectations.length - passed, total: expectations.length,
86
- time_seconds: (Date.now() - started) / 1000, tokens: 0,
87
- tool_calls: result.toolCalls, errors: expectations.length - passed },
88
- expectations,
89
- notes: ['Deterministic runtime contract; no live DingTalk write was executed.'],
90
- })
91
- }
92
-
93
- const mean = benchmarkRuns.reduce((sum, x) => sum + x.result.pass_rate, 0) / benchmarkRuns.length
94
- writeJson(join(RESULTS, 'benchmark.json'), {
95
- metadata: { skill_name: '基础行为', skill_path: ROOT,
96
- executor_model: 'deterministic-cli', analyzer_model: 'deterministic-assertions',
97
- timestamp: new Date().toISOString(), evals_run: EVALS.map((x) => x.id), runs_per_configuration: 1 },
98
- runs: benchmarkRuns,
99
- run_summary: {
100
- with_skill: {
101
- pass_rate: { mean, stddev: 0, min: mean, max: mean },
102
- time_seconds: { mean: 0, stddev: 0, min: 0, max: 0 },
103
- tokens: { mean: 0, stddev: 0, min: 0, max: 0 },
104
- },
105
- delta: { pass_rate: 'n/a', time_seconds: 'n/a', tokens: 'n/a' },
106
- },
107
- notes: ['这是可机器判定的运行时合同评测,不做无 Skill 的主观模型对照。',
108
- '全部案例禁止真实 DWS 写操作。'],
109
- })
110
-
111
- if (process.argv.includes('--smoke')) {
112
- const pass = mean === 1
113
- const mark = pass ? 'PASS' : 'FAIL'
114
- console.log(`${mark} 全局 Skill 可在无 Workspace 时安全安装`)
115
- console.log(`${mark} 本地/钉钉文档 Storage 可在无 init 时水合`)
116
- console.log(`${mark} init 可选、幂等且不污染项目级 Agent 文件`)
117
- console.log(`${mark} Prepared Run 的 Session/Wait/目标/幂等闸门通过`)
118
- if (!pass) process.exitCode = 1
119
- } else {
120
- console.log(JSON.stringify({ results: RESULTS, passRate: mean, evals: benchmarkRuns.length }, null, 2))
121
- }
122
- if (mean !== 1) process.exitCode = 1
123
-
124
- function evalMentioned() {
125
- const { root, cfg } = workspace()
126
- const event = fixture('mentioned.json')
127
- const dispatch = prepareSession(root, cfg, event)
128
- const target = readJson(join(dispatch.cwd, 'context', 'reply-target.json'))
129
- return {
130
- output: { dispatch, target }, toolCalls: 1, steps: 3,
131
- checks: [
132
- check(dispatch.eventKind === 'im.message.mentioned', `eventKind=${dispatch.eventKind}`),
133
- check(sameSet(dispatch.allowedActions, ['ack', 'reply', 'ask', 'silence']),
134
- `allowed=${dispatch.allowedActions.join(',')}`),
135
- check(target.conversationId === 'cid-project-a' && target.messageId === 'mid-mentioned-001' &&
136
- target.senderOpenDingTalkId === '$:LWCP_v1:$alice', JSON.stringify(target)),
137
- ],
138
- }
139
- }
140
-
141
- function evalAmbientGroup() {
142
- const { root, cfg } = workspace()
143
- const dispatch = prepareSession(root, cfg, fixture('group.json'))
144
- let blocked = ''
145
- try { previewAction(dispatch.cwd, 'reply', { text: '不应该发送' }) }
146
- catch (e) { blocked = e.message }
147
- const receipt = executeAction(dispatch.cwd, 'silence', { reason: 'unmentioned' })
148
- return {
149
- output: { dispatch, blocked, receipt }, toolCalls: 3, steps: 4,
150
- checks: [
151
- check(sameSet(dispatch.allowedActions, ['silence']), `allowed=${dispatch.allowedActions.join(',')}`),
152
- check(blocked.includes('不允许 reply'), blocked),
153
- check(receipt.kind === 'silence' && receipt.verified === true, JSON.stringify(receipt)),
154
- ],
155
- }
156
- }
157
-
158
- function evalDuplicate() {
159
- const { root, cfg } = workspace()
160
- const event = fixture('mentioned.json')
161
- const first = prepareSession(root, cfg, event)
162
- const second = prepareSession(root, cfg, event)
163
- const pending = listPendingDispatches(root, cfg)
164
- const ack = ackDispatch(root, cfg, first.eventId, first.runId, 'eval-controller')
165
- const third = prepareSession(root, cfg, event)
166
- return {
167
- output: { first, second, pending, ack, third }, toolCalls: 5, steps: 5,
168
- checks: [
169
- check(first.sessionId === second.sessionId && first.runId === second.runId &&
170
- pending.length === 1 && pending[0].runId === first.runId,
171
- `${first.sessionId}/${first.runId}; pending=${pending.length}`),
172
- check(second.duplicate === true && second.launch === true,
173
- `duplicate=${second.duplicate}, launch=${second.launch}`),
174
- check(ack.duplicate === false && third.duplicate === true && third.launch === false,
175
- `ack=${ack.duplicate}, duplicate=${third.duplicate}, launch=${third.launch}`),
176
- ],
177
- }
178
- }
179
-
180
- function evalTamper() {
181
- const { root, cfg } = workspace()
182
- const dispatch = prepareSession(root, cfg, fixture('mentioned.json'))
183
- const override = spawnSync(process.execPath, [CLI, 'act', 'reply', '--text', 'x',
184
- '--conversation-id', 'evil'], { cwd: dispatch.cwd, encoding: 'utf8' })
185
-
186
- const fake = join(root, 'fake-run')
187
- mkdirSync(fake)
188
- writeFileSync(join(fake, 'run.json'), readFileSync(join(dispatch.cwd, 'run.json')))
189
- let forged = ''
190
- try { previewAction(fake, 'silence') } catch (e) { forged = e.message }
191
-
192
- const targetPath = join(dispatch.cwd, 'context', 'reply-target.json')
193
- writeJson(targetPath, { conversationId: 'evil', messageId: 'evil', senderOpenDingTalkId: 'evil' })
194
- let drift = ''
195
- try { previewAction(dispatch.cwd, 'silence') } catch (e) { drift = e.message }
196
- return {
197
- output: { overrideStatus: override.status, overrideError: override.stderr.trim(), forged, drift },
198
- toolCalls: 4, steps: 5,
199
- checks: [
200
- check(override.status !== 0 && override.stderr.includes('Unknown option'), override.stderr.trim()),
201
- check(forged.includes('身份不一致'), forged),
202
- check(drift.includes('投影被修改'), drift),
203
- ],
204
- }
205
- }
206
-
207
- function evalInboxRecovery() {
208
- const { root, cfg } = workspace()
209
- const event = fixture('mentioned.json')
210
- const fieldPath = join(root, 'fields', 'default', 'field.json')
211
- const field = readJson(fieldPath)
212
- field.behavior.packs = ['behaviors/missing.json']
213
- writeJson(fieldPath, field)
214
- let failure = ''
215
- try { prepareSession(root, cfg, event) } catch (e) { failure = e.message }
216
- const inboxRoot = join(root, cfg.runtime.eventIndexRoot, 'inbox')
217
- const retained = readdirSync(inboxRoot).filter((x) => x.endsWith('.json')).length
218
-
219
- field.behavior.packs = ['behaviors/basic.json']
220
- writeJson(fieldPath, field)
221
- const pending = listPendingDispatches(root, cfg)
222
- const recovered = pending.find((x) => x.type === 'dingtalk-agent.run.redelivery')
223
- const remaining = readdirSync(inboxRoot).filter((x) => x.endsWith('.json')).length
224
- return {
225
- output: { failure, retained, pending, remaining }, toolCalls: 3, steps: 5,
226
- checks: [
227
- check(failure.includes('不存在') && retained === 1,
228
- `failure=${failure}; retained=${retained}`),
229
- check(Boolean(recovered?.launch && recovered?.runId),
230
- `type=${recovered?.type}; launch=${recovered?.launch}; run=${recovered?.runId}`),
231
- check(remaining === 0, `remaining=${remaining}`),
232
- ],
233
- }
234
- }
235
-
236
- function evalDwsStringData() {
237
- const { root, cfg } = workspace()
238
- const raw = {
239
- type: 'event',
240
- event_corp_id: 'tenant-a',
241
- data: JSON.stringify({
242
- type: 'user_im_message_receive_o2o', event_id: 'evt-dws-string-1',
243
- conversation_id: 'cid-dws-dm', message_id: 'mid-dws-dm',
244
- sender_open_dingtalk_id: '$:LWCP_v1:$peer', content: '你好',
245
- }),
246
- }
247
- const dispatch = prepareSession(root, cfg, raw)
248
- const normalized = readJson(join(dispatch.cwd, 'event.json'))
249
- let failure = ''
250
- try { prepareSession(root, cfg, { type: 'event', data: '{bad json' }) }
251
- catch (e) { failure = e.message }
252
- return {
253
- output: { dispatch, normalized, failure }, toolCalls: 2, steps: 3,
254
- checks: [
255
- check(normalized.kind === 'im.message.dm' && normalized.message.text === '你好',
256
- `${normalized.kind}/${normalized.message.text}`),
257
- check(normalized.tenant.id === 'tenant-a' &&
258
- normalized.replyTarget.conversationId === 'cid-dws-dm' &&
259
- normalized.replyTarget.messageId === 'mid-dws-dm' &&
260
- normalized.replyTarget.senderOpenDingTalkId === '$:LWCP_v1:$peer',
261
- JSON.stringify(normalized.replyTarget)),
262
- check(failure.includes('data 不是合法 JSON 字符串'), failure),
263
- ],
264
- }
265
- }
266
-
267
- function evalSessionScope() {
268
- const { root, cfg } = workspace()
269
- const base = fixture('mentioned.json')
270
- const body = JSON.parse(base.data)
271
- const messageEvent = (eventId, tenantId, messageId, extra = {}) => ({
272
- event_corp_id: tenantId,
273
- data: JSON.stringify({ ...body, event_id: eventId, message_id: messageId, ...extra }),
274
- })
275
- const firstEvent = messageEvent('evt-topic-1', 'tenant-a', 'mid-topic-1')
276
- const secondEvent = messageEvent('evt-topic-2', 'tenant-a', 'mid-topic-2')
277
- const followupEvent = messageEvent('evt-topic-3', 'tenant-a', 'mid-topic-3',
278
- { root_message_id: 'mid-topic-1' })
279
- const otherTenantEvent = messageEvent('evt-topic-4', 'tenant-b', 'mid-topic-1')
280
- const first = prepareSession(root, cfg, firstEvent)
281
- const second = prepareSession(root, cfg, secondEvent)
282
- const followup = prepareSession(root, cfg, followupEvent)
283
- const otherTenant = prepareSession(root, cfg, otherTenantEvent)
284
- return {
285
- output: { first, second, followup, otherTenant }, toolCalls: 4, steps: 4,
286
- checks: [
287
- check(first.sessionId !== second.sessionId, `${first.sessionId} != ${second.sessionId}`),
288
- check(first.sessionId === followup.sessionId && first.runId !== followup.runId,
289
- `${first.sessionId}/${first.runId} -> ${followup.sessionId}/${followup.runId}`),
290
- check(first.sessionId !== otherTenant.sessionId,
291
- `${first.sessionId} != ${otherTenant.sessionId}`),
292
- ],
293
- }
294
- }
295
-
296
- function evalSkillSnapshot() {
297
- const { root, cfg } = workspace()
298
- installWorkspaceSkillOverride(root)
299
- const event = fixture('mentioned.json')
300
- const first = prepareSession(root, cfg, event)
301
- const manifestPath = join(first.cwd, 'context', 'skills', 'manifest.json')
302
- const skillPath = join(first.cwd, 'context', 'skills', 'dingtalk-basic-behavior', 'SKILL.md')
303
- const manifest = readJson(manifestPath)
304
- const before = readFileSync(skillPath, 'utf8')
305
- const source = join(root, 'skills', 'dingtalk-basic-behavior', 'SKILL.md')
306
- writeFileSync(source, readFileSync(source, 'utf8') + '\n<!-- candidate change -->\n')
307
- const replay = prepareSession(root, cfg, event)
308
- const after = readFileSync(skillPath, 'utf8')
309
- const run = readJson(join(first.cwd, 'run.json'))
310
- writeFileSync(skillPath, after + '\n<!-- tampered in sandbox -->\n')
311
- let tamperBlocked = ''
312
- try { previewAction(first.cwd, 'silence', { reason: 'no_value' }) }
313
- catch (e) { tamperBlocked = e.message }
314
- return {
315
- output: { first, replay, manifest, run, tamperBlocked }, toolCalls: 5, steps: 6,
316
- checks: [
317
- check(manifest.entries?.[0]?.name === 'dingtalk-basic-behavior' &&
318
- before.includes('name: dingtalk-basic-behavior'), JSON.stringify(manifest.entries?.[0])),
319
- check(run.skills?.manifestHash === manifest.hash, `${run.skills?.manifestHash}/${manifest.hash}`),
320
- check(before === after && replay.runId === first.runId &&
321
- tamperBlocked.includes('Skill snapshot 内容漂移'),
322
- `snapshotSame=${before === after}; run=${replay.runId}; tamper=${tamperBlocked}`),
323
- ],
324
- }
325
- }
326
-
327
- function evalSingleEgress() {
328
- const { root, cfg } = workspace()
329
- const fieldPath = join(root, 'fields', 'default', 'field.json')
330
- const field = readJson(fieldPath)
331
- field.transport = { mode: 'robot-connect', egressOwner: 'connector' }
332
- writeJson(fieldPath, field)
333
- const dispatch = prepareSession(root, cfg, fixture('mentioned.json'))
334
- const fieldSnapshot = readJson(join(dispatch.cwd, 'context', 'field.json'))
335
- let blocked = ''
336
- try { previewAction(dispatch.cwd, 'reply', { text: '不应由 act 发送' }) }
337
- catch (e) { blocked = e.message }
338
- const receipt = executeAction(dispatch.cwd, 'silence', { reason: 'policy_denied' })
339
- return {
340
- output: { dispatch, fieldSnapshot, blocked, receipt }, toolCalls: 3, steps: 4,
341
- checks: [
342
- check(fieldSnapshot.transport?.egressOwner === 'connector',
343
- JSON.stringify(fieldSnapshot.transport)),
344
- check(blocked.includes('出口属于 connector'), blocked),
345
- check(receipt.kind === 'silence' && receipt.verified === true, JSON.stringify(receipt)),
346
- ],
347
- }
348
- }
349
-
350
- function evalHeartbeatOccurrences() {
351
- const { root, cfg } = workspace()
352
- const firstEvent = heartbeatEvent('duty:daily-review', 'beat-001')
353
- const secondEvent = heartbeatEvent('duty:daily-review', 'beat-002')
354
- const first = prepareSession(root, cfg, firstEvent)
355
- const retry = prepareSession(root, cfg, firstEvent)
356
- ackDispatch(root, cfg, first.eventId, first.runId, 'eval-controller')
357
- const second = prepareSession(root, cfg, secondEvent)
358
- return {
359
- output: { firstEvent, secondEvent, first, retry, second }, toolCalls: 4, steps: 5,
360
- checks: [
361
- check(first.sessionId === second.sessionId,
362
- `${first.sessionId} == ${second.sessionId}`),
363
- check(first.runId !== second.runId && second.launch === true,
364
- `${first.runId} != ${second.runId}; launch=${second.launch}`),
365
- check(retry.runId === first.runId && retry.duplicate === true,
366
- `retry=${retry.runId}; duplicate=${retry.duplicate}`),
367
- ],
368
- }
369
- }
370
-
371
- function evalAskWaitResume() {
372
- const { root, cfg } = workspace()
373
- enableFakeDws(root)
374
- const firstEvent = dmEvent('evt-wait-ask-1', 'mid-wait-ask-1', 'alice', '请先给我一个草稿')
375
- const first = prepareSession(root, cfg, firstEvent)
376
- const receipt = withFakeDws(root, () =>
377
- executeAction(first.cwd, 'ask', { text: '草稿需要偏正式还是偏口语?' }))
378
- const active = readWaitForEvent(root, cfg, first.fieldId, readJson(join(first.cwd, 'event.json')))
379
-
380
- const replyEvent = dmEvent('evt-wait-reply-1', 'mid-wait-reply-1', 'alice', '偏正式')
381
- const resumed = prepareSession(root, cfg, replyEvent)
382
- const resume = readJson(join(resumed.cwd, 'context', 'resume.json'))
383
- const claimed = readWaitForEvent(root, cfg, resumed.fieldId, readJson(join(resumed.cwd, 'event.json')))
384
- const secondReceipt = withFakeDws(root, () =>
385
- executeAction(resumed.cwd, 'ask', { text: '是否还需要附上数据来源?' }))
386
- const secondActive = readWaitForEvent(root, cfg, resumed.fieldId, readJson(join(resumed.cwd, 'event.json')))
387
- const replay = withFakeDws(root, () =>
388
- executeAction(first.cwd, 'ask', { text: '草稿需要偏正式还是偏口语?' }))
389
- const afterReplay = readWaitForEvent(root, cfg, resumed.fieldId, readJson(join(resumed.cwd, 'event.json')))
390
- return {
391
- output: { first, receipt, active, resumed, resume, claimed,
392
- secondReceipt, secondActive, replay, afterReplay }, toolCalls: 7, steps: 9,
393
- checks: [
394
- check(receipt.kind === 'ask' && active?.status === 'active' &&
395
- active?.sourceRunId === first.runId,
396
- `receipt=${receipt.outcome}; wait=${active?.id}/${active?.status}`),
397
- check(resumed.sessionId === first.sessionId && resumed.runId !== first.runId &&
398
- resumed.mode === 'resume',
399
- `${first.sessionId}/${first.runId} -> ${resumed.sessionId}/${resumed.runId}`),
400
- check(resume.waitId === active?.id && resume.sourceRunId === first.runId &&
401
- resume.questionText === '草稿需要偏正式还是偏口语?' &&
402
- claimed?.status === 'claimed' && claimed?.claim?.runId === resumed.runId,
403
- JSON.stringify({ resume, status: claimed?.status, claim: claimed?.claim })),
404
- check(replay.duplicate === true && secondActive?.id !== active?.id &&
405
- afterReplay?.id === secondActive?.id && afterReplay?.status === 'active',
406
- `old=${active?.id}; current=${secondActive?.id}; afterReplay=${afterReplay?.id}`),
407
- ],
408
- }
409
- }
410
-
411
- function evalWaitScopeIsolation() {
412
- const { root, cfg } = workspace()
413
- enableFakeDws(root)
414
- const firstEvent = dmEvent('evt-wait-scope-1', 'mid-wait-scope-1', 'alice', '帮我准备材料')
415
- const first = prepareSession(root, cfg, firstEvent)
416
- withFakeDws(root, () => executeAction(first.cwd, 'ask', { text: '材料用于哪个会议?' }))
417
-
418
- const otherActor = prepareSession(root, cfg,
419
- dmEvent('evt-wait-scope-2', 'mid-wait-scope-2', 'bob', '用于周会'))
420
- const explicit = prepareSession(root, cfg,
421
- dmEvent('evt-wait-scope-3', 'mid-wait-scope-3', 'alice', '另开一个事项'),
422
- { sessionKey: 'explicit-other-work' })
423
- const resumed = prepareSession(root, cfg,
424
- dmEvent('evt-wait-scope-4', 'mid-wait-scope-4', 'alice', '用于季度复盘会'))
425
- return {
426
- output: { first, otherActor, explicit, resumed }, toolCalls: 5, steps: 6,
427
- checks: [
428
- check(otherActor.sessionId !== first.sessionId && otherActor.mode === 'start',
429
- `other=${otherActor.sessionId}/${otherActor.mode}`),
430
- check(explicit.sessionId !== first.sessionId && explicit.mode === 'start',
431
- `explicit=${explicit.sessionId}/${explicit.mode}`),
432
- check(resumed.sessionId === first.sessionId && resumed.mode === 'resume',
433
- `resumed=${resumed.sessionId}/${resumed.mode}`),
434
- ],
435
- }
436
- }
437
-
438
- function evalWaitConflict() {
439
- const { root, cfg } = workspace()
440
- enableFakeDws(root)
441
- const firstEvent = dmEvent('evt-wait-conflict-1', 'mid-wait-conflict-1', 'alice', '事项一')
442
- const first = prepareSession(root, cfg, firstEvent)
443
- withFakeDws(root, () => executeAction(first.cwd, 'ask', { text: '事项一缺哪个日期?' }))
444
- const resumed = prepareSession(root, cfg,
445
- dmEvent('evt-wait-conflict-2', 'mid-wait-conflict-2', 'alice', '下周一'))
446
- const claimed = readWaitForEvent(
447
- root, cfg, first.fieldId, readJson(join(resumed.cwd, 'event.json')))
448
- const second = prepareSession(root, cfg,
449
- dmEvent('evt-wait-conflict-3', 'mid-wait-conflict-3', 'alice', '事项二'),
450
- { sessionKey: 'parallel-work' })
451
- let blocked = ''
452
- try {
453
- withFakeDws(root, () => executeAction(second.cwd, 'ask', { text: '事项二缺哪个负责人?' }))
454
- } catch (e) { blocked = e.message }
455
- const current = readWaitForEvent(root, cfg, first.fieldId, readJson(join(first.cwd, 'event.json')))
456
- return {
457
- output: { first, resumed, claimed, second, blocked, current }, toolCalls: 5, steps: 6,
458
- checks: [
459
- check(first.sessionId !== second.sessionId && resumed.sessionId === first.sessionId,
460
- `${first.sessionId} != ${second.sessionId}; resumed=${resumed.sessionId}`),
461
- check(blocked.includes('单链 v1 拒绝并行追问'), blocked),
462
- check(claimed?.status === 'claimed' && current?.sessionId === first.sessionId &&
463
- current?.status === 'claimed',
464
- `${current?.sessionId}/${current?.status}`),
465
- ],
466
- }
467
- }
468
-
469
- function evalWaitCancelFence() {
470
- const { root, cfg } = workspace()
471
- enableFakeDws(root)
472
- const firstEvent = dmEvent('evt-wait-cancel-1', 'mid-wait-cancel-1', 'alice', '请准备发布稿')
473
- const first = prepareSession(root, cfg, firstEvent)
474
- withFakeDws(root, () => executeAction(first.cwd, 'ask', { text: '确认按当前版本发布吗?' }))
475
-
476
- // 下一条消息已 claim 并物化成 resumed Run,但尚未执行;/stop 仍必须能截断它。
477
- const resumed = prepareSession(root, cfg,
478
- dmEvent('evt-wait-cancel-2', 'mid-wait-cancel-2', 'alice', '确认'))
479
- const beforeStop = readWaitForEvent(
480
- root, cfg, first.fieldId, readJson(join(resumed.cwd, 'event.json')))
481
-
482
- // 故意破坏当前 Behavior 引用,证明取消闸门先于 Field/Behavior/Skill 物化执行。
483
- const fieldPath = join(root, 'fields', 'default', 'field.json')
484
- const field = readJson(fieldPath)
485
- writeJson(fieldPath, {
486
- ...field,
487
- behavior: { ...(field.behavior || {}), packs: ['behaviors/missing.json'] },
488
- })
489
- const stopEvent = dmEvent('evt-wait-cancel-3', 'mid-wait-cancel-3', 'alice', '/stop')
490
- let interrupted = ''
491
- try { prepareSession(root, cfg, stopEvent) }
492
- catch (e) { interrupted = e.message }
493
- const fencedBeforeMaterialize = readJson(join(first.sessionPath, 'session.json'))
494
- writeJson(fieldPath, field)
495
-
496
- // 旧 stop 尚未落 event-index 时,新事项已经在同一 scope 建立 Wait。
497
- // 重放旧 stop 必须按 eventId ledger 找回旧 Wait,不能误取消这个新 head。
498
- const newer = prepareSession(root, cfg,
499
- dmEvent('evt-wait-cancel-4', 'mid-wait-cancel-4', 'alice', '另开一个发布事项'),
500
- { sessionKey: 'post-cancel-new-work' })
501
- withFakeDws(root, () => executeAction(
502
- newer.cwd, 'ask', { text: '新事项按哪个版本?' }))
503
- const newActiveBeforeReplay = readWaitForEvent(
504
- root, cfg, newer.fieldId, readJson(join(newer.cwd, 'event.json')))
505
- const cancelled = prepareSession(root, cfg, stopEvent)
506
- const duplicate = prepareSession(root, cfg, stopEvent)
507
- const session = readJson(join(first.sessionPath, 'session.json'))
508
- const waitAfterReplay = readWaitForEvent(
509
- root, cfg, newer.fieldId, readJson(join(newer.cwd, 'event.json')))
510
- let fenced = ''
511
- try { previewAction(resumed.cwd, 'reply', { text: '已唤醒 Run 不应再发送' }) }
512
- catch (e) { fenced = e.message }
513
- return {
514
- output: { first, resumed, beforeStop, interrupted, fencedBeforeMaterialize,
515
- newer, newActiveBeforeReplay, cancelled, duplicate, session, waitAfterReplay,
516
- fenced }, toolCalls: 10, steps: 13,
517
- checks: [
518
- check(beforeStop?.status === 'claimed' && cancelled.sessionId === first.sessionId &&
519
- cancelled.mode === 'cancel' &&
520
- newActiveBeforeReplay?.sessionId === newer.sessionId &&
521
- waitAfterReplay?.id === newActiveBeforeReplay?.id &&
522
- waitAfterReplay?.status === 'active' &&
523
- sameSet(cancelled.allowedActions, ['reply', 'silence']),
524
- `before=${beforeStop?.status}; cancel=${cancelled.sessionId}/${cancelled.mode}; ` +
525
- `new=${waitAfterReplay?.id}/${waitAfterReplay?.status}`),
526
- check(interrupted.includes('missing.json') && fencedBeforeMaterialize.generation === 2 &&
527
- session.generation === 2 && duplicate.runId === cancelled.runId,
528
- `interrupted=${interrupted}; generations=${fencedBeforeMaterialize.generation}/${session.generation}`),
529
- check(fenced.includes('Session generation 已失效'), fenced),
530
- ],
531
- }
532
- }
533
-
534
- function evalWaitReceiptRepair() {
535
- const { root, cfg } = workspace()
536
- enableFakeDws(root)
537
- const firstEvent = dmEvent('evt-wait-repair-1', 'mid-wait-repair-1', 'alice', '帮我写总结')
538
- const first = prepareSession(root, cfg, firstEvent)
539
- withFakeDws(root, () => executeAction(first.cwd, 'ask', { text: '总结覆盖哪个时间段?' }))
540
- const event = readJson(join(first.cwd, 'event.json'))
541
- const active = readWaitForEvent(root, cfg, first.fieldId, event)
542
-
543
- // 模拟进程在 Receipt 落盘后、Wait 状态更新前崩溃。
544
- const crashed = {
545
- ...active,
546
- status: 'arming',
547
- delivery: null,
548
- lastAppliedReceiptActionId: undefined,
549
- }
550
- const authorityRoot = waitRoot(root, cfg)
551
- writeJson(join(authorityRoot, `${active.scopeHash}.json`), crashed)
552
- writeJson(join(authorityRoot, 'history', `${active.id}.json`), crashed)
553
-
554
- const resumed = prepareSession(root, cfg,
555
- dmEvent('evt-wait-repair-2', 'mid-wait-repair-2', 'alice', '本周'))
556
- const repaired = readWaitForEvent(
557
- root, cfg, resumed.fieldId, readJson(join(resumed.cwd, 'event.json')))
558
- const resume = readJson(join(resumed.cwd, 'context', 'resume.json'))
559
- return {
560
- output: { first, active, crashed, resumed, repaired, resume }, toolCalls: 4, steps: 6,
561
- checks: [
562
- check(active?.status === 'active' && crashed.status === 'arming',
563
- `${active?.status} -> ${crashed.status}`),
564
- check(resumed.sessionId === first.sessionId && resumed.mode === 'resume' &&
565
- repaired?.status === 'claimed',
566
- `${resumed.sessionId}/${resumed.mode}; wait=${repaired?.status}`),
567
- check(resume.waitId === active.id && resume.questionText === '总结覆盖哪个时间段?' &&
568
- repaired?.claim?.runId === resumed.runId,
569
- JSON.stringify({ resume, claim: repaired?.claim })),
570
- ],
571
- }
572
- }
573
-
574
- function evalClaimSupersededByCancel() {
575
- const { root, cfg } = workspace()
576
- enableFakeDws(root)
577
- const firstEvent = dmEvent('evt-claim-race-1', 'mid-claim-race-1', 'alice', '帮我发布评价')
578
- const first = prepareSession(root, cfg, firstEvent)
579
- withFakeDws(root, () => executeAction(
580
- first.cwd, 'ask', { text: '确认使用当前评价吗?' }))
581
-
582
- // 精确构造:回复 E 已绑定并 claim Wait,但尚未来得及 materialize Run;/stop 抢先取消。
583
- const replyRaw = dmEvent('evt-claim-race-2', 'mid-claim-race-2', 'alice', '确认')
584
- const replyEvent = normalizeEvent(replyRaw)
585
- const claimed = claimWait(root, cfg, first.fieldId, replyEvent)
586
- // 模拟 claimed head 已落盘、event ledger 尚未落盘时 SIGKILL。
587
- rmSync(join(waitRoot(root, cfg), 'events', `${digest(replyEvent.id)}.json`), { force: true })
588
- const stopRaw = dmEvent('evt-claim-race-3', 'mid-claim-race-3', 'alice', '/stop')
589
- const stopEvent = normalizeEvent(stopRaw)
590
- const cancelled = cancelWaitForEvent(root, cfg, stopEvent)
591
-
592
- // E 重放只能回到原 Session 的安全 no-op Run,绝不能 fallback 成新事项。
593
- const suppressed = prepareSession(root, cfg, replyRaw)
594
- const resume = readJson(join(suppressed.cwd, 'context', 'resume.json'))
595
- let replyBlocked = ''
596
- try { previewAction(suppressed.cwd, 'reply', { text: '不应发送' }) }
597
- catch (e) { replyBlocked = e.message }
598
- const receipt = executeAction(suppressed.cwd, 'silence', { reason: 'policy_denied' })
599
- const session = readJson(join(first.sessionPath, 'session.json'))
600
- return {
601
- output: { first, claimed, cancelled, suppressed, resume, replyBlocked, receipt, session },
602
- toolCalls: 7, steps: 9,
603
- checks: [
604
- check(claimed?.status === 'claimed' && cancelled?.status === 'cancelled' &&
605
- session.generation === 2,
606
- `claim=${claimed?.status}; cancel=${cancelled?.status}; generation=${session.generation}`),
607
- check(suppressed.sessionId === first.sessionId && suppressed.mode === 'cancel' &&
608
- resume.suppressedInput === true && resume.cancelEventId === stopEvent.id &&
609
- sameSet(suppressed.allowedActions, ['silence']),
610
- JSON.stringify({ sessionId: suppressed.sessionId, mode: suppressed.mode,
611
- resume, allowed: suppressed.allowedActions })),
612
- check(replyBlocked.includes('不允许 reply') && receipt.kind === 'silence' &&
613
- receipt.verified === true,
614
- `blocked=${replyBlocked}; receipt=${receipt.kind}/${receipt.verified}`),
615
- ],
616
- }
617
- }
618
-
619
- function evalNegativeWaitRouting() {
620
- const { root, cfg } = workspace()
621
- enableFakeDws(root)
622
-
623
- // Alice 的旧消息第一次 lookup 时没有 Wait:先固化 NO_MATCH,再创建一条新 Wait。
624
- const oldReplyRaw = dmEvent('evt-negative-1', 'mid-negative-1', 'alice', '旧消息')
625
- const oldReplyEvent = normalizeEvent(oldReplyRaw)
626
- const noClaim = claimWait(root, cfg, 'default', oldReplyEvent)
627
- const aliceWork = prepareSession(root, cfg,
628
- dmEvent('evt-negative-2', 'mid-negative-2', 'alice', '后来事项'),
629
- { sessionKey: 'later-alice-work' })
630
- withFakeDws(root, () => executeAction(
631
- aliceWork.cwd, 'ask', { text: '后来事项需要哪个日期?' }))
632
- const aliceBeforeReplay = readWaitForEvent(
633
- root, cfg, aliceWork.fieldId, readJson(join(aliceWork.cwd, 'event.json')))
634
- const oldReplyReplay = prepareSession(root, cfg, oldReplyRaw)
635
- const aliceAfterReplay = readWaitForEvent(
636
- root, cfg, aliceWork.fieldId, readJson(join(aliceWork.cwd, 'event.json')))
637
-
638
- // Bob 的旧 stop 第一次没有目标;后来出现的 Wait 不能被旧 stop 重放误杀。
639
- const oldStopRaw = dmEvent('evt-negative-3', 'mid-negative-3', 'bob', '/stop')
640
- const noCancel = cancelWaitForEvent(root, cfg, normalizeEvent(oldStopRaw))
641
- const bobWork = prepareSession(root, cfg,
642
- dmEvent('evt-negative-4', 'mid-negative-4', 'bob', '后来事项'),
643
- { sessionKey: 'later-bob-work' })
644
- withFakeDws(root, () => executeAction(
645
- bobWork.cwd, 'ask', { text: '后来事项需要哪个负责人?' }))
646
- const bobBeforeReplay = readWaitForEvent(
647
- root, cfg, bobWork.fieldId, readJson(join(bobWork.cwd, 'event.json')))
648
- const oldStopReplay = prepareSession(root, cfg, oldStopRaw)
649
- const bobAfterReplay = readWaitForEvent(
650
- root, cfg, bobWork.fieldId, readJson(join(bobWork.cwd, 'event.json')))
651
- return {
652
- output: { noClaim, aliceWork, aliceBeforeReplay, oldReplyReplay, aliceAfterReplay,
653
- noCancel, bobWork, bobBeforeReplay, oldStopReplay, bobAfterReplay },
654
- toolCalls: 10, steps: 12,
655
- checks: [
656
- check(noClaim === null && noCancel === null, `claim=${noClaim}; cancel=${noCancel}`),
657
- check(oldReplyReplay.mode === 'start' &&
658
- oldReplyReplay.sessionId !== aliceWork.sessionId &&
659
- aliceAfterReplay?.id === aliceBeforeReplay?.id && aliceAfterReplay?.status === 'active',
660
- `oldReply=${oldReplyReplay.mode}/${oldReplyReplay.sessionId}; ` +
661
- `newWait=${aliceAfterReplay?.id}/${aliceAfterReplay?.status}`),
662
- check(oldStopReplay.mode === 'start' && oldStopReplay.sessionId !== bobWork.sessionId &&
663
- bobAfterReplay?.id === bobBeforeReplay?.id && bobAfterReplay?.status === 'active',
664
- `oldStop=${oldStopReplay.mode}/${oldStopReplay.sessionId}; ` +
665
- `newWait=${bobAfterReplay?.id}/${bobAfterReplay?.status}`),
666
- ],
667
- }
668
- }
669
-
670
- function evalWorkspaceAuthority() {
671
- const { root, cfg } = workspace()
672
- enableFakeDws(root)
673
- const workspacePath = join(root, '.dingtalk-agent', 'workspace.json')
674
- const manifest = readJson(workspacePath)
675
- manifest.dws = { profile: 'workspace-profile', expectedUserId: 'workspace-agent' }
676
- manifest.mounts.memory = 'dingtalk-doc:node-memory'
677
- writeJson(workspacePath, manifest)
678
-
679
- const dispatch = withFakeDws(root, () =>
680
- prepareSession(root, cfg, fixture('mentioned.json'), { contextId: manifest.contextId }))
681
- const field = readJson(join(dispatch.cwd, 'context', 'field.json'))
682
- const mounts = readJson(join(dispatch.cwd, 'context', 'mounts', 'manifest.json'))
683
- const memory = readFileSync(join(dispatch.cwd, 'context', 'mounts', 'memory.md'), 'utf8')
684
- const trigger = readJson(join(dispatch.cwd, 'trigger.json'))
685
- let wrongContext = ''
686
- try { prepareSession(root, cfg, fixture('mentioned.json'), { contextId: 'wrong-context' }) }
687
- catch (e) { wrongContext = e.message }
688
- return {
689
- output: { dispatch, field: field.dws, mounts, memory, trigger, wrongContext },
690
- toolCalls: 2, steps: 5,
691
- checks: [
692
- check(trigger.contextId === manifest.contextId && dispatch.contextId === manifest.contextId,
693
- `${trigger.contextId}/${dispatch.contextId}`),
694
- check(mounts.entries.some((x) => x.slot === 'memory' &&
695
- x.provider === 'dingtalk-doc') && memory.includes('Remote Memory'),
696
- `${JSON.stringify(mounts.entries)}; memory=${memory.trim()}`),
697
- check(field.dws?.profile === 'workspace-profile' &&
698
- field.dws?.expectedUserId === 'workspace-agent' &&
699
- wrongContext.includes('不属于当前 Workspace'),
700
- `${JSON.stringify(field.dws)}; wrong=${wrongContext}`),
701
- ],
702
- }
703
- }
704
-
705
- function evalSessionSkillBinding() {
706
- const { root, cfg } = workspace()
707
- installWorkspaceSkillOverride(root)
708
- const firstRaw = fixture('mentioned.json')
709
- const first = prepareSession(root, cfg, firstRaw)
710
- const firstSkill = join(first.cwd, 'context', 'skills', 'dingtalk-basic-behavior', 'SKILL.md')
711
- const before = readFileSync(firstSkill, 'utf8')
712
- const source = join(root, 'skills', 'dingtalk-basic-behavior', 'SKILL.md')
713
- writeFileSync(source, readFileSync(source, 'utf8') + '\n<!-- next-session-only -->\n')
714
-
715
- const followBody = JSON.parse(firstRaw.data)
716
- followBody.event_id = 'evt-mentioned-followup-001'
717
- followBody.message_id = 'mid-mentioned-followup-001'
718
- followBody.root_message_id = 'mid-mentioned-001'
719
- followBody.content = '确认,继续原事项'
720
- const follow = prepareSession(root, cfg, { ...firstRaw, data: JSON.stringify(followBody) })
721
- const followSkill = readFileSync(join(
722
- follow.cwd, 'context', 'skills', 'dingtalk-basic-behavior', 'SKILL.md'), 'utf8')
723
-
724
- const nextBody = { ...followBody,
725
- event_id: 'evt-mentioned-new-session-001',
726
- message_id: 'mid-mentioned-new-session-001',
727
- root_message_id: '',
728
- content: '这是另一件事',
729
- }
730
- const next = prepareSession(root, cfg, { ...firstRaw, data: JSON.stringify(nextBody) })
731
- const nextSkill = readFileSync(join(
732
- next.cwd, 'context', 'skills', 'dingtalk-basic-behavior', 'SKILL.md'), 'utf8')
733
- const session = readJson(join(first.sessionPath, 'session.json'))
734
- return {
735
- output: { first, follow, next, sessionSkills: session.skills },
736
- toolCalls: 3, steps: 6,
737
- checks: [
738
- check(follow.sessionId === first.sessionId && follow.runId !== first.runId,
739
- `${first.sessionId}/${first.runId} -> ${follow.sessionId}/${follow.runId}`),
740
- check(before === followSkill && !followSkill.includes('next-session-only') &&
741
- Boolean(session.skills?.manifestHash),
742
- `same=${before === followSkill}; manifest=${session.skills?.manifestHash}`),
743
- check(next.sessionId !== first.sessionId && nextSkill.includes('next-session-only'),
744
- `${next.sessionId}; updated=${nextSkill.includes('next-session-only')}`),
745
- ],
746
- }
747
- }
748
-
749
- function evalGlobalSkillInstall() {
750
- const home = mkdtempSync(join(tmpdir(), 'dta-skill-home-'))
751
- const cwd = mkdtempSync(join(tmpdir(), 'dta-skill-cwd-'))
752
- WORKSPACES.push(home, cwd)
753
- const env = { ...process.env, HOME: home }
754
- const first = spawnSync(process.execPath, [CLI, 'skill', 'install', '--json'], {
755
- cwd, env, encoding: 'utf8',
756
- })
757
- const second = spawnSync(process.execPath, [CLI, 'skill', 'install', '--json'], {
758
- cwd, env, encoding: 'utf8',
759
- })
760
- const status = JSON.parse(second.stdout)
761
- const canonical = join(home, '.agents', 'skills', 'dingtalk-basic-behavior')
762
- const claude = join(home, '.claude', 'skills', 'dingtalk-basic-behavior')
763
-
764
- const marker = join(canonical, '.dingtalk-agent-install.json')
765
- return {
766
- output: {
767
- firstStatus: first.status, secondStatus: second.status, status,
768
- claudeTarget: readlinkSync(claude), markerExists: existsSync(marker),
769
- },
770
- toolCalls: 3, steps: 6,
771
- checks: [
772
- check(first.status === 0 && existsSync(join(canonical, 'SKILL.md')),
773
- `status=${first.status}; canonical=${canonical}`),
774
- check(second.status === 0 && status.manager === 'skills-cli' &&
775
- status.canonical.current === true && status.operation?.command.includes('skills@latest -- skills add'),
776
- `status=${second.status}; manager=${status.manager}; current=${status.canonical.current}`),
777
- check(lstatSync(claude).isSymbolicLink() &&
778
- resolve(dirname(claude), readlinkSync(claude)) === resolve(canonical),
779
- `target=${readlinkSync(claude)}`),
780
- check(!existsSync(marker), '上游 skills CLI 接管后不存在 dta 自定义 marker'),
781
- ],
782
- }
783
- }
784
-
785
- function evalBootstrapLocal() {
786
- const root = mkdtempSync(join(tmpdir(), 'dta-bootstrap-local-'))
787
- WORKSPACES.push(root)
788
- mkdirSync(join(root, 'knowledge'), { recursive: true })
789
- writeFileSync(join(root, 'WORKSPACE.md'), '# Identity\n')
790
- writeFileSync(join(root, 'MEMORY.md'), '# Memory\n')
791
- writeFileSync(join(root, 'knowledge', 'INDEX.md'), '# Knowledge\n')
792
- const run = spawnSync(process.execPath, [CLI, 'bootstrap', '--json'], {
793
- cwd: root, encoding: 'utf8',
794
- })
795
- const out = JSON.parse(run.stdout)
796
- return {
797
- output: out, toolCalls: 1, steps: 3,
798
- checks: [
799
- check(run.status === 0 && out.mode === 'direct' && out.initialized === false,
800
- `status=${run.status}; mode=${out.mode}; initialized=${out.initialized}`),
801
- check(out.mounts.map((x) => x.slot).join(',') === 'profile,memory,knowledge',
802
- JSON.stringify(out.mounts)),
803
- check(!existsSync(join(root, '.dingtalk-agent')), 'bootstrap 未创建 .dingtalk-agent'),
804
- ],
805
- }
806
- }
807
-
808
- function evalBootstrapRemote() {
809
- const root = mkdtempSync(join(tmpdir(), 'dta-bootstrap-remote-'))
810
- WORKSPACES.push(root)
811
- const bin = enableFakeDws(root)
812
- const state = join(root, '.state')
813
- const run = spawnSync(process.execPath, [
814
- CLI, 'bootstrap', '--storage', 'dingtalk-doc:node-memory',
815
- '--state-dir', state, '--json',
816
- ], {
817
- cwd: root,
818
- env: { ...process.env, PATH: `${bin}:${process.env.PATH || ''}` },
819
- encoding: 'utf8',
820
- })
821
- const out = JSON.parse(run.stdout)
822
- const snapshot = out.mounts?.[0]?.path || ''
823
- return {
824
- output: { out, snapshot: snapshot ? readFileSync(snapshot, 'utf8') : '', stderr: run.stderr },
825
- toolCalls: 1, steps: 4,
826
- checks: [
827
- check(run.status === 0 && out.mode === 'direct' && out.initialized === false,
828
- `status=${run.status}; mode=${out.mode}; initialized=${out.initialized}`),
829
- check(out.mounts.length === 1 && out.mounts[0].provider === 'dingtalk-doc' &&
830
- readFileSync(snapshot, 'utf8').includes('Remote Memory'), JSON.stringify(out.mounts)),
831
- check(existsSync(join(dirname(snapshot), 'manifest.json')) &&
832
- !existsSync(join(root, '.dingtalk-agent', 'workspace.json')),
833
- `manifest=${join(dirname(snapshot), 'manifest.json')}`),
834
- ],
835
- }
836
- }
837
-
838
- function evalOptionalInit() {
839
- const root = mkdtempSync(join(tmpdir(), 'dta-init-optional-'))
840
- WORKSPACES.push(root)
841
- const first = spawnSync(process.execPath, [CLI, 'init'], { cwd: root, encoding: 'utf8' })
842
- const workspaceFile = join(root, '.dingtalk-agent', 'workspace.json')
843
- const before = readFileSync(workspaceFile, 'utf8')
844
- const second = spawnSync(process.execPath, [CLI, 'init'], { cwd: root, encoding: 'utf8' })
845
- const after = readFileSync(workspaceFile, 'utf8')
846
- const manifest = JSON.parse(after)
847
- return {
848
- output: { first: first.stdout, second: second.stdout, manifest },
849
- toolCalls: 2, steps: 4,
850
- checks: [
851
- check(first.status === 0 && manifest.contextId === root.split('/').pop(),
852
- `status=${first.status}; context=${manifest.contextId}`),
853
- check(before === after && second.stdout.includes('Workspace 已存在'),
854
- `same=${before === after}; second=${second.stdout.trim()}`),
855
- check(!existsSync(join(root, 'AGENTS.md')) && !existsSync(join(root, 'CLAUDE.md')) &&
856
- !existsSync(join(root, 'ontology')) && !existsSync(join(root, 'skills')),
857
- 'init 未写 AGENTS.md/CLAUDE.md/ontology/skills'),
858
- check(manifest.skills[0] === 'dingtalk-basic-behavior', JSON.stringify(manifest.skills)),
859
- ],
860
- }
861
- }
862
-
863
- function evalManagedSkillLifecycle() {
864
- const home = mkdtempSync(join(tmpdir(), 'dta-skill-lifecycle-'))
865
- const cwd = mkdtempSync(join(tmpdir(), 'dta-skill-lifecycle-cwd-'))
866
- WORKSPACES.push(home, cwd)
867
- const env = { ...process.env, HOME: home }
868
- spawnSync(process.execPath, [CLI, 'skill', 'install'], { cwd, env, encoding: 'utf8' })
869
- const canonical = join(home, '.agents', 'skills', 'dingtalk-basic-behavior')
870
- const file = join(canonical, 'SKILL.md')
871
- writeFileSync(file, readFileSync(file, 'utf8') + '\n<!-- local edit -->\n')
872
- const statusRun = spawnSync(process.execPath, [CLI, 'skill', 'status', '--json'], {
873
- cwd, env, encoding: 'utf8',
874
- })
875
- const modified = JSON.parse(statusRun.stdout)
876
- const repaired = spawnSync(process.execPath, [CLI, 'skill', 'upgrade', '--json'], {
877
- cwd, env, encoding: 'utf8',
878
- })
879
- const repairedStatus = JSON.parse(repaired.stdout)
880
- const repairedBody = readFileSync(file, 'utf8')
881
- const removed = spawnSync(process.execPath, [CLI, 'skill', 'uninstall', '--json'], {
882
- cwd, env, encoding: 'utf8',
883
- })
884
- const removedStatus = JSON.parse(removed.stdout)
885
- return {
886
- output: {
887
- modified, repairedStatus, removedStatus,
888
- },
889
- toolCalls: 4, steps: 6,
890
- checks: [
891
- check(modified.manager === 'skills-cli' && modified.canonical.exists === true,
892
- JSON.stringify(modified.canonical)),
893
- check(repaired.status === 0 && !repairedBody.includes('local edit') &&
894
- repairedStatus.operation?.command.includes('skills@latest -- skills add'),
895
- `status=${repaired.status}; command=${repairedStatus.operation?.command}`),
896
- check(removed.status === 0 && removedStatus.operation?.command.includes('skills@latest -- skills remove'),
897
- `status=${removed.status}; command=${removedStatus.operation?.command}`),
898
- check(repairedStatus.canonical.current === true &&
899
- removed.status === 0 && removedStatus.canonical.exists === false &&
900
- removedStatus.claude.state === 'missing',
901
- `repaired=${repairedStatus.canonical.current}; removed=${removedStatus.canonical.exists}; ` +
902
- `claude=${removedStatus.claude.state}`),
903
- ],
904
- }
905
- }
906
-
907
- function evalRemoteStorageTypeGate() {
908
- const root = mkdtempSync(join(tmpdir(), 'dta-bootstrap-wrong-type-'))
909
- WORKSPACES.push(root)
910
- const bin = enableFakeDws(root)
911
- const run = spawnSync(process.execPath, [
912
- CLI, 'bootstrap', '--storage', 'dingtalk-doc:sheet-node',
913
- '--state-dir', join(root, '.state'), '--json',
914
- ], {
915
- cwd: root,
916
- env: {
917
- ...process.env,
918
- PATH: `${bin}:${process.env.PATH || ''}`,
919
- DTA_FAKE_DOC_EXTENSION: 'axls',
920
- },
921
- encoding: 'utf8',
922
- })
923
- return {
924
- output: { status: run.status, stderr: run.stderr.trim() }, toolCalls: 1, steps: 2,
925
- checks: [
926
- check(run.status !== 0 && run.stderr.includes('只接受在线文档 adoc'), run.stderr.trim()),
927
- check(!existsSync(join(root, '.state')), '类型不符时没有创建远端快照目录'),
928
- ],
929
- }
930
- }
931
-
932
- function evalDirectActGate() {
933
- const root = mkdtempSync(join(tmpdir(), 'dta-direct-act-'))
934
- WORKSPACES.push(root)
935
- const run = spawnSync(process.execPath, [CLI, 'act', 'reply', '--text', '越权回复'], {
936
- cwd: root, encoding: 'utf8',
937
- })
938
- return {
939
- output: { status: run.status, stderr: run.stderr.trim() }, toolCalls: 1, steps: 2,
940
- checks: [
941
- check(run.status !== 0 && run.stderr.includes('找不到当前 Run'), run.stderr.trim()),
942
- check(!existsSync(join(root, '.dingtalk-agent')), 'act fail closed 时不自动 init'),
943
- ],
944
- }
945
- }
946
-
947
- function evalDoctorMissingDependencies() {
948
- const home = mkdtempSync(join(tmpdir(), 'dta-doctor-missing-home-'))
949
- const cwd = mkdtempSync(join(tmpdir(), 'dta-doctor-missing-cwd-'))
950
- WORKSPACES.push(home, cwd)
951
- const run = spawnSync(process.execPath, [CLI, 'doctor', '--json'], {
952
- cwd,
953
- env: { ...process.env, HOME: home, PATH: '' },
954
- encoding: 'utf8',
955
- })
956
- const out = JSON.parse(run.stdout)
957
- const byId = Object.fromEntries(out.checks.map((check) => [check.id, check]))
958
- return {
959
- output: { status: run.status, report: out }, toolCalls: 1, steps: 3,
960
- checks: [
961
- check(run.status === 2 && out.ready === false && byId.cli.level === 'fail',
962
- `status=${run.status}; ready=${out.ready}; cli=${byId.cli.level}`),
963
- check(byId.dws.level === 'fail' && byId['dws-auth'].level === 'fail',
964
- `dws=${byId.dws.level}; auth=${byId['dws-auth'].level}`),
965
- check(out.nextSteps.some((step) => step.includes('npx --yes')) &&
966
- out.nextSteps.some((step) => step.includes('dws auth login')),
967
- JSON.stringify(out.nextSteps)),
968
- ],
969
- }
970
- }
971
-
972
- function evalFirstRunSetup() {
973
- const home = mkdtempSync(join(tmpdir(), 'dta-setup-home-'))
974
- const cwd = mkdtempSync(join(tmpdir(), 'dta-setup-cwd-'))
975
- WORKSPACES.push(home, cwd)
976
- const fakeBin = enableFakeDws(cwd)
977
- const env = {
978
- ...process.env,
979
- HOME: home,
980
- SHELL: '/bin/zsh',
981
- PATH: `${fakeBin}:${process.env.PATH || ''}`,
982
- DTA_SETUP_PACKAGE_SOURCE: ROOT,
983
- }
984
- const run = spawnSync(process.execPath, [CLI, 'setup', '--json'], {
985
- cwd, env, encoding: 'utf8', timeout: 120_000,
986
- })
987
- const out = run.stdout ? JSON.parse(run.stdout) : null
988
- const executable = join(home, '.local', 'bin', 'dingtalk-agent')
989
- const alias = join(home, '.local', 'bin', 'dta')
990
- const version = existsSync(executable)
991
- ? spawnSync(executable, ['--version'], { env, encoding: 'utf8' })
992
- : { status: -1, stdout: '' }
993
- const aliasVersion = existsSync(alias)
994
- ? spawnSync(alias, ['--version'], { env, encoding: 'utf8' })
995
- : { status: -1, stdout: '' }
996
- const rc = existsSync(join(home, '.zshrc')) ? readFileSync(join(home, '.zshrc'), 'utf8') : ''
997
- const canonical = join(home, '.agents', 'skills', 'dingtalk-basic-behavior')
998
- const claude = join(home, '.claude', 'skills', 'dingtalk-basic-behavior')
999
- return {
1000
- output: {
1001
- status: run.status, stderr: run.stderr.trim(), report: out,
1002
- executable, alias, version: String(version.stdout || '').trim(),
1003
- aliasVersion: String(aliasVersion.stdout || '').trim(), rc,
1004
- },
1005
- toolCalls: 1, steps: 6,
1006
- checks: [
1007
- check(run.status === 0 && existsSync(executable) && version.status === 0 &&
1008
- existsSync(alias) && aliasVersion.status === 0 &&
1009
- aliasVersion.stdout === version.stdout,
1010
- `status=${run.status}; executable=${existsSync(executable)}; alias=${existsSync(alias)}; ` +
1011
- `version=${version.status}/${aliasVersion.status}`),
1012
- check(out?.shell.updated === true && rc.includes('dingtalk-agent PATH') &&
1013
- rc.includes('$HOME/.local/bin'),
1014
- `updated=${out?.shell.updated}; rc=${JSON.stringify(rc)}`),
1015
- check(existsSync(join(canonical, 'SKILL.md')) && lstatSync(claude).isSymbolicLink() &&
1016
- out?.skill.clients.every((client) => client.state === 'ok'),
1017
- `canonical=${existsSync(canonical)}; clients=${JSON.stringify(out?.skill.clients)}`),
1018
- check(out?.doctor.ready === true &&
1019
- out.doctor.checks.find((check) => check.id === 'dws')?.level === 'pass' &&
1020
- out.doctor.checks.find((check) => check.id === 'dws-auth')?.level === 'pass',
1021
- `ready=${out?.doctor.ready}; checks=${JSON.stringify(out?.doctor.checks)}`),
1022
- ],
1023
- }
1024
- }
1025
-
1026
- function evalCliErrorHints() {
1027
- const typo = spawnSync(process.execPath, [CLI, 'doctro'], { encoding: 'utf8' })
1028
- const subcommand = spawnSync(process.execPath, [CLI, 'skill', 'instal'], { encoding: 'utf8' })
1029
- const option = spawnSync(process.execPath, [CLI, 'doctor', '--bogus'], { encoding: 'utf8' })
1030
- return {
1031
- output: {
1032
- typo: { status: typo.status, stderr: typo.stderr.trim() },
1033
- subcommand: { status: subcommand.status, stderr: subcommand.stderr.trim() },
1034
- option: { status: option.status, stderr: option.stderr.trim() },
1035
- },
1036
- toolCalls: 3, steps: 3,
1037
- checks: [
1038
- check(typo.status === 1 && typo.stderr.includes('错误:') &&
1039
- typo.stderr.includes('提示:') && typo.stderr.includes('dta doctor'), typo.stderr.trim()),
1040
- check(subcommand.status === 1 && subcommand.stderr.includes('dta skill install'),
1041
- subcommand.stderr.trim()),
1042
- check(option.status === 2 && option.stderr.includes('检查参数拼写') &&
1043
- option.stderr.includes('dta --help'), option.stderr.trim()),
1044
- ],
1045
- }
1046
- }
1047
-
1048
- function evalSelfUpgrade() {
1049
- const home = mkdtempSync(join(tmpdir(), 'dta-upgrade-home-'))
1050
- const cwd = mkdtempSync(join(tmpdir(), 'dta-upgrade-cwd-'))
1051
- WORKSPACES.push(home, cwd)
1052
- const fakeBin = enableFakeDws(cwd)
1053
- const packageVersion = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8')).version
1054
- const env = {
1055
- ...process.env,
1056
- HOME: home,
1057
- SHELL: '/bin/zsh',
1058
- PATH: `${fakeBin}:${process.env.PATH || ''}`,
1059
- DTA_UPGRADE_PACKAGE_SOURCE: ROOT,
1060
- DTA_UPGRADE_TARGET_VERSION: packageVersion,
1061
- }
1062
- const run = spawnSync(process.execPath, [CLI, 'upgrade', '--force', '--json'], {
1063
- cwd, env, encoding: 'utf8', timeout: 240_000,
1064
- })
1065
- const out = run.stdout ? JSON.parse(run.stdout) : null
1066
- const executable = join(home, '.local', 'bin', 'dta')
1067
- const version = existsSync(executable)
1068
- ? spawnSync(executable, ['--version'], { env, encoding: 'utf8' })
1069
- : { status: -1, stdout: '' }
1070
- return {
1071
- output: {
1072
- status: run.status, stderr: run.stderr.trim(), report: out,
1073
- installedVersion: String(version.stdout || '').trim(), executable,
1074
- },
1075
- toolCalls: 3, steps: 6,
1076
- checks: [
1077
- check(run.status === 0 && out?.currentVersion === packageVersion &&
1078
- out?.targetVersion === packageVersion && out?.changed === true,
1079
- `status=${run.status}; current=${out?.currentVersion}; target=${out?.targetVersion}`),
1080
- check(version.status === 0 && String(version.stdout || '').trim() === packageVersion,
1081
- `versionStatus=${version.status}; version=${String(version.stdout || '').trim()}`),
1082
- check(out?.ready === true && out?.setup?.doctor?.ready === true,
1083
- `ready=${out?.ready}; doctor=${out?.setup?.doctor?.ready}`),
1084
- check(out?.setup?.skill?.manager === 'skills-cli' &&
1085
- out.setup.skill.clients.every((client) => client.state === 'ok'),
1086
- JSON.stringify(out?.setup?.skill?.clients)),
1087
- ],
1088
- }
1089
- }
1090
-
1091
- function dmEvent(eventId, messageId, actor, text, extra = {}) {
1092
- return {
1093
- event_corp_id: extra.tenantId || 'tenant-wait-eval',
1094
- data: JSON.stringify({
1095
- type: 'user_im_message_receive_o2o',
1096
- event_id: eventId,
1097
- conversation_id: extra.conversationId || 'cid-wait-eval',
1098
- message_id: messageId,
1099
- sender: actor,
1100
- sender_open_dingtalk_id: `$:LWCP_v1:$${actor}`,
1101
- content: text,
1102
- }),
1103
- }
1104
- }
1105
-
1106
- function enableFakeDws(root) {
1107
- const fieldPath = join(root, 'fields', 'default', 'field.json')
1108
- if (existsSync(fieldPath)) {
1109
- const field = readJson(fieldPath)
1110
- field.dws = { profile: 'eval-profile', expectedUserId: 'agent-eval' }
1111
- writeJson(fieldPath, field)
1112
- }
1113
-
1114
- const bin = join(root, '.fake-bin')
1115
- mkdirSync(bin, { recursive: true })
1116
- const script = join(bin, 'dws')
1117
- writeFileSync(script, `#!/usr/bin/env node
1118
- const a = process.argv.slice(2)
1119
- let data = {}
1120
- if (a[0] === 'version') { console.log(JSON.stringify({ version: 'v1.0.99-test' })); process.exit(0) }
1121
- if (a[0] === 'auth' && a[1] === 'status') { console.log(JSON.stringify({ authenticated: true, token_valid: true, corp_name: 'Test Corp', user_name: 'Test User' })); process.exit(0) }
1122
- if (a.includes('get-self')) data = { userId: 'agent-eval' }
1123
- else if (a.includes('doc') && a.includes('read')) data = { markdown: '# Remote Memory\\n\\n来自钉钉文档。' }
1124
- else if (a.includes('doc') && a.includes('info')) data = { contentType: 'ALIDOC', extension: process.env.DTA_FAKE_DOC_EXTENSION || 'adoc', nodeType: 'file' }
1125
- else if (a.includes('reply')) data = { openMessageId: 'fake-sent-message' }
1126
- else if (a.includes('list-by-ids')) data = { messages: [{ openMessageId: 'fake-sent-message' }] }
1127
- console.log(JSON.stringify({ data }))
1128
- `)
1129
- chmodSync(script, 0o755)
1130
- return bin
1131
- }
1132
-
1133
- function withFakeDws(root, fn) {
1134
- const before = process.env.PATH || ''
1135
- process.env.PATH = `${join(root, '.fake-bin')}:${before}`
1136
- try { return fn() } finally { process.env.PATH = before }
1137
- }
1138
-
1139
- function installWorkspaceSkillOverride(root) {
1140
- cpSync(
1141
- join(ROOT, 'skills', 'dingtalk-basic-behavior'),
1142
- join(root, 'skills', 'dingtalk-basic-behavior'),
1143
- { recursive: true },
1144
- )
1145
- }
1146
-
1147
- function workspace() {
1148
- const root = mkdtempSync(join(tmpdir(), 'dta-eval-'))
1149
- WORKSPACES.push(root)
1150
- const log = console.log
1151
- console.log = () => {}
1152
- try { init(root) } finally { console.log = log }
1153
- return { root, cfg: config.must(root) }
1154
- }
1155
-
1156
- function fixture(name) {
1157
- return JSON.parse(readFileSync(join(HERE, 'fixtures', name), 'utf8'))
1158
- }
1159
-
1160
- function sameSet(a, b) {
1161
- return a.length === b.length && a.every((x) => b.includes(x))
1162
- }
1163
-
1164
- function check(passed, evidence) { return { passed, evidence } }
1165
-
1166
- function writeJson(path, value) {
1167
- mkdirSync(dirname(path), { recursive: true })
1168
- writeFileSync(path, JSON.stringify(value, null, 2) + '\n')
1169
- }