@xdxer/dingtalk-agent 0.1.4-beta.0 → 0.1.4-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/README.en.md +397 -0
  3. package/README.md +497 -46
  4. package/dist/bin/dingtalk-agent.js +1162 -340
  5. package/dist/bin/dingtalk-agent.js.map +1 -1
  6. package/dist/src/actions.js +98 -14
  7. package/dist/src/actions.js.map +1 -1
  8. package/dist/src/agent-audit.js +460 -0
  9. package/dist/src/agent-audit.js.map +1 -0
  10. package/dist/src/agent-bindings.js +132 -0
  11. package/dist/src/agent-bindings.js.map +1 -0
  12. package/dist/src/agent-definition.js +182 -0
  13. package/dist/src/agent-definition.js.map +1 -0
  14. package/dist/src/agent-enhance.js +678 -0
  15. package/dist/src/agent-enhance.js.map +1 -0
  16. package/dist/src/bootstrap.js +125 -17
  17. package/dist/src/bootstrap.js.map +1 -1
  18. package/dist/src/config.js +1 -7
  19. package/dist/src/config.js.map +1 -1
  20. package/dist/src/development-workspace.js +729 -0
  21. package/dist/src/development-workspace.js.map +1 -0
  22. package/dist/src/doctor.js +20 -9
  23. package/dist/src/doctor.js.map +1 -1
  24. package/dist/src/dws.js +145 -0
  25. package/dist/src/dws.js.map +1 -1
  26. package/dist/src/eval-evidence.js +193 -0
  27. package/dist/src/eval-evidence.js.map +1 -0
  28. package/dist/src/init.js +1 -1
  29. package/dist/src/init.js.map +1 -1
  30. package/dist/src/invocation.js +36 -0
  31. package/dist/src/invocation.js.map +1 -0
  32. package/dist/src/lab.js +679 -0
  33. package/dist/src/lab.js.map +1 -0
  34. package/dist/src/lease.js +100 -0
  35. package/dist/src/lease.js.map +1 -0
  36. package/dist/src/memory/candidates.js +451 -0
  37. package/dist/src/memory/candidates.js.map +1 -0
  38. package/dist/src/memory/completion-evidence.js +536 -0
  39. package/dist/src/memory/completion-evidence.js.map +1 -0
  40. package/dist/src/memory/operational.js +263 -0
  41. package/dist/src/memory/operational.js.map +1 -0
  42. package/dist/src/memory/remote-state.js +478 -0
  43. package/dist/src/memory/remote-state.js.map +1 -0
  44. package/dist/src/memory/task-checkpoints.js +204 -0
  45. package/dist/src/memory/task-checkpoints.js.map +1 -0
  46. package/dist/src/multica-deploy.js +1480 -0
  47. package/dist/src/multica-deploy.js.map +1 -0
  48. package/dist/src/multica-provider.js +685 -0
  49. package/dist/src/multica-provider.js.map +1 -0
  50. package/dist/src/opencode-evals.js +1062 -0
  51. package/dist/src/opencode-evals.js.map +1 -0
  52. package/dist/src/opencode-provider.js +531 -0
  53. package/dist/src/opencode-provider.js.map +1 -0
  54. package/dist/src/opencode-workspace.js +197 -0
  55. package/dist/src/opencode-workspace.js.map +1 -0
  56. package/dist/src/perception.js +225 -0
  57. package/dist/src/perception.js.map +1 -0
  58. package/dist/src/personal-event-evals.js +595 -0
  59. package/dist/src/personal-event-evals.js.map +1 -0
  60. package/dist/src/promotion.js +786 -0
  61. package/dist/src/promotion.js.map +1 -0
  62. package/dist/src/remote-semantic-state-live-evals.js +888 -0
  63. package/dist/src/remote-semantic-state-live-evals.js.map +1 -0
  64. package/dist/src/remote-semantic-state-worker.js +38 -0
  65. package/dist/src/remote-semantic-state-worker.js.map +1 -0
  66. package/dist/src/remote-state-evals.js +501 -0
  67. package/dist/src/remote-state-evals.js.map +1 -0
  68. package/dist/src/response-gate.js +51 -0
  69. package/dist/src/response-gate.js.map +1 -0
  70. package/dist/src/robot-evals.js +770 -0
  71. package/dist/src/robot-evals.js.map +1 -0
  72. package/dist/src/sessions.js +66 -105
  73. package/dist/src/sessions.js.map +1 -1
  74. package/dist/src/setup.js +10 -9
  75. package/dist/src/setup.js.map +1 -1
  76. package/dist/src/skill-manager.js +102 -203
  77. package/dist/src/skill-manager.js.map +1 -1
  78. package/dist/src/skills.js.map +1 -1
  79. package/dist/src/storage-evals.js +26 -0
  80. package/dist/src/storage-evals.js.map +1 -0
  81. package/dist/src/types.js.map +1 -1
  82. package/dist/src/upgrade.js +137 -0
  83. package/dist/src/upgrade.js.map +1 -0
  84. package/dist/src/waits.js +5 -1
  85. package/dist/src/waits.js.map +1 -1
  86. package/dist/src/workspace.js +28 -3
  87. package/dist/src/workspace.js.map +1 -1
  88. package/docs/INSTALLATION.md +118 -13
  89. package/docs/SECOND-AGENT-ACCEPTANCE.md +62 -0
  90. package/docs/architecture/agent-memory-topology.png +0 -0
  91. package/docs/architecture/agent-memory-topology.svg +132 -0
  92. package/docs/architecture/general-agent-kernel-topology.png +0 -0
  93. package/docs/architecture/general-agent-kernel-topology.svg +149 -0
  94. package/docs/architecture/provider-bound-development-workspace.png +0 -0
  95. package/docs/architecture/provider-bound-development-workspace.svg +141 -0
  96. package/docs/architecture/task-completion-gate.png +0 -0
  97. package/docs/architecture/task-completion-gate.svg +191 -0
  98. package/docs/schemas/agent-audit-load-evidence.schema.json +14 -0
  99. package/docs/schemas/agent-audit.schema.json +92 -0
  100. package/docs/schemas/agent-bindings.schema.json +54 -0
  101. package/docs/schemas/agent-definition.schema.json +78 -0
  102. package/docs/schemas/agent-enhancement-plan.schema.json +88 -0
  103. package/docs/schemas/agent-enhancement-receipt.schema.json +37 -0
  104. package/docs/schemas/enriched-invocation.schema.json +46 -0
  105. package/docs/schemas/eval-candidate-plan.schema.json +28 -0
  106. package/docs/schemas/eval-candidate-result.schema.json +20 -0
  107. package/docs/schemas/eval-candidate.schema.json +30 -0
  108. package/docs/schemas/invocation.schema.json +19 -0
  109. package/docs/schemas/memory-candidate-proposal.schema.json +18 -0
  110. package/docs/schemas/memory-candidate.schema.json +76 -0
  111. package/docs/schemas/memory-publish-target.schema.json +25 -0
  112. package/docs/schemas/multica-deployment-list.schema.json +29 -0
  113. package/docs/schemas/multica-deployment-operation.schema.json +51 -0
  114. package/docs/schemas/multica-deployment-plan.schema.json +70 -0
  115. package/docs/schemas/multica-deployment-receipt.schema.json +87 -0
  116. package/docs/schemas/multica-deployment-status.schema.json +23 -0
  117. package/docs/schemas/multica-workspace-inspection.schema.json +77 -0
  118. package/docs/schemas/multica-workspace-plan.schema.json +68 -0
  119. package/docs/schemas/multica-workspace-resource-list.schema.json +27 -0
  120. package/docs/schemas/multica-workspace-status.schema.json +34 -0
  121. package/docs/schemas/observation.schema.json +21 -0
  122. package/docs/schemas/operational-memory-provider.schema.json +36 -0
  123. package/docs/schemas/operational-memory-record.schema.json +24 -0
  124. package/docs/schemas/perception-input.schema.json +56 -0
  125. package/docs/schemas/project.schema.json +112 -0
  126. package/docs/schemas/promotion-list.schema.json +36 -0
  127. package/docs/schemas/promotion-plan.schema.json +60 -0
  128. package/docs/schemas/promotion-policy.schema.json +29 -0
  129. package/docs/schemas/promotion-receipt.schema.json +43 -0
  130. package/docs/schemas/promotion-status.schema.json +23 -0
  131. package/docs/schemas/release-readiness.schema.json +65 -0
  132. package/docs/schemas/remote-semantic-state-live-eval.schema.json +60 -0
  133. package/docs/schemas/remote-semantic-state-manifest.schema.json +79 -0
  134. package/docs/schemas/remote-semantic-state-provider.schema.json +98 -0
  135. package/docs/schemas/response-gate.schema.json +20 -0
  136. package/docs/schemas/task-checkpoint.schema.json +71 -0
  137. package/docs/schemas/task-completion-evidence.schema.json +154 -0
  138. package/docs/schemas/workspace-doctor.schema.json +56 -0
  139. package/docs/schemas/workspace-state.schema.json +39 -0
  140. package/evals/baselines/2026-07-16/opencode-basic-010-completion-summary.json +123 -0
  141. package/evals/baselines/2026-07-16/opencode-basic-skill-required-summary.json +69 -0
  142. package/evals/baselines/2026-07-16/opencode-multi-surface-summary.json +63 -0
  143. package/evals/baselines/2026-07-16/remote-state-live-summary.json +70 -0
  144. package/evals/baselines/2026-07-17/agent-enhance-opencode-dogfood-summary.json +98 -0
  145. package/evals/baselines/2026-07-17/personal-event-live-readiness-summary.json +68 -0
  146. package/examples/agents/fde-coach/AGENTS.md +26 -0
  147. package/examples/agents/fde-coach/MEMORY.md +3 -0
  148. package/examples/agents/fde-coach/fields/default/field.json +24 -0
  149. package/examples/agents/fde-coach/knowledge/INDEX.md +4 -0
  150. package/examples/agents/fde-coach/skills/fde-coach/SKILL.md +13 -0
  151. package/examples/agents/release-manager/AGENTS.md +26 -0
  152. package/examples/agents/release-manager/MEMORY.md +3 -0
  153. package/examples/agents/release-manager/fields/default/field.json +24 -0
  154. package/examples/agents/release-manager/knowledge/INDEX.md +4 -0
  155. package/examples/agents/release-manager/skills/release-manager/SKILL.md +13 -0
  156. package/lab/README.md +109 -0
  157. package/lab/agent-eval/catalog.json +91 -0
  158. package/lab/agent-eval/classic-failures.json +177 -0
  159. package/lab/agent-eval/completion-gate-regression.json +99 -0
  160. package/lab/agent-eval/personal-event-live.example.json +94 -0
  161. package/lab/agent-eval/remote-semantic-state-live.example.json +70 -0
  162. package/lab/agent-eval/remote-semantic-state-provider.fixture.json +47 -0
  163. package/lab/agent-eval/remote-state-workspace/AGENTS.md +8 -0
  164. package/lab/agent-eval/remote-state-workspace/opencode.json +7 -0
  165. package/lab/agent-eval/remote-state-workspace/skills/remote-state-operator/SKILL.md +13 -0
  166. package/lab/agent-eval/remote-state.example.json +31 -0
  167. package/lab/agent-eval/workspace/AGENTS.md +7 -0
  168. package/lab/agent-eval/workspace/MEMORY.md +4 -0
  169. package/lab/agent-eval/workspace/artifacts/pending-review.md +3 -0
  170. package/lab/agent-eval/workspace/knowledge/INDEX.md +4 -0
  171. package/lab/agent-eval/workspace/opencode.json +20 -0
  172. package/lab/manifest.example.json +27 -0
  173. package/lab/manifest.personal-event.example.json +27 -0
  174. package/lab/project-workspace/README.md +11 -0
  175. package/lab/project-workspace/fake-multica-provider.mjs +266 -0
  176. package/lab/project-workspace/multica-deploy.fixture.json +29 -0
  177. package/lab/project-workspace/multica-readonly.fixture.json +69 -0
  178. package/lab/project-workspace/observation.fixture.json +14 -0
  179. package/lab/project-workspace/opencode-provider-suite.json +65 -0
  180. package/lab/project-workspace/project.fixture.json +44 -0
  181. package/lab/project-workspace/promotion-policy.fixture.json +15 -0
  182. package/lab/robot-eval/pool.example.json +30 -0
  183. package/lab/robot-eval/suite.json +123 -0
  184. package/lab/robot-eval/workspace/AGENTS.md +21 -0
  185. package/lab/robot-eval/workspace/MEMORY.md +3 -0
  186. package/lab/robot-eval/workspace/knowledge/INDEX.md +5 -0
  187. package/lab/robot-eval/workspace/opencode.json +22 -0
  188. package/lab/schemas/agent-eval-catalog.schema.json +47 -0
  189. package/lab/schemas/lab-manifest.schema.json +70 -0
  190. package/lab/schemas/personal-event-eval.schema.json +91 -0
  191. package/lab/schemas/remote-state-eval.schema.json +66 -0
  192. package/lab/schemas/robot-eval-suite.schema.json +184 -0
  193. package/lab/schemas/robot-pool.schema.json +56 -0
  194. package/package.json +26 -14
  195. package/skills/dingtalk-agent-boot-multica/SKILL.md +40 -0
  196. package/skills/dingtalk-agent-compose/SKILL.md +110 -0
  197. package/skills/dingtalk-agent-compose/assets/AGENTS.template.md +26 -0
  198. package/skills/dingtalk-agent-compose/assets/agent.bindings.dingtalk-doc.template.json +13 -0
  199. package/skills/dingtalk-agent-compose/assets/agent.bindings.local.template.json +13 -0
  200. package/skills/dingtalk-agent-compose/assets/opencode.template.json +12 -0
  201. package/skills/dingtalk-agent-compose/assets/role-skill.template.md +24 -0
  202. package/skills/dingtalk-agent-compose/evals/evals.json +94 -0
  203. package/skills/dingtalk-agent-compose/references/agent-definition-contract.md +55 -0
  204. package/skills/dingtalk-agent-compose/references/opencode-host-contract.md +65 -0
  205. package/skills/dingtalk-agent-compose/references/storage-routing.md +20 -0
  206. package/skills/dingtalk-agent-deploy/SKILL.md +60 -0
  207. package/skills/dingtalk-agent-deploy/references/multica-deployment-contract.md +49 -0
  208. package/skills/dingtalk-agent-deploy/references/promotion-observation-contract.md +49 -0
  209. package/skills/dingtalk-agent-eval/SKILL.md +116 -0
  210. package/skills/dingtalk-agent-eval/assets/eval-catalog.template.json +18 -0
  211. package/skills/dingtalk-agent-eval/evals/evals.json +61 -0
  212. package/skills/dingtalk-agent-eval/references/eval-topology.md +34 -0
  213. package/skills/dingtalk-agent-eval/references/evidence-contract.md +31 -0
  214. package/skills/dingtalk-agent-eval/references/scenario-taxonomy.md +25 -0
  215. package/skills/dingtalk-agent-eval/references/storage-modes.md +75 -0
  216. package/skills/dingtalk-basic-behavior/SKILL.md +63 -3
  217. package/skills/dingtalk-basic-behavior/assets/memory-candidate-proposal.json +10 -0
  218. package/skills/dingtalk-basic-behavior/assets/{task-checkpoint.md → task-checkpoint.json} +2 -21
  219. package/skills/dingtalk-basic-behavior/references/action-contract.md +2 -0
  220. package/skills/dingtalk-basic-behavior/references/memory-and-evolution.md +15 -1
  221. package/skills/dingtalk-basic-behavior/references/perception-and-gates.md +28 -0
  222. package/skills/dingtalk-basic-behavior/references/task-lifecycle.md +32 -7
  223. package/dist/src/boot.js +0 -70
  224. package/dist/src/boot.js.map +0 -1
  225. package/dist/src/duty.js +0 -74
  226. package/dist/src/duty.js.map +0 -1
  227. package/dist/src/kb.js +0 -240
  228. package/dist/src/kb.js.map +0 -1
  229. package/dist/src/runs.js +0 -79
  230. package/dist/src/runs.js.map +0 -1
  231. package/docs/ARCHITECTURE.md +0 -217
  232. package/docs/MINIMAL-WORKSPACE-V1.md +0 -172
  233. package/docs/OPEN-SOURCE-REFERENCES.md +0 -107
  234. package/docs/SELF-TEST.md +0 -252
  235. package/docs//345/206/205/347/275/221/345/256/236/347/233/270.md +0 -77
  236. package/evals/baselines/2026-07-14/behavior-summary.json +0 -28
  237. package/evals/baselines/2026-07-14/contract-summary.json +0 -18
  238. package/evals/baselines/2026-07-14/live-canary-summary.json +0 -25
  239. package/evals/baselines/2026-07-15/dingtalk-basic-behavior-0.3.0/SKILL.md +0 -72
  240. package/evals/baselines/2026-07-15/dingtalk-basic-behavior-0.3.0/references/action-contract.md +0 -31
  241. package/evals/baselines/2026-07-15/dingtalk-basic-behavior-0.3.0/references/event-to-behavior.md +0 -22
  242. package/evals/baselines/2026-07-15/dingtalk-basic-behavior-0.3.0/references/memory-and-evolution.md +0 -25
  243. package/evals/baselines/2026-07-15/dingtalk-basic-behavior-0.3.0/references/runtime-modes.md +0 -34
  244. package/evals/baselines/2026-07-15/task-lifecycle-summary.json +0 -50
  245. package/evals/evals.json +0 -316
  246. package/evals/fixtures/dm-ambiguous-send.json +0 -4
  247. package/evals/fixtures/dm-blocked.json +0 -4
  248. package/evals/fixtures/dm-clear.json +0 -4
  249. package/evals/fixtures/dm-discussion.json +0 -4
  250. package/evals/fixtures/dm-doc-write-no-tool.json +0 -4
  251. package/evals/fixtures/dm-long-task-ack.json +0 -4
  252. package/evals/fixtures/dm-nonblocking-gap.json +0 -4
  253. package/evals/fixtures/dm-structured-task.json +0 -4
  254. package/evals/fixtures/group.json +0 -10
  255. package/evals/fixtures/mentioned.json +0 -3
  256. package/evals/run-contract-evals.mjs +0 -1106
  257. package/evals/run-shadow-evals.mjs +0 -267
  258. package/evals/runners/README.md +0 -66
  259. package/evals/runners/claude-shadow.mjs +0 -533
  260. package/evals/schemas/action-request.schema.json +0 -77
  261. package/evals/shadow-evals.json +0 -133
  262. package/skills/AGENTS.md +0 -104
  263. package/skills//344/273/273/345/212/241.md +0 -48
  264. package/skills//345/237/272/347/241/200/350/241/214/344/270/272.md +0 -44
  265. package/skills//345/277/203/350/267/263.md +0 -79
  266. package/skills//346/266/210/346/201/257.md +0 -50
  267. package/skills//347/237/245/350/257/206.md +0 -55
  268. package/skills//350/257/204/346/265/213.md +0 -62
  269. package/skills//351/222/211/351/222/211.md +0 -64
  270. package/templates/ontology/index.md +0 -24
  271. package/templates/ontology/self/access.md +0 -38
  272. package/templates/ontology/self/role-spec.md +0 -34
  273. package/templates/ontology/self/workspace.md +0 -41
@@ -1,1106 +0,0 @@
1
- #!/usr/bin/env node
2
-
3
- // 纯本地合同评测:不调用真实 DWS 写能力。
4
- import {
5
- mkdtempSync, readFileSync, writeFileSync, mkdirSync, rmSync, chmodSync,
6
- readdirSync, cpSync, existsSync, lstatSync, readlinkSync,
7
- } from 'node:fs'
8
- import { tmpdir } from 'node:os'
9
- import { join, dirname } from 'node:path'
10
- import { fileURLToPath } from 'node:url'
11
- import { spawnSync } from 'node:child_process'
12
- import { init } from '../dist/src/init.js'
13
- import * as config from '../dist/src/config.js'
14
- import { ackDispatch, listPendingDispatches, prepareSession, readJson } from '../dist/src/sessions.js'
15
- import { previewAction, executeAction } from '../dist/src/actions.js'
16
- import { heartbeatEvent } from '../dist/src/driver.js'
17
- import { digest, normalizeEvent } from '../dist/src/events.js'
18
- import {
19
- cancelWaitForEvent, claimWait, readWaitForEvent, waitRoot,
20
- } from '../dist/src/waits.js'
21
-
22
- const HERE = dirname(fileURLToPath(import.meta.url))
23
- const ROOT = dirname(HERE)
24
- // 合同结果只拥有自己的子目录,不能清掉同级 Claude/live 评测证据。
25
- const RESULTS = join(HERE, 'results', 'contract')
26
- const EVALS = readJson(join(HERE, 'evals.json')).evals
27
- const CLI = join(ROOT, 'dist', 'bin', 'dingtalk-agent.js')
28
- const WORKSPACES = []
29
- process.on('exit', () => {
30
- for (const root of WORKSPACES) rmSync(root, { recursive: true, force: true })
31
- })
32
-
33
- rmSync(RESULTS, { recursive: true, force: true })
34
- mkdirSync(RESULTS, { recursive: true })
35
-
36
- const cases = [
37
- evalMentioned, evalAmbientGroup, evalDuplicate, evalTamper, evalInboxRecovery,
38
- evalDwsStringData, evalSessionScope, evalSkillSnapshot, evalSingleEgress,
39
- evalHeartbeatOccurrences, evalAskWaitResume, evalWaitScopeIsolation,
40
- evalWaitConflict, evalWaitCancelFence, evalWaitReceiptRepair,
41
- evalClaimSupersededByCancel, evalNegativeWaitRouting,
42
- evalWorkspaceAuthority, evalSessionSkillBinding,
43
- evalGlobalSkillInstall, evalBootstrapLocal, evalBootstrapRemote, evalOptionalInit,
44
- evalManagedSkillLifecycle, evalRemoteStorageTypeGate, evalDirectActGate,
45
- evalDoctorMissingDependencies, evalFirstRunSetup,
46
- ]
47
- const benchmarkRuns = []
48
- for (let i = 0; i < cases.length; i++) {
49
- const started = Date.now()
50
- const result = cases[i]()
51
- const expectations = EVALS[i].expectations.map((text, n) => ({
52
- text,
53
- passed: Boolean(result.checks[n]?.passed),
54
- evidence: result.checks[n]?.evidence || '没有证据',
55
- }))
56
- const passed = expectations.filter((x) => x.passed).length
57
- const runDir = join(RESULTS, `eval-${i + 1}`, 'with_skill')
58
- mkdirSync(join(runDir, 'outputs'), { recursive: true })
59
- writeJson(join(runDir, 'eval_metadata.json'), {
60
- eval_id: i + 1,
61
- prompt: EVALS[i].prompt,
62
- expected_output: EVALS[i].expected_output,
63
- })
64
- writeJson(join(runDir, 'outputs', 'result.json'), result.output)
65
- writeFileSync(join(runDir, 'outputs', 'summary.md'),
66
- `# Eval ${i + 1}\n\n${EVALS[i].expected_output}\n\n` +
67
- expectations.map((x) => `- ${x.passed ? '✅' : '❌'} ${x.text}: ${x.evidence}`).join('\n') + '\n')
68
- writeJson(join(runDir, 'grading.json'), {
69
- expectations,
70
- summary: { passed, failed: expectations.length - passed,
71
- total: expectations.length, pass_rate: passed / expectations.length },
72
- execution_metrics: { tool_calls: { cli: result.toolCalls }, total_tool_calls: result.toolCalls,
73
- total_steps: result.steps, errors_encountered: expectations.length - passed,
74
- output_chars: JSON.stringify(result.output).length, transcript_chars: 0 },
75
- timing: { executor_duration_seconds: (Date.now() - started) / 1000,
76
- grader_duration_seconds: 0, total_duration_seconds: (Date.now() - started) / 1000 },
77
- })
78
- benchmarkRuns.push({
79
- eval_id: i + 1,
80
- eval_name: `Contract ${i + 1}`,
81
- configuration: 'with_skill',
82
- run_number: 1,
83
- result: { pass_rate: passed / expectations.length, passed,
84
- failed: expectations.length - passed, total: expectations.length,
85
- time_seconds: (Date.now() - started) / 1000, tokens: 0,
86
- tool_calls: result.toolCalls, errors: expectations.length - passed },
87
- expectations,
88
- notes: ['Deterministic runtime contract; no live DingTalk write was executed.'],
89
- })
90
- }
91
-
92
- const mean = benchmarkRuns.reduce((sum, x) => sum + x.result.pass_rate, 0) / benchmarkRuns.length
93
- writeJson(join(RESULTS, 'benchmark.json'), {
94
- metadata: { skill_name: '基础行为', skill_path: ROOT,
95
- executor_model: 'deterministic-cli', analyzer_model: 'deterministic-assertions',
96
- timestamp: new Date().toISOString(), evals_run: EVALS.map((x) => x.id), runs_per_configuration: 1 },
97
- runs: benchmarkRuns,
98
- run_summary: {
99
- with_skill: {
100
- pass_rate: { mean, stddev: 0, min: mean, max: mean },
101
- time_seconds: { mean: 0, stddev: 0, min: 0, max: 0 },
102
- tokens: { mean: 0, stddev: 0, min: 0, max: 0 },
103
- },
104
- delta: { pass_rate: 'n/a', time_seconds: 'n/a', tokens: 'n/a' },
105
- },
106
- notes: ['这是可机器判定的运行时合同评测,不做无 Skill 的主观模型对照。',
107
- '全部案例禁止真实 DWS 写操作。'],
108
- })
109
-
110
- if (process.argv.includes('--smoke')) {
111
- const pass = mean === 1
112
- const mark = pass ? 'PASS' : 'FAIL'
113
- console.log(`${mark} 全局 Skill 可在无 Workspace 时安全安装`)
114
- console.log(`${mark} 本地/钉钉文档 Storage 可在无 init 时水合`)
115
- console.log(`${mark} init 可选、幂等且不污染项目级 Agent 文件`)
116
- console.log(`${mark} Prepared Run 的 Session/Wait/目标/幂等闸门通过`)
117
- if (!pass) process.exitCode = 1
118
- } else {
119
- console.log(JSON.stringify({ results: RESULTS, passRate: mean, evals: benchmarkRuns.length }, null, 2))
120
- }
121
- if (mean !== 1) process.exitCode = 1
122
-
123
- function evalMentioned() {
124
- const { root, cfg } = workspace()
125
- const event = fixture('mentioned.json')
126
- const dispatch = prepareSession(root, cfg, event)
127
- const target = readJson(join(dispatch.cwd, 'context', 'reply-target.json'))
128
- return {
129
- output: { dispatch, target }, toolCalls: 1, steps: 3,
130
- checks: [
131
- check(dispatch.eventKind === 'im.message.mentioned', `eventKind=${dispatch.eventKind}`),
132
- check(sameSet(dispatch.allowedActions, ['ack', 'reply', 'ask', 'silence']),
133
- `allowed=${dispatch.allowedActions.join(',')}`),
134
- check(target.conversationId === 'cid-project-a' && target.messageId === 'mid-mentioned-001' &&
135
- target.senderOpenDingTalkId === '$:LWCP_v1:$alice', JSON.stringify(target)),
136
- ],
137
- }
138
- }
139
-
140
- function evalAmbientGroup() {
141
- const { root, cfg } = workspace()
142
- const dispatch = prepareSession(root, cfg, fixture('group.json'))
143
- let blocked = ''
144
- try { previewAction(dispatch.cwd, 'reply', { text: '不应该发送' }) }
145
- catch (e) { blocked = e.message }
146
- const receipt = executeAction(dispatch.cwd, 'silence', { reason: 'unmentioned' })
147
- return {
148
- output: { dispatch, blocked, receipt }, toolCalls: 3, steps: 4,
149
- checks: [
150
- check(sameSet(dispatch.allowedActions, ['silence']), `allowed=${dispatch.allowedActions.join(',')}`),
151
- check(blocked.includes('不允许 reply'), blocked),
152
- check(receipt.kind === 'silence' && receipt.verified === true, JSON.stringify(receipt)),
153
- ],
154
- }
155
- }
156
-
157
- function evalDuplicate() {
158
- const { root, cfg } = workspace()
159
- const event = fixture('mentioned.json')
160
- const first = prepareSession(root, cfg, event)
161
- const second = prepareSession(root, cfg, event)
162
- const pending = listPendingDispatches(root, cfg)
163
- const ack = ackDispatch(root, cfg, first.eventId, first.runId, 'eval-controller')
164
- const third = prepareSession(root, cfg, event)
165
- return {
166
- output: { first, second, pending, ack, third }, toolCalls: 5, steps: 5,
167
- checks: [
168
- check(first.sessionId === second.sessionId && first.runId === second.runId &&
169
- pending.length === 1 && pending[0].runId === first.runId,
170
- `${first.sessionId}/${first.runId}; pending=${pending.length}`),
171
- check(second.duplicate === true && second.launch === true,
172
- `duplicate=${second.duplicate}, launch=${second.launch}`),
173
- check(ack.duplicate === false && third.duplicate === true && third.launch === false,
174
- `ack=${ack.duplicate}, duplicate=${third.duplicate}, launch=${third.launch}`),
175
- ],
176
- }
177
- }
178
-
179
- function evalTamper() {
180
- const { root, cfg } = workspace()
181
- const dispatch = prepareSession(root, cfg, fixture('mentioned.json'))
182
- const override = spawnSync(process.execPath, [CLI, 'act', 'reply', '--text', 'x',
183
- '--conversation-id', 'evil'], { cwd: dispatch.cwd, encoding: 'utf8' })
184
-
185
- const fake = join(root, 'fake-run')
186
- mkdirSync(fake)
187
- writeFileSync(join(fake, 'run.json'), readFileSync(join(dispatch.cwd, 'run.json')))
188
- let forged = ''
189
- try { previewAction(fake, 'silence') } catch (e) { forged = e.message }
190
-
191
- const targetPath = join(dispatch.cwd, 'context', 'reply-target.json')
192
- writeJson(targetPath, { conversationId: 'evil', messageId: 'evil', senderOpenDingTalkId: 'evil' })
193
- let drift = ''
194
- try { previewAction(dispatch.cwd, 'silence') } catch (e) { drift = e.message }
195
- return {
196
- output: { overrideStatus: override.status, overrideError: override.stderr.trim(), forged, drift },
197
- toolCalls: 4, steps: 5,
198
- checks: [
199
- check(override.status !== 0 && override.stderr.includes('Unknown option'), override.stderr.trim()),
200
- check(forged.includes('身份不一致'), forged),
201
- check(drift.includes('投影被修改'), drift),
202
- ],
203
- }
204
- }
205
-
206
- function evalInboxRecovery() {
207
- const { root, cfg } = workspace()
208
- const event = fixture('mentioned.json')
209
- const fieldPath = join(root, 'fields', 'default', 'field.json')
210
- const field = readJson(fieldPath)
211
- field.behavior.packs = ['behaviors/missing.json']
212
- writeJson(fieldPath, field)
213
- let failure = ''
214
- try { prepareSession(root, cfg, event) } catch (e) { failure = e.message }
215
- const inboxRoot = join(root, cfg.runtime.eventIndexRoot, 'inbox')
216
- const retained = readdirSync(inboxRoot).filter((x) => x.endsWith('.json')).length
217
-
218
- field.behavior.packs = ['behaviors/basic.json']
219
- writeJson(fieldPath, field)
220
- const pending = listPendingDispatches(root, cfg)
221
- const recovered = pending.find((x) => x.type === 'dingtalk-agent.run.redelivery')
222
- const remaining = readdirSync(inboxRoot).filter((x) => x.endsWith('.json')).length
223
- return {
224
- output: { failure, retained, pending, remaining }, toolCalls: 3, steps: 5,
225
- checks: [
226
- check(failure.includes('不存在') && retained === 1,
227
- `failure=${failure}; retained=${retained}`),
228
- check(Boolean(recovered?.launch && recovered?.runId),
229
- `type=${recovered?.type}; launch=${recovered?.launch}; run=${recovered?.runId}`),
230
- check(remaining === 0, `remaining=${remaining}`),
231
- ],
232
- }
233
- }
234
-
235
- function evalDwsStringData() {
236
- const { root, cfg } = workspace()
237
- const raw = {
238
- type: 'event',
239
- event_corp_id: 'tenant-a',
240
- data: JSON.stringify({
241
- type: 'user_im_message_receive_o2o', event_id: 'evt-dws-string-1',
242
- conversation_id: 'cid-dws-dm', message_id: 'mid-dws-dm',
243
- sender_open_dingtalk_id: '$:LWCP_v1:$peer', content: '你好',
244
- }),
245
- }
246
- const dispatch = prepareSession(root, cfg, raw)
247
- const normalized = readJson(join(dispatch.cwd, 'event.json'))
248
- let failure = ''
249
- try { prepareSession(root, cfg, { type: 'event', data: '{bad json' }) }
250
- catch (e) { failure = e.message }
251
- return {
252
- output: { dispatch, normalized, failure }, toolCalls: 2, steps: 3,
253
- checks: [
254
- check(normalized.kind === 'im.message.dm' && normalized.message.text === '你好',
255
- `${normalized.kind}/${normalized.message.text}`),
256
- check(normalized.tenant.id === 'tenant-a' &&
257
- normalized.replyTarget.conversationId === 'cid-dws-dm' &&
258
- normalized.replyTarget.messageId === 'mid-dws-dm' &&
259
- normalized.replyTarget.senderOpenDingTalkId === '$:LWCP_v1:$peer',
260
- JSON.stringify(normalized.replyTarget)),
261
- check(failure.includes('data 不是合法 JSON 字符串'), failure),
262
- ],
263
- }
264
- }
265
-
266
- function evalSessionScope() {
267
- const { root, cfg } = workspace()
268
- const base = fixture('mentioned.json')
269
- const body = JSON.parse(base.data)
270
- const messageEvent = (eventId, tenantId, messageId, extra = {}) => ({
271
- event_corp_id: tenantId,
272
- data: JSON.stringify({ ...body, event_id: eventId, message_id: messageId, ...extra }),
273
- })
274
- const firstEvent = messageEvent('evt-topic-1', 'tenant-a', 'mid-topic-1')
275
- const secondEvent = messageEvent('evt-topic-2', 'tenant-a', 'mid-topic-2')
276
- const followupEvent = messageEvent('evt-topic-3', 'tenant-a', 'mid-topic-3',
277
- { root_message_id: 'mid-topic-1' })
278
- const otherTenantEvent = messageEvent('evt-topic-4', 'tenant-b', 'mid-topic-1')
279
- const first = prepareSession(root, cfg, firstEvent)
280
- const second = prepareSession(root, cfg, secondEvent)
281
- const followup = prepareSession(root, cfg, followupEvent)
282
- const otherTenant = prepareSession(root, cfg, otherTenantEvent)
283
- return {
284
- output: { first, second, followup, otherTenant }, toolCalls: 4, steps: 4,
285
- checks: [
286
- check(first.sessionId !== second.sessionId, `${first.sessionId} != ${second.sessionId}`),
287
- check(first.sessionId === followup.sessionId && first.runId !== followup.runId,
288
- `${first.sessionId}/${first.runId} -> ${followup.sessionId}/${followup.runId}`),
289
- check(first.sessionId !== otherTenant.sessionId,
290
- `${first.sessionId} != ${otherTenant.sessionId}`),
291
- ],
292
- }
293
- }
294
-
295
- function evalSkillSnapshot() {
296
- const { root, cfg } = workspace()
297
- installWorkspaceSkillOverride(root)
298
- const event = fixture('mentioned.json')
299
- const first = prepareSession(root, cfg, event)
300
- const manifestPath = join(first.cwd, 'context', 'skills', 'manifest.json')
301
- const skillPath = join(first.cwd, 'context', 'skills', 'dingtalk-basic-behavior', 'SKILL.md')
302
- const manifest = readJson(manifestPath)
303
- const before = readFileSync(skillPath, 'utf8')
304
- const source = join(root, 'skills', 'dingtalk-basic-behavior', 'SKILL.md')
305
- writeFileSync(source, readFileSync(source, 'utf8') + '\n<!-- candidate change -->\n')
306
- const replay = prepareSession(root, cfg, event)
307
- const after = readFileSync(skillPath, 'utf8')
308
- const run = readJson(join(first.cwd, 'run.json'))
309
- writeFileSync(skillPath, after + '\n<!-- tampered in sandbox -->\n')
310
- let tamperBlocked = ''
311
- try { previewAction(first.cwd, 'silence', { reason: 'no_value' }) }
312
- catch (e) { tamperBlocked = e.message }
313
- return {
314
- output: { first, replay, manifest, run, tamperBlocked }, toolCalls: 5, steps: 6,
315
- checks: [
316
- check(manifest.entries?.[0]?.name === 'dingtalk-basic-behavior' &&
317
- before.includes('name: dingtalk-basic-behavior'), JSON.stringify(manifest.entries?.[0])),
318
- check(run.skills?.manifestHash === manifest.hash, `${run.skills?.manifestHash}/${manifest.hash}`),
319
- check(before === after && replay.runId === first.runId &&
320
- tamperBlocked.includes('Skill snapshot 内容漂移'),
321
- `snapshotSame=${before === after}; run=${replay.runId}; tamper=${tamperBlocked}`),
322
- ],
323
- }
324
- }
325
-
326
- function evalSingleEgress() {
327
- const { root, cfg } = workspace()
328
- const fieldPath = join(root, 'fields', 'default', 'field.json')
329
- const field = readJson(fieldPath)
330
- field.transport = { mode: 'robot-connect', egressOwner: 'connector' }
331
- writeJson(fieldPath, field)
332
- const dispatch = prepareSession(root, cfg, fixture('mentioned.json'))
333
- const fieldSnapshot = readJson(join(dispatch.cwd, 'context', 'field.json'))
334
- let blocked = ''
335
- try { previewAction(dispatch.cwd, 'reply', { text: '不应由 act 发送' }) }
336
- catch (e) { blocked = e.message }
337
- const receipt = executeAction(dispatch.cwd, 'silence', { reason: 'policy_denied' })
338
- return {
339
- output: { dispatch, fieldSnapshot, blocked, receipt }, toolCalls: 3, steps: 4,
340
- checks: [
341
- check(fieldSnapshot.transport?.egressOwner === 'connector',
342
- JSON.stringify(fieldSnapshot.transport)),
343
- check(blocked.includes('出口属于 connector'), blocked),
344
- check(receipt.kind === 'silence' && receipt.verified === true, JSON.stringify(receipt)),
345
- ],
346
- }
347
- }
348
-
349
- function evalHeartbeatOccurrences() {
350
- const { root, cfg } = workspace()
351
- const firstEvent = heartbeatEvent('duty:daily-review', 'beat-001')
352
- const secondEvent = heartbeatEvent('duty:daily-review', 'beat-002')
353
- const first = prepareSession(root, cfg, firstEvent)
354
- const retry = prepareSession(root, cfg, firstEvent)
355
- ackDispatch(root, cfg, first.eventId, first.runId, 'eval-controller')
356
- const second = prepareSession(root, cfg, secondEvent)
357
- return {
358
- output: { firstEvent, secondEvent, first, retry, second }, toolCalls: 4, steps: 5,
359
- checks: [
360
- check(first.sessionId === second.sessionId,
361
- `${first.sessionId} == ${second.sessionId}`),
362
- check(first.runId !== second.runId && second.launch === true,
363
- `${first.runId} != ${second.runId}; launch=${second.launch}`),
364
- check(retry.runId === first.runId && retry.duplicate === true,
365
- `retry=${retry.runId}; duplicate=${retry.duplicate}`),
366
- ],
367
- }
368
- }
369
-
370
- function evalAskWaitResume() {
371
- const { root, cfg } = workspace()
372
- enableFakeDws(root)
373
- const firstEvent = dmEvent('evt-wait-ask-1', 'mid-wait-ask-1', 'alice', '请先给我一个草稿')
374
- const first = prepareSession(root, cfg, firstEvent)
375
- const receipt = withFakeDws(root, () =>
376
- executeAction(first.cwd, 'ask', { text: '草稿需要偏正式还是偏口语?' }))
377
- const active = readWaitForEvent(root, cfg, first.fieldId, readJson(join(first.cwd, 'event.json')))
378
-
379
- const replyEvent = dmEvent('evt-wait-reply-1', 'mid-wait-reply-1', 'alice', '偏正式')
380
- const resumed = prepareSession(root, cfg, replyEvent)
381
- const resume = readJson(join(resumed.cwd, 'context', 'resume.json'))
382
- const claimed = readWaitForEvent(root, cfg, resumed.fieldId, readJson(join(resumed.cwd, 'event.json')))
383
- const secondReceipt = withFakeDws(root, () =>
384
- executeAction(resumed.cwd, 'ask', { text: '是否还需要附上数据来源?' }))
385
- const secondActive = readWaitForEvent(root, cfg, resumed.fieldId, readJson(join(resumed.cwd, 'event.json')))
386
- const replay = withFakeDws(root, () =>
387
- executeAction(first.cwd, 'ask', { text: '草稿需要偏正式还是偏口语?' }))
388
- const afterReplay = readWaitForEvent(root, cfg, resumed.fieldId, readJson(join(resumed.cwd, 'event.json')))
389
- return {
390
- output: { first, receipt, active, resumed, resume, claimed,
391
- secondReceipt, secondActive, replay, afterReplay }, toolCalls: 7, steps: 9,
392
- checks: [
393
- check(receipt.kind === 'ask' && active?.status === 'active' &&
394
- active?.sourceRunId === first.runId,
395
- `receipt=${receipt.outcome}; wait=${active?.id}/${active?.status}`),
396
- check(resumed.sessionId === first.sessionId && resumed.runId !== first.runId &&
397
- resumed.mode === 'resume',
398
- `${first.sessionId}/${first.runId} -> ${resumed.sessionId}/${resumed.runId}`),
399
- check(resume.waitId === active?.id && resume.sourceRunId === first.runId &&
400
- resume.questionText === '草稿需要偏正式还是偏口语?' &&
401
- claimed?.status === 'claimed' && claimed?.claim?.runId === resumed.runId,
402
- JSON.stringify({ resume, status: claimed?.status, claim: claimed?.claim })),
403
- check(replay.duplicate === true && secondActive?.id !== active?.id &&
404
- afterReplay?.id === secondActive?.id && afterReplay?.status === 'active',
405
- `old=${active?.id}; current=${secondActive?.id}; afterReplay=${afterReplay?.id}`),
406
- ],
407
- }
408
- }
409
-
410
- function evalWaitScopeIsolation() {
411
- const { root, cfg } = workspace()
412
- enableFakeDws(root)
413
- const firstEvent = dmEvent('evt-wait-scope-1', 'mid-wait-scope-1', 'alice', '帮我准备材料')
414
- const first = prepareSession(root, cfg, firstEvent)
415
- withFakeDws(root, () => executeAction(first.cwd, 'ask', { text: '材料用于哪个会议?' }))
416
-
417
- const otherActor = prepareSession(root, cfg,
418
- dmEvent('evt-wait-scope-2', 'mid-wait-scope-2', 'bob', '用于周会'))
419
- const explicit = prepareSession(root, cfg,
420
- dmEvent('evt-wait-scope-3', 'mid-wait-scope-3', 'alice', '另开一个事项'),
421
- { sessionKey: 'explicit-other-work' })
422
- const resumed = prepareSession(root, cfg,
423
- dmEvent('evt-wait-scope-4', 'mid-wait-scope-4', 'alice', '用于季度复盘会'))
424
- return {
425
- output: { first, otherActor, explicit, resumed }, toolCalls: 5, steps: 6,
426
- checks: [
427
- check(otherActor.sessionId !== first.sessionId && otherActor.mode === 'start',
428
- `other=${otherActor.sessionId}/${otherActor.mode}`),
429
- check(explicit.sessionId !== first.sessionId && explicit.mode === 'start',
430
- `explicit=${explicit.sessionId}/${explicit.mode}`),
431
- check(resumed.sessionId === first.sessionId && resumed.mode === 'resume',
432
- `resumed=${resumed.sessionId}/${resumed.mode}`),
433
- ],
434
- }
435
- }
436
-
437
- function evalWaitConflict() {
438
- const { root, cfg } = workspace()
439
- enableFakeDws(root)
440
- const firstEvent = dmEvent('evt-wait-conflict-1', 'mid-wait-conflict-1', 'alice', '事项一')
441
- const first = prepareSession(root, cfg, firstEvent)
442
- withFakeDws(root, () => executeAction(first.cwd, 'ask', { text: '事项一缺哪个日期?' }))
443
- const resumed = prepareSession(root, cfg,
444
- dmEvent('evt-wait-conflict-2', 'mid-wait-conflict-2', 'alice', '下周一'))
445
- const claimed = readWaitForEvent(
446
- root, cfg, first.fieldId, readJson(join(resumed.cwd, 'event.json')))
447
- const second = prepareSession(root, cfg,
448
- dmEvent('evt-wait-conflict-3', 'mid-wait-conflict-3', 'alice', '事项二'),
449
- { sessionKey: 'parallel-work' })
450
- let blocked = ''
451
- try {
452
- withFakeDws(root, () => executeAction(second.cwd, 'ask', { text: '事项二缺哪个负责人?' }))
453
- } catch (e) { blocked = e.message }
454
- const current = readWaitForEvent(root, cfg, first.fieldId, readJson(join(first.cwd, 'event.json')))
455
- return {
456
- output: { first, resumed, claimed, second, blocked, current }, toolCalls: 5, steps: 6,
457
- checks: [
458
- check(first.sessionId !== second.sessionId && resumed.sessionId === first.sessionId,
459
- `${first.sessionId} != ${second.sessionId}; resumed=${resumed.sessionId}`),
460
- check(blocked.includes('单链 v1 拒绝并行追问'), blocked),
461
- check(claimed?.status === 'claimed' && current?.sessionId === first.sessionId &&
462
- current?.status === 'claimed',
463
- `${current?.sessionId}/${current?.status}`),
464
- ],
465
- }
466
- }
467
-
468
- function evalWaitCancelFence() {
469
- const { root, cfg } = workspace()
470
- enableFakeDws(root)
471
- const firstEvent = dmEvent('evt-wait-cancel-1', 'mid-wait-cancel-1', 'alice', '请准备发布稿')
472
- const first = prepareSession(root, cfg, firstEvent)
473
- withFakeDws(root, () => executeAction(first.cwd, 'ask', { text: '确认按当前版本发布吗?' }))
474
-
475
- // 下一条消息已 claim 并物化成 resumed Run,但尚未执行;/stop 仍必须能截断它。
476
- const resumed = prepareSession(root, cfg,
477
- dmEvent('evt-wait-cancel-2', 'mid-wait-cancel-2', 'alice', '确认'))
478
- const beforeStop = readWaitForEvent(
479
- root, cfg, first.fieldId, readJson(join(resumed.cwd, 'event.json')))
480
-
481
- // 故意破坏当前 Behavior 引用,证明取消闸门先于 Field/Behavior/Skill 物化执行。
482
- const fieldPath = join(root, 'fields', 'default', 'field.json')
483
- const field = readJson(fieldPath)
484
- writeJson(fieldPath, {
485
- ...field,
486
- behavior: { ...(field.behavior || {}), packs: ['behaviors/missing.json'] },
487
- })
488
- const stopEvent = dmEvent('evt-wait-cancel-3', 'mid-wait-cancel-3', 'alice', '/stop')
489
- let interrupted = ''
490
- try { prepareSession(root, cfg, stopEvent) }
491
- catch (e) { interrupted = e.message }
492
- const fencedBeforeMaterialize = readJson(join(first.sessionPath, 'session.json'))
493
- writeJson(fieldPath, field)
494
-
495
- // 旧 stop 尚未落 event-index 时,新事项已经在同一 scope 建立 Wait。
496
- // 重放旧 stop 必须按 eventId ledger 找回旧 Wait,不能误取消这个新 head。
497
- const newer = prepareSession(root, cfg,
498
- dmEvent('evt-wait-cancel-4', 'mid-wait-cancel-4', 'alice', '另开一个发布事项'),
499
- { sessionKey: 'post-cancel-new-work' })
500
- withFakeDws(root, () => executeAction(
501
- newer.cwd, 'ask', { text: '新事项按哪个版本?' }))
502
- const newActiveBeforeReplay = readWaitForEvent(
503
- root, cfg, newer.fieldId, readJson(join(newer.cwd, 'event.json')))
504
- const cancelled = prepareSession(root, cfg, stopEvent)
505
- const duplicate = prepareSession(root, cfg, stopEvent)
506
- const session = readJson(join(first.sessionPath, 'session.json'))
507
- const waitAfterReplay = readWaitForEvent(
508
- root, cfg, newer.fieldId, readJson(join(newer.cwd, 'event.json')))
509
- let fenced = ''
510
- try { previewAction(resumed.cwd, 'reply', { text: '已唤醒 Run 不应再发送' }) }
511
- catch (e) { fenced = e.message }
512
- return {
513
- output: { first, resumed, beforeStop, interrupted, fencedBeforeMaterialize,
514
- newer, newActiveBeforeReplay, cancelled, duplicate, session, waitAfterReplay,
515
- fenced }, toolCalls: 10, steps: 13,
516
- checks: [
517
- check(beforeStop?.status === 'claimed' && cancelled.sessionId === first.sessionId &&
518
- cancelled.mode === 'cancel' &&
519
- newActiveBeforeReplay?.sessionId === newer.sessionId &&
520
- waitAfterReplay?.id === newActiveBeforeReplay?.id &&
521
- waitAfterReplay?.status === 'active' &&
522
- sameSet(cancelled.allowedActions, ['reply', 'silence']),
523
- `before=${beforeStop?.status}; cancel=${cancelled.sessionId}/${cancelled.mode}; ` +
524
- `new=${waitAfterReplay?.id}/${waitAfterReplay?.status}`),
525
- check(interrupted.includes('missing.json') && fencedBeforeMaterialize.generation === 2 &&
526
- session.generation === 2 && duplicate.runId === cancelled.runId,
527
- `interrupted=${interrupted}; generations=${fencedBeforeMaterialize.generation}/${session.generation}`),
528
- check(fenced.includes('Session generation 已失效'), fenced),
529
- ],
530
- }
531
- }
532
-
533
- function evalWaitReceiptRepair() {
534
- const { root, cfg } = workspace()
535
- enableFakeDws(root)
536
- const firstEvent = dmEvent('evt-wait-repair-1', 'mid-wait-repair-1', 'alice', '帮我写总结')
537
- const first = prepareSession(root, cfg, firstEvent)
538
- withFakeDws(root, () => executeAction(first.cwd, 'ask', { text: '总结覆盖哪个时间段?' }))
539
- const event = readJson(join(first.cwd, 'event.json'))
540
- const active = readWaitForEvent(root, cfg, first.fieldId, event)
541
-
542
- // 模拟进程在 Receipt 落盘后、Wait 状态更新前崩溃。
543
- const crashed = {
544
- ...active,
545
- status: 'arming',
546
- delivery: null,
547
- lastAppliedReceiptActionId: undefined,
548
- }
549
- const authorityRoot = waitRoot(root, cfg)
550
- writeJson(join(authorityRoot, `${active.scopeHash}.json`), crashed)
551
- writeJson(join(authorityRoot, 'history', `${active.id}.json`), crashed)
552
-
553
- const resumed = prepareSession(root, cfg,
554
- dmEvent('evt-wait-repair-2', 'mid-wait-repair-2', 'alice', '本周'))
555
- const repaired = readWaitForEvent(
556
- root, cfg, resumed.fieldId, readJson(join(resumed.cwd, 'event.json')))
557
- const resume = readJson(join(resumed.cwd, 'context', 'resume.json'))
558
- return {
559
- output: { first, active, crashed, resumed, repaired, resume }, toolCalls: 4, steps: 6,
560
- checks: [
561
- check(active?.status === 'active' && crashed.status === 'arming',
562
- `${active?.status} -> ${crashed.status}`),
563
- check(resumed.sessionId === first.sessionId && resumed.mode === 'resume' &&
564
- repaired?.status === 'claimed',
565
- `${resumed.sessionId}/${resumed.mode}; wait=${repaired?.status}`),
566
- check(resume.waitId === active.id && resume.questionText === '总结覆盖哪个时间段?' &&
567
- repaired?.claim?.runId === resumed.runId,
568
- JSON.stringify({ resume, claim: repaired?.claim })),
569
- ],
570
- }
571
- }
572
-
573
- function evalClaimSupersededByCancel() {
574
- const { root, cfg } = workspace()
575
- enableFakeDws(root)
576
- const firstEvent = dmEvent('evt-claim-race-1', 'mid-claim-race-1', 'alice', '帮我发布评价')
577
- const first = prepareSession(root, cfg, firstEvent)
578
- withFakeDws(root, () => executeAction(
579
- first.cwd, 'ask', { text: '确认使用当前评价吗?' }))
580
-
581
- // 精确构造:回复 E 已绑定并 claim Wait,但尚未来得及 materialize Run;/stop 抢先取消。
582
- const replyRaw = dmEvent('evt-claim-race-2', 'mid-claim-race-2', 'alice', '确认')
583
- const replyEvent = normalizeEvent(replyRaw)
584
- const claimed = claimWait(root, cfg, first.fieldId, replyEvent)
585
- // 模拟 claimed head 已落盘、event ledger 尚未落盘时 SIGKILL。
586
- rmSync(join(waitRoot(root, cfg), 'events', `${digest(replyEvent.id)}.json`), { force: true })
587
- const stopRaw = dmEvent('evt-claim-race-3', 'mid-claim-race-3', 'alice', '/stop')
588
- const stopEvent = normalizeEvent(stopRaw)
589
- const cancelled = cancelWaitForEvent(root, cfg, stopEvent)
590
-
591
- // E 重放只能回到原 Session 的安全 no-op Run,绝不能 fallback 成新事项。
592
- const suppressed = prepareSession(root, cfg, replyRaw)
593
- const resume = readJson(join(suppressed.cwd, 'context', 'resume.json'))
594
- let replyBlocked = ''
595
- try { previewAction(suppressed.cwd, 'reply', { text: '不应发送' }) }
596
- catch (e) { replyBlocked = e.message }
597
- const receipt = executeAction(suppressed.cwd, 'silence', { reason: 'policy_denied' })
598
- const session = readJson(join(first.sessionPath, 'session.json'))
599
- return {
600
- output: { first, claimed, cancelled, suppressed, resume, replyBlocked, receipt, session },
601
- toolCalls: 7, steps: 9,
602
- checks: [
603
- check(claimed?.status === 'claimed' && cancelled?.status === 'cancelled' &&
604
- session.generation === 2,
605
- `claim=${claimed?.status}; cancel=${cancelled?.status}; generation=${session.generation}`),
606
- check(suppressed.sessionId === first.sessionId && suppressed.mode === 'cancel' &&
607
- resume.suppressedInput === true && resume.cancelEventId === stopEvent.id &&
608
- sameSet(suppressed.allowedActions, ['silence']),
609
- JSON.stringify({ sessionId: suppressed.sessionId, mode: suppressed.mode,
610
- resume, allowed: suppressed.allowedActions })),
611
- check(replyBlocked.includes('不允许 reply') && receipt.kind === 'silence' &&
612
- receipt.verified === true,
613
- `blocked=${replyBlocked}; receipt=${receipt.kind}/${receipt.verified}`),
614
- ],
615
- }
616
- }
617
-
618
- function evalNegativeWaitRouting() {
619
- const { root, cfg } = workspace()
620
- enableFakeDws(root)
621
-
622
- // Alice 的旧消息第一次 lookup 时没有 Wait:先固化 NO_MATCH,再创建一条新 Wait。
623
- const oldReplyRaw = dmEvent('evt-negative-1', 'mid-negative-1', 'alice', '旧消息')
624
- const oldReplyEvent = normalizeEvent(oldReplyRaw)
625
- const noClaim = claimWait(root, cfg, 'default', oldReplyEvent)
626
- const aliceWork = prepareSession(root, cfg,
627
- dmEvent('evt-negative-2', 'mid-negative-2', 'alice', '后来事项'),
628
- { sessionKey: 'later-alice-work' })
629
- withFakeDws(root, () => executeAction(
630
- aliceWork.cwd, 'ask', { text: '后来事项需要哪个日期?' }))
631
- const aliceBeforeReplay = readWaitForEvent(
632
- root, cfg, aliceWork.fieldId, readJson(join(aliceWork.cwd, 'event.json')))
633
- const oldReplyReplay = prepareSession(root, cfg, oldReplyRaw)
634
- const aliceAfterReplay = readWaitForEvent(
635
- root, cfg, aliceWork.fieldId, readJson(join(aliceWork.cwd, 'event.json')))
636
-
637
- // Bob 的旧 stop 第一次没有目标;后来出现的 Wait 不能被旧 stop 重放误杀。
638
- const oldStopRaw = dmEvent('evt-negative-3', 'mid-negative-3', 'bob', '/stop')
639
- const noCancel = cancelWaitForEvent(root, cfg, normalizeEvent(oldStopRaw))
640
- const bobWork = prepareSession(root, cfg,
641
- dmEvent('evt-negative-4', 'mid-negative-4', 'bob', '后来事项'),
642
- { sessionKey: 'later-bob-work' })
643
- withFakeDws(root, () => executeAction(
644
- bobWork.cwd, 'ask', { text: '后来事项需要哪个负责人?' }))
645
- const bobBeforeReplay = readWaitForEvent(
646
- root, cfg, bobWork.fieldId, readJson(join(bobWork.cwd, 'event.json')))
647
- const oldStopReplay = prepareSession(root, cfg, oldStopRaw)
648
- const bobAfterReplay = readWaitForEvent(
649
- root, cfg, bobWork.fieldId, readJson(join(bobWork.cwd, 'event.json')))
650
- return {
651
- output: { noClaim, aliceWork, aliceBeforeReplay, oldReplyReplay, aliceAfterReplay,
652
- noCancel, bobWork, bobBeforeReplay, oldStopReplay, bobAfterReplay },
653
- toolCalls: 10, steps: 12,
654
- checks: [
655
- check(noClaim === null && noCancel === null, `claim=${noClaim}; cancel=${noCancel}`),
656
- check(oldReplyReplay.mode === 'start' &&
657
- oldReplyReplay.sessionId !== aliceWork.sessionId &&
658
- aliceAfterReplay?.id === aliceBeforeReplay?.id && aliceAfterReplay?.status === 'active',
659
- `oldReply=${oldReplyReplay.mode}/${oldReplyReplay.sessionId}; ` +
660
- `newWait=${aliceAfterReplay?.id}/${aliceAfterReplay?.status}`),
661
- check(oldStopReplay.mode === 'start' && oldStopReplay.sessionId !== bobWork.sessionId &&
662
- bobAfterReplay?.id === bobBeforeReplay?.id && bobAfterReplay?.status === 'active',
663
- `oldStop=${oldStopReplay.mode}/${oldStopReplay.sessionId}; ` +
664
- `newWait=${bobAfterReplay?.id}/${bobAfterReplay?.status}`),
665
- ],
666
- }
667
- }
668
-
669
- function evalWorkspaceAuthority() {
670
- const { root, cfg } = workspace()
671
- enableFakeDws(root)
672
- const workspacePath = join(root, '.dingtalk-agent', 'workspace.json')
673
- const manifest = readJson(workspacePath)
674
- manifest.dws = { profile: 'workspace-profile', expectedUserId: 'workspace-agent' }
675
- manifest.mounts.memory = 'dingtalk-doc:node-memory'
676
- writeJson(workspacePath, manifest)
677
-
678
- const dispatch = withFakeDws(root, () =>
679
- prepareSession(root, cfg, fixture('mentioned.json'), { contextId: manifest.contextId }))
680
- const field = readJson(join(dispatch.cwd, 'context', 'field.json'))
681
- const mounts = readJson(join(dispatch.cwd, 'context', 'mounts', 'manifest.json'))
682
- const memory = readFileSync(join(dispatch.cwd, 'context', 'mounts', 'memory.md'), 'utf8')
683
- const trigger = readJson(join(dispatch.cwd, 'trigger.json'))
684
- let wrongContext = ''
685
- try { prepareSession(root, cfg, fixture('mentioned.json'), { contextId: 'wrong-context' }) }
686
- catch (e) { wrongContext = e.message }
687
- return {
688
- output: { dispatch, field: field.dws, mounts, memory, trigger, wrongContext },
689
- toolCalls: 2, steps: 5,
690
- checks: [
691
- check(trigger.contextId === manifest.contextId && dispatch.contextId === manifest.contextId,
692
- `${trigger.contextId}/${dispatch.contextId}`),
693
- check(mounts.entries.some((x) => x.slot === 'memory' &&
694
- x.provider === 'dingtalk-doc') && memory.includes('Remote Memory'),
695
- `${JSON.stringify(mounts.entries)}; memory=${memory.trim()}`),
696
- check(field.dws?.profile === 'workspace-profile' &&
697
- field.dws?.expectedUserId === 'workspace-agent' &&
698
- wrongContext.includes('不属于当前 Workspace'),
699
- `${JSON.stringify(field.dws)}; wrong=${wrongContext}`),
700
- ],
701
- }
702
- }
703
-
704
- function evalSessionSkillBinding() {
705
- const { root, cfg } = workspace()
706
- installWorkspaceSkillOverride(root)
707
- const firstRaw = fixture('mentioned.json')
708
- const first = prepareSession(root, cfg, firstRaw)
709
- const firstSkill = join(first.cwd, 'context', 'skills', 'dingtalk-basic-behavior', 'SKILL.md')
710
- const before = readFileSync(firstSkill, 'utf8')
711
- const source = join(root, 'skills', 'dingtalk-basic-behavior', 'SKILL.md')
712
- writeFileSync(source, readFileSync(source, 'utf8') + '\n<!-- next-session-only -->\n')
713
-
714
- const followBody = JSON.parse(firstRaw.data)
715
- followBody.event_id = 'evt-mentioned-followup-001'
716
- followBody.message_id = 'mid-mentioned-followup-001'
717
- followBody.root_message_id = 'mid-mentioned-001'
718
- followBody.content = '确认,继续原事项'
719
- const follow = prepareSession(root, cfg, { ...firstRaw, data: JSON.stringify(followBody) })
720
- const followSkill = readFileSync(join(
721
- follow.cwd, 'context', 'skills', 'dingtalk-basic-behavior', 'SKILL.md'), 'utf8')
722
-
723
- const nextBody = { ...followBody,
724
- event_id: 'evt-mentioned-new-session-001',
725
- message_id: 'mid-mentioned-new-session-001',
726
- root_message_id: '',
727
- content: '这是另一件事',
728
- }
729
- const next = prepareSession(root, cfg, { ...firstRaw, data: JSON.stringify(nextBody) })
730
- const nextSkill = readFileSync(join(
731
- next.cwd, 'context', 'skills', 'dingtalk-basic-behavior', 'SKILL.md'), 'utf8')
732
- const session = readJson(join(first.sessionPath, 'session.json'))
733
- return {
734
- output: { first, follow, next, sessionSkills: session.skills },
735
- toolCalls: 3, steps: 6,
736
- checks: [
737
- check(follow.sessionId === first.sessionId && follow.runId !== first.runId,
738
- `${first.sessionId}/${first.runId} -> ${follow.sessionId}/${follow.runId}`),
739
- check(before === followSkill && !followSkill.includes('next-session-only') &&
740
- Boolean(session.skills?.manifestHash),
741
- `same=${before === followSkill}; manifest=${session.skills?.manifestHash}`),
742
- check(next.sessionId !== first.sessionId && nextSkill.includes('next-session-only'),
743
- `${next.sessionId}; updated=${nextSkill.includes('next-session-only')}`),
744
- ],
745
- }
746
- }
747
-
748
- function evalGlobalSkillInstall() {
749
- const home = mkdtempSync(join(tmpdir(), 'dta-skill-home-'))
750
- const cwd = mkdtempSync(join(tmpdir(), 'dta-skill-cwd-'))
751
- const foreignHome = mkdtempSync(join(tmpdir(), 'dta-skill-foreign-'))
752
- WORKSPACES.push(home, cwd, foreignHome)
753
- const env = { ...process.env, HOME: home }
754
- const first = spawnSync(process.execPath, [CLI, 'skill', 'install', '--json'], {
755
- cwd, env, encoding: 'utf8',
756
- })
757
- const second = spawnSync(process.execPath, [CLI, 'skill', 'install', '--json'], {
758
- cwd, env, encoding: 'utf8',
759
- })
760
- const status = JSON.parse(second.stdout)
761
- const canonical = join(home, '.agents', 'skills', 'dingtalk-basic-behavior')
762
- const claude = join(home, '.claude', 'skills', 'dingtalk-basic-behavior')
763
-
764
- const foreign = join(foreignHome, '.agents', 'skills', 'dingtalk-basic-behavior')
765
- mkdirSync(foreign, { recursive: true })
766
- writeFileSync(join(foreign, 'SKILL.md'), '---\nname: dingtalk-basic-behavior\ndescription: foreign\n---\n')
767
- const conflict = spawnSync(process.execPath, [CLI, 'skill', 'install'], {
768
- cwd, env: { ...process.env, HOME: foreignHome }, encoding: 'utf8',
769
- })
770
- return {
771
- output: {
772
- firstStatus: first.status, secondStatus: second.status, status,
773
- claudeTarget: readlinkSync(claude), conflictStatus: conflict.status,
774
- conflictError: conflict.stderr.trim(),
775
- },
776
- toolCalls: 3, steps: 6,
777
- checks: [
778
- check(first.status === 0 && existsSync(join(canonical, 'SKILL.md')),
779
- `status=${first.status}; canonical=${canonical}`),
780
- check(second.status === 0 && status.canonical.current === true &&
781
- status.canonical.modified === false,
782
- `status=${second.status}; current=${status.canonical.current}; modified=${status.canonical.modified}`),
783
- check(lstatSync(claude).isSymbolicLink() &&
784
- readlinkSync(claude) === '../../.agents/skills/dingtalk-basic-behavior',
785
- `target=${readlinkSync(claude)}`),
786
- check(conflict.status !== 0 && conflict.stderr.includes('拒绝覆盖'), conflict.stderr.trim()),
787
- ],
788
- }
789
- }
790
-
791
- function evalBootstrapLocal() {
792
- const root = mkdtempSync(join(tmpdir(), 'dta-bootstrap-local-'))
793
- WORKSPACES.push(root)
794
- mkdirSync(join(root, 'knowledge'), { recursive: true })
795
- writeFileSync(join(root, 'WORKSPACE.md'), '# Identity\n')
796
- writeFileSync(join(root, 'MEMORY.md'), '# Memory\n')
797
- writeFileSync(join(root, 'knowledge', 'INDEX.md'), '# Knowledge\n')
798
- const run = spawnSync(process.execPath, [CLI, 'bootstrap', '--json'], {
799
- cwd: root, encoding: 'utf8',
800
- })
801
- const out = JSON.parse(run.stdout)
802
- return {
803
- output: out, toolCalls: 1, steps: 3,
804
- checks: [
805
- check(run.status === 0 && out.mode === 'direct' && out.initialized === false,
806
- `status=${run.status}; mode=${out.mode}; initialized=${out.initialized}`),
807
- check(out.mounts.map((x) => x.slot).join(',') === 'profile,memory,knowledge',
808
- JSON.stringify(out.mounts)),
809
- check(!existsSync(join(root, '.dingtalk-agent')), 'bootstrap 未创建 .dingtalk-agent'),
810
- ],
811
- }
812
- }
813
-
814
- function evalBootstrapRemote() {
815
- const root = mkdtempSync(join(tmpdir(), 'dta-bootstrap-remote-'))
816
- WORKSPACES.push(root)
817
- const bin = enableFakeDws(root)
818
- const state = join(root, '.state')
819
- const run = spawnSync(process.execPath, [
820
- CLI, 'bootstrap', '--storage', 'dingtalk-doc:node-memory',
821
- '--state-dir', state, '--json',
822
- ], {
823
- cwd: root,
824
- env: { ...process.env, PATH: `${bin}:${process.env.PATH || ''}` },
825
- encoding: 'utf8',
826
- })
827
- const out = JSON.parse(run.stdout)
828
- const snapshot = out.mounts?.[0]?.path || ''
829
- return {
830
- output: { out, snapshot: snapshot ? readFileSync(snapshot, 'utf8') : '', stderr: run.stderr },
831
- toolCalls: 1, steps: 4,
832
- checks: [
833
- check(run.status === 0 && out.mode === 'direct' && out.initialized === false,
834
- `status=${run.status}; mode=${out.mode}; initialized=${out.initialized}`),
835
- check(out.mounts.length === 1 && out.mounts[0].provider === 'dingtalk-doc' &&
836
- readFileSync(snapshot, 'utf8').includes('Remote Memory'), JSON.stringify(out.mounts)),
837
- check(existsSync(join(dirname(snapshot), 'manifest.json')) &&
838
- !existsSync(join(root, '.dingtalk-agent', 'workspace.json')),
839
- `manifest=${join(dirname(snapshot), 'manifest.json')}`),
840
- ],
841
- }
842
- }
843
-
844
- function evalOptionalInit() {
845
- const root = mkdtempSync(join(tmpdir(), 'dta-init-optional-'))
846
- WORKSPACES.push(root)
847
- const first = spawnSync(process.execPath, [CLI, 'init'], { cwd: root, encoding: 'utf8' })
848
- const workspaceFile = join(root, '.dingtalk-agent', 'workspace.json')
849
- const before = readFileSync(workspaceFile, 'utf8')
850
- const second = spawnSync(process.execPath, [CLI, 'init'], { cwd: root, encoding: 'utf8' })
851
- const after = readFileSync(workspaceFile, 'utf8')
852
- const manifest = JSON.parse(after)
853
- return {
854
- output: { first: first.stdout, second: second.stdout, manifest },
855
- toolCalls: 2, steps: 4,
856
- checks: [
857
- check(first.status === 0 && manifest.contextId === root.split('/').pop(),
858
- `status=${first.status}; context=${manifest.contextId}`),
859
- check(before === after && second.stdout.includes('Workspace 已存在'),
860
- `same=${before === after}; second=${second.stdout.trim()}`),
861
- check(!existsSync(join(root, 'AGENTS.md')) && !existsSync(join(root, 'CLAUDE.md')) &&
862
- !existsSync(join(root, 'ontology')) && !existsSync(join(root, 'skills')),
863
- 'init 未写 AGENTS.md/CLAUDE.md/ontology/skills'),
864
- check(manifest.skills[0] === 'dingtalk-basic-behavior', JSON.stringify(manifest.skills)),
865
- ],
866
- }
867
- }
868
-
869
- function evalManagedSkillLifecycle() {
870
- const home = mkdtempSync(join(tmpdir(), 'dta-skill-lifecycle-'))
871
- const cwd = mkdtempSync(join(tmpdir(), 'dta-skill-lifecycle-cwd-'))
872
- WORKSPACES.push(home, cwd)
873
- const env = { ...process.env, HOME: home }
874
- spawnSync(process.execPath, [CLI, 'skill', 'install'], { cwd, env, encoding: 'utf8' })
875
- const canonical = join(home, '.agents', 'skills', 'dingtalk-basic-behavior')
876
- const file = join(canonical, 'SKILL.md')
877
- writeFileSync(file, readFileSync(file, 'utf8') + '\n<!-- local edit -->\n')
878
- const statusRun = spawnSync(process.execPath, [CLI, 'skill', 'status', '--json'], {
879
- cwd, env, encoding: 'utf8',
880
- })
881
- const modified = JSON.parse(statusRun.stdout)
882
- const guardedUpgrade = spawnSync(process.execPath, [CLI, 'skill', 'upgrade'], {
883
- cwd, env, encoding: 'utf8',
884
- })
885
- const guardedUninstall = spawnSync(process.execPath, [CLI, 'skill', 'uninstall'], {
886
- cwd, env, encoding: 'utf8',
887
- })
888
- const repaired = spawnSync(process.execPath, [
889
- CLI, 'skill', 'upgrade', '--discard-local-changes', '--json',
890
- ], { cwd, env, encoding: 'utf8' })
891
- const repairedStatus = JSON.parse(repaired.stdout)
892
- const removed = spawnSync(process.execPath, [CLI, 'skill', 'uninstall', '--json'], {
893
- cwd, env, encoding: 'utf8',
894
- })
895
- const removedStatus = JSON.parse(removed.stdout)
896
- return {
897
- output: {
898
- modified, guardedUpgrade: guardedUpgrade.stderr.trim(),
899
- guardedUninstall: guardedUninstall.stderr.trim(), repairedStatus, removedStatus,
900
- },
901
- toolCalls: 6, steps: 7,
902
- checks: [
903
- check(modified.canonical.modified === true, JSON.stringify(modified.canonical)),
904
- check(guardedUpgrade.status !== 0 && guardedUpgrade.stderr.includes('本地修改'),
905
- guardedUpgrade.stderr.trim()),
906
- check(guardedUninstall.status !== 0 && guardedUninstall.stderr.includes('本地修改'),
907
- guardedUninstall.stderr.trim()),
908
- check(repaired.status === 0 && repairedStatus.canonical.current === true &&
909
- removed.status === 0 && removedStatus.canonical.exists === false &&
910
- removedStatus.claude.state === 'missing',
911
- `repaired=${repairedStatus.canonical.current}; removed=${removedStatus.canonical.exists}; ` +
912
- `claude=${removedStatus.claude.state}`),
913
- ],
914
- }
915
- }
916
-
917
- function evalRemoteStorageTypeGate() {
918
- const root = mkdtempSync(join(tmpdir(), 'dta-bootstrap-wrong-type-'))
919
- WORKSPACES.push(root)
920
- const bin = enableFakeDws(root)
921
- const run = spawnSync(process.execPath, [
922
- CLI, 'bootstrap', '--storage', 'dingtalk-doc:sheet-node',
923
- '--state-dir', join(root, '.state'), '--json',
924
- ], {
925
- cwd: root,
926
- env: {
927
- ...process.env,
928
- PATH: `${bin}:${process.env.PATH || ''}`,
929
- DTA_FAKE_DOC_EXTENSION: 'axls',
930
- },
931
- encoding: 'utf8',
932
- })
933
- return {
934
- output: { status: run.status, stderr: run.stderr.trim() }, toolCalls: 1, steps: 2,
935
- checks: [
936
- check(run.status !== 0 && run.stderr.includes('只接受在线文档 adoc'), run.stderr.trim()),
937
- check(!existsSync(join(root, '.state')), '类型不符时没有创建远端快照目录'),
938
- ],
939
- }
940
- }
941
-
942
- function evalDirectActGate() {
943
- const root = mkdtempSync(join(tmpdir(), 'dta-direct-act-'))
944
- WORKSPACES.push(root)
945
- const run = spawnSync(process.execPath, [CLI, 'act', 'reply', '--text', '越权回复'], {
946
- cwd: root, encoding: 'utf8',
947
- })
948
- return {
949
- output: { status: run.status, stderr: run.stderr.trim() }, toolCalls: 1, steps: 2,
950
- checks: [
951
- check(run.status !== 0 && run.stderr.includes('找不到当前 Run'), run.stderr.trim()),
952
- check(!existsSync(join(root, '.dingtalk-agent')), 'act fail closed 时不自动 init'),
953
- ],
954
- }
955
- }
956
-
957
- function evalDoctorMissingDependencies() {
958
- const home = mkdtempSync(join(tmpdir(), 'dta-doctor-missing-home-'))
959
- const cwd = mkdtempSync(join(tmpdir(), 'dta-doctor-missing-cwd-'))
960
- WORKSPACES.push(home, cwd)
961
- const run = spawnSync(process.execPath, [CLI, 'doctor', '--json'], {
962
- cwd,
963
- env: { ...process.env, HOME: home, PATH: '' },
964
- encoding: 'utf8',
965
- })
966
- const out = JSON.parse(run.stdout)
967
- const byId = Object.fromEntries(out.checks.map((check) => [check.id, check]))
968
- return {
969
- output: { status: run.status, report: out }, toolCalls: 1, steps: 3,
970
- checks: [
971
- check(run.status === 2 && out.ready === false && byId.cli.level === 'fail',
972
- `status=${run.status}; ready=${out.ready}; cli=${byId.cli.level}`),
973
- check(byId.dws.level === 'fail' && byId['dws-auth'].level === 'fail',
974
- `dws=${byId.dws.level}; auth=${byId['dws-auth'].level}`),
975
- check(out.nextSteps.some((step) => step.includes('npx --yes')) &&
976
- out.nextSteps.some((step) => step.includes('dws auth login')),
977
- JSON.stringify(out.nextSteps)),
978
- ],
979
- }
980
- }
981
-
982
- function evalFirstRunSetup() {
983
- const home = mkdtempSync(join(tmpdir(), 'dta-setup-home-'))
984
- const cwd = mkdtempSync(join(tmpdir(), 'dta-setup-cwd-'))
985
- WORKSPACES.push(home, cwd)
986
- const fakeBin = enableFakeDws(cwd)
987
- const env = {
988
- ...process.env,
989
- HOME: home,
990
- SHELL: '/bin/zsh',
991
- PATH: `${fakeBin}:${process.env.PATH || ''}`,
992
- DTA_SETUP_PACKAGE_SOURCE: ROOT,
993
- }
994
- const run = spawnSync(process.execPath, [CLI, 'setup', '--json'], {
995
- cwd, env, encoding: 'utf8', timeout: 120_000,
996
- })
997
- const out = run.stdout ? JSON.parse(run.stdout) : null
998
- const executable = join(home, '.local', 'bin', 'dingtalk-agent')
999
- const version = existsSync(executable)
1000
- ? spawnSync(executable, ['--version'], { env, encoding: 'utf8' })
1001
- : { status: -1, stdout: '' }
1002
- const rc = existsSync(join(home, '.zshrc')) ? readFileSync(join(home, '.zshrc'), 'utf8') : ''
1003
- const canonical = join(home, '.agents', 'skills', 'dingtalk-basic-behavior')
1004
- const claude = join(home, '.claude', 'skills', 'dingtalk-basic-behavior')
1005
- return {
1006
- output: {
1007
- status: run.status, stderr: run.stderr.trim(), report: out,
1008
- executable, version: String(version.stdout || '').trim(), rc,
1009
- },
1010
- toolCalls: 1, steps: 6,
1011
- checks: [
1012
- check(run.status === 0 && existsSync(executable) && version.status === 0,
1013
- `status=${run.status}; executable=${existsSync(executable)}; version=${version.status}`),
1014
- check(out?.shell.updated === true && rc.includes('dingtalk-agent PATH') &&
1015
- rc.includes('$HOME/.local/bin'),
1016
- `updated=${out?.shell.updated}; rc=${JSON.stringify(rc)}`),
1017
- check(existsSync(join(canonical, 'SKILL.md')) && lstatSync(claude).isSymbolicLink() &&
1018
- out?.skill.clients.every((client) => client.state === 'ok'),
1019
- `canonical=${existsSync(canonical)}; clients=${JSON.stringify(out?.skill.clients)}`),
1020
- check(out?.doctor.ready === true &&
1021
- out.doctor.checks.find((check) => check.id === 'dws')?.level === 'pass' &&
1022
- out.doctor.checks.find((check) => check.id === 'dws-auth')?.level === 'pass',
1023
- `ready=${out?.doctor.ready}; checks=${JSON.stringify(out?.doctor.checks)}`),
1024
- ],
1025
- }
1026
- }
1027
-
1028
- function dmEvent(eventId, messageId, actor, text, extra = {}) {
1029
- return {
1030
- event_corp_id: extra.tenantId || 'tenant-wait-eval',
1031
- data: JSON.stringify({
1032
- type: 'user_im_message_receive_o2o',
1033
- event_id: eventId,
1034
- conversation_id: extra.conversationId || 'cid-wait-eval',
1035
- message_id: messageId,
1036
- sender: actor,
1037
- sender_open_dingtalk_id: `$:LWCP_v1:$${actor}`,
1038
- content: text,
1039
- }),
1040
- }
1041
- }
1042
-
1043
- function enableFakeDws(root) {
1044
- const fieldPath = join(root, 'fields', 'default', 'field.json')
1045
- if (existsSync(fieldPath)) {
1046
- const field = readJson(fieldPath)
1047
- field.dws = { profile: 'eval-profile', expectedUserId: 'agent-eval' }
1048
- writeJson(fieldPath, field)
1049
- }
1050
-
1051
- const bin = join(root, '.fake-bin')
1052
- mkdirSync(bin, { recursive: true })
1053
- const script = join(bin, 'dws')
1054
- writeFileSync(script, `#!/usr/bin/env node
1055
- const a = process.argv.slice(2)
1056
- let data = {}
1057
- if (a[0] === 'version') { console.log(JSON.stringify({ version: 'v1.0.99-test' })); process.exit(0) }
1058
- if (a[0] === 'auth' && a[1] === 'status') { console.log(JSON.stringify({ authenticated: true, token_valid: true, corp_name: 'Test Corp', user_name: 'Test User' })); process.exit(0) }
1059
- if (a.includes('get-self')) data = { userId: 'agent-eval' }
1060
- else if (a.includes('doc') && a.includes('read')) data = { markdown: '# Remote Memory\\n\\n来自钉钉文档。' }
1061
- else if (a.includes('doc') && a.includes('info')) data = { contentType: 'ALIDOC', extension: process.env.DTA_FAKE_DOC_EXTENSION || 'adoc', nodeType: 'file' }
1062
- else if (a.includes('reply')) data = { openMessageId: 'fake-sent-message' }
1063
- else if (a.includes('list-by-ids')) data = { messages: [{ openMessageId: 'fake-sent-message' }] }
1064
- console.log(JSON.stringify({ data }))
1065
- `)
1066
- chmodSync(script, 0o755)
1067
- return bin
1068
- }
1069
-
1070
- function withFakeDws(root, fn) {
1071
- const before = process.env.PATH || ''
1072
- process.env.PATH = `${join(root, '.fake-bin')}:${before}`
1073
- try { return fn() } finally { process.env.PATH = before }
1074
- }
1075
-
1076
- function installWorkspaceSkillOverride(root) {
1077
- cpSync(
1078
- join(ROOT, 'skills', 'dingtalk-basic-behavior'),
1079
- join(root, 'skills', 'dingtalk-basic-behavior'),
1080
- { recursive: true },
1081
- )
1082
- }
1083
-
1084
- function workspace() {
1085
- const root = mkdtempSync(join(tmpdir(), 'dta-eval-'))
1086
- WORKSPACES.push(root)
1087
- const log = console.log
1088
- console.log = () => {}
1089
- try { init(root) } finally { console.log = log }
1090
- return { root, cfg: config.must(root) }
1091
- }
1092
-
1093
- function fixture(name) {
1094
- return JSON.parse(readFileSync(join(HERE, 'fixtures', name), 'utf8'))
1095
- }
1096
-
1097
- function sameSet(a, b) {
1098
- return a.length === b.length && a.every((x) => b.includes(x))
1099
- }
1100
-
1101
- function check(passed, evidence) { return { passed, evidence } }
1102
-
1103
- function writeJson(path, value) {
1104
- mkdirSync(dirname(path), { recursive: true })
1105
- writeFileSync(path, JSON.stringify(value, null, 2) + '\n')
1106
- }